diff --git a/.github/actions/checkout-release-source/action.yml b/.github/actions/checkout-release-source/action.yml new file mode 100644 index 00000000..300c8732 --- /dev/null +++ b/.github/actions/checkout-release-source/action.yml @@ -0,0 +1,26 @@ +name: Checkout complete release source +description: Avoid unshallowing a reused CI checkout before fetching release history. +inputs: + ref: + description: Exact source ref or commit accepted by the calling release job. + default: '' + fetch-depth: + description: History depth; release jobs require complete history. + default: '0' + persist-credentials: + description: Preserve the calling job's existing checkout credential policy. + default: 'true' +runs: + using: composite + steps: + - name: Retire only the inherited shallow CI Git metadata + uses: actions/github-script@v9 + env: + ZSTANDARD_CHECKOUT_BOOTSTRAP: ${{ github.action_path }}/retire-shallow-cache.cjs + with: + script: return require(process.env.ZSTANDARD_CHECKOUT_BOOTSTRAP)({core}); + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + fetch-depth: ${{ inputs.fetch-depth }} + persist-credentials: ${{ inputs.persist-credentials }} diff --git a/.github/actions/checkout-release-source/retire-shallow-cache.cjs b/.github/actions/checkout-release-source/retire-shallow-cache.cjs new file mode 100644 index 00000000..2f5e0f75 --- /dev/null +++ b/.github/actions/checkout-release-source/retire-shallow-cache.cjs @@ -0,0 +1,64 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const {execFileSync} = require('node:child_process'); + +module.exports = function retireShallowCache({core, env = process.env}) { + const repository = env.GITHUB_REPOSITORY || ''; + if (env.GITHUB_ACTIONS !== 'true' || !/^[\w.-]+\/[\w.-]+$/.test(repository)) { + throw new Error('Release checkout requires a GitHub Actions repository workspace.'); + } + const workspace = fs.realpathSync.native(env.GITHUB_WORKSPACE); + const temporary = fs.realpathSync.native(env.RUNNER_TEMP); + const name = repository.split('/')[1]; + const expected = path.join(path.dirname(temporary), name, name); + const equal = (a, b) => process.platform === 'win32' + ? a.toLowerCase() === b.toLowerCase() : a === b; + if (!equal(workspace, expected) || fs.lstatSync(env.GITHUB_WORKSPACE).isSymbolicLink()) { + throw new Error('Refusing to alter Git metadata outside the runner-owned repository workspace.'); + } + const metadata = path.join(workspace, '.git'); + let information; + try { information = fs.lstatSync(metadata); } + catch (error) { if (error.code === 'ENOENT') return {retired: false}; throw error; } + if (!information.isDirectory() || information.isSymbolicLink()) { + throw new Error('Refusing a linked Git directory or worktree.'); + } + const git = (...args) => execFileSync('git', ['-C', workspace, ...args], { + env: {...env, GIT_OPTIONAL_LOCKS: '0'}, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], + }).trim(); + if (git('rev-parse', '--is-shallow-repository') !== 'true') return {retired: false}; + const origin = new URL(git('config', '--get', 'remote.origin.url')); + if (origin.protocol !== 'https:' || origin.hostname !== 'github.com' + || origin.username || origin.password + || origin.pathname.replace(/\.git$/, '').replace(/^\//, '').toLowerCase() !== repository.toLowerCase()) { + throw new Error('The shallow cache does not belong to the calling GitHub repository.'); + } + for (const name of ['worktrees', 'modules']) { + const directory = path.join(metadata, name); + if (fs.existsSync(directory) && fs.readdirSync(directory).length) { + throw new Error('Retaining Git metadata with registered worktrees or submodules.'); + } + } + for (const name of ['index.lock', 'shallow.lock', 'HEAD.lock']) { + if (fs.existsSync(path.join(metadata, name))) throw new Error('Retaining locked Git metadata.'); + } + try { git('diff', '--quiet'); git('diff', '--cached', '--quiet'); } + catch { throw new Error('Could not verify clean tracked source; retaining the CI Git metadata.'); } + if (git('ls-files', '--others', '--exclude-standard') || git('for-each-ref', '--format=%(objectname)', 'refs/stash')) { + throw new Error('Retaining unexpected untracked files or stashed work in the CI workspace.'); + } + const publicRefs = new Set(git('for-each-ref', '--format=%(objectname)', 'refs/remotes', 'refs/tags').split('\n')); + const branches = git('for-each-ref', '--format=%(objectname)', 'refs/heads').split('\n').filter(Boolean); + if (branches.some(oid => !publicRefs.has(oid))) { + throw new Error('Retaining a local branch that is not represented by the cached remote refs.'); + } + const backup = fs.mkdtempSync(path.join(temporary, 'zstandard-shallow-git-')); + fs.chmodSync(backup, 0o700); + fs.writeFileSync(path.join(backup, '.owner'), JSON.stringify({repository, run: env.GITHUB_RUN_ID, + attempt: env.GITHUB_RUN_ATTEMPT, head: git('rev-parse', 'HEAD')}), {mode: 0o600}); + fs.renameSync(metadata, path.join(backup, 'git')); + core.notice('Retired shallow CI Git metadata to private job temporary storage; checkout will fetch complete history.'); + return {retired: true}; +}; diff --git a/.github/actions/copilot-verification-setup/action.yml b/.github/actions/copilot-verification-setup/action.yml index 35405515..2c990f67 100644 --- a/.github/actions/copilot-verification-setup/action.yml +++ b/.github/actions/copilot-verification-setup/action.yml @@ -1,13 +1,40 @@ -# Created once by Copilot setup. This action and its assets belong to your project. -# Replace the placeholder with reviewed runtime/dependency setup, then merge onto MAIN_BRANCH. +# Project-owned dependency setup, executed by Copilot's existing coordinator. name: Prepare project verification -description: Configure project dependencies before enabling VERIFICATION_SETUP. +description: Provision the tested Flutter/Dart SDK before project verification. runs: using: composite steps: - - name: Configure project verification dependencies + - name: Select private package and configuration paths shell: bash run: | - printf '%s\n' 'Configure .github/actions/copilot-verification-setup/action.yml with your project dependencies before enabling VERIFICATION_SETUP.' - printf '%s\n' 'Replace this failing placeholder, review and merge it onto the configured MAIN_BRANCH, then run a real verification.' - exit 1 + set -euo pipefail + mkdir -p "$HOME/.pub-cache/bin" "$HOME/.config" + printf 'PUB_CACHE=%s\nXDG_CONFIG_HOME=%s\n' "$HOME/.pub-cache" "$HOME/.config" >> "$GITHUB_ENV" + - name: Install the tested Flutter SDK + uses: subosito/flutter-action@v2 + with: + channel: stable + flutter-version: '3.47.2' + architecture: ${{ runner.arch == 'ARM64' && 'arm64' || 'x64' }} + cache-path: '${{ runner.tool_cache }}/zstandard-flutter-v2/:channel:-:version:-:arch:' + cache: false + pub-cache: false + - name: Prepare private Linux test dependencies + if: runner.os == 'Linux' + shell: bash + env: + DESKTOP: 'true' + COVERAGE: 'false' + ANDROID: 'false' + ARM64: 'false' + BROWSER: 'false' + run: ZSTANDARD_LINUX_STATE_ROOT="$HOME" bash "$GITHUB_ACTION_PATH/linux-dependencies.sh" + - name: Preserve native tool settings through the supported PATH handoff + shell: bash + run: bash "$GITHUB_ACTION_PATH/runtime-launchers.sh" + - name: Verify the provisioned SDK + shell: bash + run: | + set -euo pipefail + flutter --version + dart --version diff --git a/.github/actions/copilot-verification-setup/linux-dependencies.sh b/.github/actions/copilot-verification-setup/linux-dependencies.sh new file mode 100644 index 00000000..47d94fc7 --- /dev/null +++ b/.github/actions/copilot-verification-setup/linux-dependencies.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Shared private Linux dependencies for platform CI and project verification. +set -euo pipefail +command -v apt-get >/dev/null +command -v apt-cache >/dev/null +command -v dpkg-query >/dev/null +packages=(build-essential clang cmake coreutils ninja-build pkg-config curl unzip) +browser_missing=() +if [[ "$ANDROID" == 'true' ]]; then + packages+=(libdbus-1-3 libfontconfig1 libgl1 libnss3 libpulse0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxi6 libxrandr2 libxrender1 libxtst6) +fi +if [[ "$BROWSER" == 'true' ]]; then + packages+=(ca-certificates libdbus-1-3 libdrm2 libgbm1 libnspr4 libnss3 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 libxext6 libxfixes3 libxkbcommon0 libxrandr2) + # Ubuntu/Debian releases before the time64 transition use the base + # names. Reuse installed variants or resolve against private indexes. + for legacy in libasound2 libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libcups2; do + selected='' + for candidate in "${legacy}t64" "$legacy"; do + if [[ "$(dpkg-query -W -f='${Status}' "$candidate" 2>/dev/null || true)" == 'install ok installed' ]]; then + selected="$candidate" + break + fi + done + if [[ -n "$selected" ]]; then packages+=("$selected"); else browser_missing+=("$legacy"); fi + done +fi +if [[ "$DESKTOP" == 'true' ]]; then packages+=(libgtk-3-dev xvfb); fi +if [[ "$COVERAGE" == 'true' ]]; then packages+=(bc lcov); fi +if [[ "$ARM64" == 'true' ]]; then packages+=(gcc-aarch64-linux-gnu g++-aarch64-linux-gnu); fi +missing=() +for package in "${packages[@]}"; do + if [[ "$(dpkg-query -W -f='${Status}' "$package" 2>/dev/null || true)" != 'install ok installed' ]]; then + missing+=("$package") + fi +done +if [[ "${#missing[@]}" -eq 0 && "${#browser_missing[@]}" -eq 0 ]]; then + echo 'All selected Linux prerequisites are already installed.' + exit 0 +fi +state_root="${ZSTANDARD_LINUX_STATE_ROOT:-$RUNNER_TEMP}" +state="$(mktemp -d "$state_root/zstandard-linux-tools.XXXXXX")" +mkdir -p "$state/apt/lists/partial" "$state/packages/partial" "$state/prefix" +cp /var/lib/dpkg/status "$state/apt/status" +apt_options=(-o "Dir::State::status=$state/apt/status" -o "Dir::State::Lists=$state/apt/lists" -o "Dir::Cache::archives=$state/packages" -o "Dir::Cache::pkgcache=$state/apt/pkgcache.bin" -o "Dir::Cache::srcpkgcache=$state/apt/srcpkgcache.bin" -o Acquire::Retries=0 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30) +apt-get "${apt_options[@]}" update +for legacy in "${browser_missing[@]}"; do + apt_candidate="$(LC_ALL=C apt-cache "${apt_options[@]}" policy "${legacy}t64" | sed -n 's/^ Candidate: //p')" + if [[ -n "$apt_candidate" && "$apt_candidate" != '(none)' ]]; then + missing+=("${legacy}t64") + else + missing+=("$legacy") + fi +done +apt-get "${apt_options[@]}" --download-only --assume-yes --no-install-recommends install "${missing[@]}" +for package in "$state/packages/"*.deb; do + [[ -f "$package" ]] || continue + dpkg-deb --extract "$package" "$state/prefix" +done +prefix="$state/prefix" +# pkg-config needs both extracted and preinstalled headers/libraries. +# Fill only missing private paths with links; never traverse a target link. +overlay_system() { + local source="$1" target="$2" child destination + [[ -d "$source" && ! -L "$source" && ! -L "$target" ]] || return 0 + mkdir -p "$target" + for child in "$source/"*; do + [[ -e "$child" || -L "$child" ]] || continue + destination="$target/${child##*/}" + if [[ -L "$destination" ]]; then + continue + elif [[ -d "$destination" && -d "$child" && ! -L "$child" ]]; then + overlay_system "$child" "$destination" + elif [[ ! -e "$destination" ]]; then + ln -s "$child" "$destination" + fi + done +} +case "$(dpkg --print-architecture)" in + amd64) multiarch=x86_64-linux-gnu ;; + arm64) multiarch=aarch64-linux-gnu ;; + *) echo 'Unsupported Linux runner architecture.' >&2; exit 1 ;; +esac +overlay_system /usr/include "$prefix/usr/include" +overlay_system "/usr/lib/$multiarch" "$prefix/usr/lib/$multiarch" +overlay_system /usr/share/pkgconfig "$prefix/usr/share/pkgconfig" +for directory in lib lib64 bin sbin; do + if [[ ! -e "$prefix/$directory" && ! -L "$prefix/$directory" ]]; then + ln -s "usr/$directory" "$prefix/$directory" + fi +done +overlay_system "/lib/$multiarch" "$prefix/lib/$multiarch" +# GNU linker scripts carry absolute development-library paths. Relocate +# only regular private scripts; shared host libraries remain read-only. +while IFS= read -r -d '' linker_script; do + if grep -Iq '^/\* GNU ld script' "$linker_script"; then + ZSTANDARD_LINUX_PREFIX="$prefix" /usr/bin/perl -0pi -e ' + s{(?> "$GITHUB_ENV" +printf '%s/usr/bin\n' "$prefix" >> "$GITHUB_PATH" diff --git a/.github/actions/copilot-verification-setup/runtime-launchers.sh b/.github/actions/copilot-verification-setup/runtime-launchers.sh new file mode 100644 index 00000000..495c4b39 --- /dev/null +++ b/.github/actions/copilot-verification-setup/runtime-launchers.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Copilot forwards validated PATH entries; keep native tool configuration there. +set -euo pipefail + +runtime_bin="$HOME/.zstandard-verification-runtime/bin" +mkdir -p "$runtime_bin" +for tool in flutter dart; do + executable="$(command -v "$tool")" + test -x "$executable" + { + printf '#!/usr/bin/env bash\nset -euo pipefail\n' + for setting in CPATH LIBRARY_PATH PKG_CONFIG_PATH PKG_CONFIG_SYSROOT_DIR CMAKE_PREFIX_PATH PUB_CACHE XDG_CONFIG_HOME; do + value="${!setting-}" + if [[ -n "$value" ]]; then + printf 'export %s=%q\n' "$setting" "$value" + fi + done + # The verifier may add the freshly built test plugin to its own search path. + if [[ -n "${LD_LIBRARY_PATH:-}" ]]; then + printf 'export LD_LIBRARY_PATH=%q"${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"\n' "$LD_LIBRARY_PATH" + fi + printf 'exec %q "$@"\n' "$executable" + } > "$runtime_bin/$tool" + chmod 700 "$runtime_bin/$tool" +done +printf '%s\n' "$runtime_bin" >> "$GITHUB_PATH" diff --git a/.github/actions/preserve-flutter-project-files/action.yml b/.github/actions/preserve-flutter-project-files/action.yml index c7c9207f..663012e3 100644 --- a/.github/actions/preserve-flutter-project-files/action.yml +++ b/.github/actions/preserve-flutter-project-files/action.yml @@ -1,5 +1,5 @@ name: Preserve Flutter project files -description: Preserve files that Flutter may rewrite during dependency resolution. +description: Preserve analysis options and tracked Apple host files that Flutter may rewrite. inputs: package: @@ -12,7 +12,7 @@ inputs: runs: using: composite steps: - - name: Save or restore analysis options + - name: Save or restore Flutter project files shell: bash env: PACKAGE: ${{ inputs.package }} @@ -21,6 +21,14 @@ runs: set -euo pipefail BACKUP_DIR="$RUNNER_TEMP/flutter-project-files/${PACKAGE//\//_}" files=("analysis_options.yaml" "example/analysis_options.yaml") + case "$PACKAGE" in + zstandard_ios|zstandard_macos) + platform="${PACKAGE#zstandard_}" + while IFS= read -r -d '' file; do + files+=("${file#"$PACKAGE/"}") + done < <(git ls-files -z -- "$PACKAGE/example/$platform/Runner.xcodeproj") + ;; + esac if [[ "$MODE" == 'save' ]]; then mkdir -p "$BACKUP_DIR" @@ -33,6 +41,7 @@ runs: elif [[ "$MODE" == 'restore' ]]; then for file in "${files[@]}"; do if [[ -f "$BACKUP_DIR/$file" ]]; then + mkdir -p "$PACKAGE/$(dirname "$file")" cp "$BACKUP_DIR/$file" "$PACKAGE/$file" fi done diff --git a/.github/actions/publish-package/action.yml b/.github/actions/publish-package/action.yml new file mode 100644 index 00000000..b4f86c16 --- /dev/null +++ b/.github/actions/publish-package/action.yml @@ -0,0 +1,41 @@ +name: Publish one immutable package +description: Renew OIDC for each package and resolve only the distributable package. +inputs: + package: + required: true + description: Name of the matched release package. + version: + required: true + description: Accepted stable release version. + dart-version: + required: true + description: Exact Dart SDK version bundled with the tested Flutter SDK. +runs: + using: composite + steps: + # Each package gets a fresh credential after the previous indexing wait. + - uses: dart-lang/setup-dart@v1 + with: + sdk: ${{ inputs.dart-version }} + # A nested SDK action inherits this composite's path, which has no + # analyzer matcher. Publication keeps its dry runs and CI gates. + problem-matcher: false + - name: Resolve, publish and wait for the exact version + shell: bash + env: + ZSTANDARD_PUBLISH_PACKAGE: ${{ inputs.package }} + ZSTANDARD_PUBLISH_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + case "$ZSTANDARD_PUBLISH_PACKAGE" in + zstandard|zstandard_platform_interface|zstandard_native|zstandard_android|zstandard_ios|zstandard_linux|zstandard_macos|zstandard_web|zstandard_windows|zstandard_cli) ;; + *) echo 'Unknown release package.' >&2; exit 2 ;; + esac + case "$ZSTANDARD_PUBLISH_PACKAGE" in + zstandard_native|zstandard_cli) + (cd "$ZSTANDARD_PUBLISH_PACKAGE" && dart pub get --no-example) ;; + *) + (cd "$ZSTANDARD_PUBLISH_PACKAGE" && flutter pub get --no-example) ;; + esac + dart run scripts/publish_if_missing.dart "$ZSTANDARD_PUBLISH_PACKAGE" "$ZSTANDARD_PUBLISH_VERSION" + dart run scripts/wait_for_pub_version.dart "$ZSTANDARD_PUBLISH_PACKAGE" "$ZSTANDARD_PUBLISH_VERSION" 1200 diff --git a/.github/actions/setup-flutter-windows/action.yml b/.github/actions/setup-flutter-windows/action.yml index fd464415..a31e8f32 100644 --- a/.github/actions/setup-flutter-windows/action.yml +++ b/.github/actions/setup-flutter-windows/action.yml @@ -47,6 +47,34 @@ runs: Write-Host 'Removing the incomplete repository-specific Flutter installation before preparing the SDK.' Remove-Item -LiteralPath $sdkDirectory -Recurse -Force } + - name: Expose installed CMake and CTest + shell: powershell + run: | + $ErrorActionPreference = 'Stop' + $directories = @( + (Join-Path $env:ProgramFiles 'CMake/bin'), + (Join-Path ${env:ProgramFiles(x86)} 'CMake/bin') + ) + $command = Get-Command cmake.exe -CommandType Application -ErrorAction SilentlyContinue + if ($command) { $directories = @((Split-Path $command.Source)) + $directories } + $vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio/Installer/vswhere.exe' + if (Test-Path -LiteralPath $vswhere) { + $installations = & $vswhere -products * -format json | ConvertFrom-Json + foreach ($installation in ($installations | Sort-Object { [Version]$_.installationVersion } -Descending)) { + $directories += Join-Path $installation.installationPath 'Common7/IDE/CommonExtensions/Microsoft/CMake/CMake/bin' + } + } + $cmakeDirectory = $directories | Where-Object { + (Test-Path -LiteralPath (Join-Path $_ 'cmake.exe')) -and + (Test-Path -LiteralPath (Join-Path $_ 'ctest.exe')) + } | Select-Object -First 1 + if (-not $cmakeDirectory) { throw 'Install CMake and CTest, or the Visual Studio C++ CMake tools component.' } + $cmakeDirectory | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + Write-Host "Using CMake and CTest from $cmakeDirectory" + & (Join-Path $cmakeDirectory 'cmake.exe') --version + if ($LASTEXITCODE -ne 0) { throw 'The installed CMake could not run.' } + & (Join-Path $cmakeDirectory 'ctest.exe') --version + if ($LASTEXITCODE -ne 0) { throw 'The installed CTest could not run.' } - name: Prepare Flutter SDK uses: subosito/flutter-action@v2 with: diff --git a/.github/actions/setup-flutter/action.yml b/.github/actions/setup-flutter/action.yml index 788aed4b..856ab63a 100644 --- a/.github/actions/setup-flutter/action.yml +++ b/.github/actions/setup-flutter/action.yml @@ -21,23 +21,29 @@ runs: cache: false pub-cache: false - - name: Isolate Flutter settings for concurrent Unix jobs + - name: Isolate Flutter and Dart settings for concurrent Unix jobs if: runner.os != 'Windows' shell: bash run: | set -euo pipefail state="$(mktemp -d "$RUNNER_TEMP/zstandard-flutter-profile.XXXXXX")" mkdir -p "$state/home" "$state/bin" - cat > "$state/bin/flutter" <<'WRAPPER' - #!/bin/bash - set -euo pipefail - exec env HOME="$ZSTANDARD_FLUTTER_SETTINGS_HOME" \ - XDG_CONFIG_HOME="$ZSTANDARD_FLUTTER_SETTINGS_HOME/.config" \ - "$FLUTTER_ROOT/bin/flutter" "$@" - WRAPPER - chmod 700 "$state/bin/flutter" + for tool in flutter dart; do + { + printf '#!/bin/bash\nset -euo pipefail\n' + printf 'sdk_root=%q\n' "$FLUTTER_ROOT" + # iOS validation selects a disposable SDK containing its backport. + printf 'sdk_root="${FLUTTER_ROOT:-$sdk_root}"\n' + printf 'exec env HOME=%q XDG_CONFIG_HOME=%q "$sdk_root/bin/%s" "$@"\n' \ + "$state/home" "$state/home/.config" "$tool" + } > "$state/bin/$tool" + chmod 700 "$state/bin/$tool" + done { printf 'ZSTANDARD_FLUTTER_SETTINGS_HOME=%s\n' "$state/home" + # setup-dart invokes its SDK directly when provisioning OIDC. Isolate + # its token-reference store too, while retaining the package cache. + printf 'XDG_CONFIG_HOME=%s\n' "$state/home/.config" printf 'PUB_CACHE=%s\n' "${PUB_CACHE:-$HOME/.pub-cache}" } >> "$GITHUB_ENV" printf '%s\n' "$state/bin" >> "$GITHUB_PATH" diff --git a/.github/actions/setup-linux-dependencies/action.yml b/.github/actions/setup-linux-dependencies/action.yml index 67866ba3..6be9dcba 100644 --- a/.github/actions/setup-linux-dependencies/action.yml +++ b/.github/actions/setup-linux-dependencies/action.yml @@ -35,131 +35,4 @@ runs: ARM64: ${{ inputs.arm64 }} BROWSER: ${{ inputs.browser }} run: | - set -euo pipefail - command -v apt-get >/dev/null - command -v apt-cache >/dev/null - command -v dpkg-query >/dev/null - packages=(build-essential clang cmake coreutils ninja-build pkg-config curl unzip) - browser_missing=() - if [[ "$ANDROID" == 'true' ]]; then - packages+=(libdbus-1-3 libfontconfig1 libgl1 libnss3 libpulse0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxi6 libxrandr2 libxrender1 libxtst6) - fi - if [[ "$BROWSER" == 'true' ]]; then - packages+=(ca-certificates libdbus-1-3 libdrm2 libgbm1 libnspr4 libnss3 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 libxext6 libxfixes3 libxkbcommon0 libxrandr2) - # Ubuntu/Debian releases before the time64 transition use the base - # names. Reuse installed variants or resolve against private indexes. - for legacy in libasound2 libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libcups2; do - selected='' - for candidate in "${legacy}t64" "$legacy"; do - if [[ "$(dpkg-query -W -f='${Status}' "$candidate" 2>/dev/null || true)" == 'install ok installed' ]]; then - selected="$candidate" - break - fi - done - if [[ -n "$selected" ]]; then packages+=("$selected"); else browser_missing+=("$legacy"); fi - done - fi - if [[ "$DESKTOP" == 'true' ]]; then packages+=(libgtk-3-dev xvfb); fi - if [[ "$COVERAGE" == 'true' ]]; then packages+=(bc lcov); fi - if [[ "$ARM64" == 'true' ]]; then packages+=(gcc-aarch64-linux-gnu g++-aarch64-linux-gnu); fi - missing=() - for package in "${packages[@]}"; do - if [[ "$(dpkg-query -W -f='${Status}' "$package" 2>/dev/null || true)" != 'install ok installed' ]]; then - missing+=("$package") - fi - done - if [[ "${#missing[@]}" -eq 0 && "${#browser_missing[@]}" -eq 0 ]]; then - echo 'All selected Linux prerequisites are already installed.' - exit 0 - fi - state="$(mktemp -d "$RUNNER_TEMP/zstandard-linux-tools.XXXXXX")" - mkdir -p "$state/apt/lists/partial" "$state/packages/partial" "$state/prefix" - cp /var/lib/dpkg/status "$state/apt/status" - apt_options=(-o "Dir::State::status=$state/apt/status" -o "Dir::State::Lists=$state/apt/lists" -o "Dir::Cache::archives=$state/packages" -o "Dir::Cache::pkgcache=$state/apt/pkgcache.bin" -o "Dir::Cache::srcpkgcache=$state/apt/srcpkgcache.bin" -o Acquire::Retries=0 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30) - apt-get "${apt_options[@]}" update - for legacy in "${browser_missing[@]}"; do - apt_candidate="$(LC_ALL=C apt-cache "${apt_options[@]}" policy "${legacy}t64" | sed -n 's/^ Candidate: //p')" - if [[ -n "$apt_candidate" && "$apt_candidate" != '(none)' ]]; then - missing+=("${legacy}t64") - else - missing+=("$legacy") - fi - done - apt-get "${apt_options[@]}" --download-only --assume-yes --no-install-recommends install "${missing[@]}" - for package in "$state/packages/"*.deb; do - [[ -f "$package" ]] || continue - dpkg-deb --extract "$package" "$state/prefix" - done - prefix="$state/prefix" - # pkg-config needs both extracted and preinstalled headers/libraries. - # Fill only missing private paths with links; never traverse a target link. - overlay_system() { - local source="$1" target="$2" child destination - [[ -d "$source" && ! -L "$source" && ! -L "$target" ]] || return 0 - mkdir -p "$target" - for child in "$source/"*; do - [[ -e "$child" || -L "$child" ]] || continue - destination="$target/${child##*/}" - if [[ -L "$destination" ]]; then - continue - elif [[ -d "$destination" && -d "$child" && ! -L "$child" ]]; then - overlay_system "$child" "$destination" - elif [[ ! -e "$destination" ]]; then - ln -s "$child" "$destination" - fi - done - } - case "$(dpkg --print-architecture)" in - amd64) multiarch=x86_64-linux-gnu ;; - arm64) multiarch=aarch64-linux-gnu ;; - *) echo 'Unsupported Linux runner architecture.' >&2; exit 1 ;; - esac - overlay_system /usr/include "$prefix/usr/include" - overlay_system "/usr/lib/$multiarch" "$prefix/usr/lib/$multiarch" - overlay_system /usr/share/pkgconfig "$prefix/usr/share/pkgconfig" - for directory in lib lib64 bin sbin; do - if [[ ! -e "$prefix/$directory" && ! -L "$prefix/$directory" ]]; then - ln -s "usr/$directory" "$prefix/$directory" - fi - done - overlay_system "/lib/$multiarch" "$prefix/lib/$multiarch" - # GNU linker scripts carry absolute development-library paths. Relocate - # only regular private scripts; shared host libraries remain read-only. - while IFS= read -r -d '' linker_script; do - if grep -Iq '^/\* GNU ld script' "$linker_script"; then - ZSTANDARD_LINUX_PREFIX="$prefix" /usr/bin/perl -0pi -e ' - s{(?> "$GITHUB_ENV" - printf '%s/usr/bin\n' "$prefix" >> "$GITHUB_PATH" + bash "$GITHUB_ACTION_PATH/../copilot-verification-setup/linux-dependencies.sh" diff --git a/.github/scripts/check_action_versions.py b/.github/scripts/check_action_versions.py index 192ab221..0afb450f 100644 --- a/.github/scripts/check_action_versions.py +++ b/.github/scripts/check_action_versions.py @@ -29,7 +29,7 @@ def main(root): for path in paths: for reference in references(yaml.safe_load(path.read_text(encoding="utf-8"))): count += 1 - if isinstance(reference, str) and reference.startswith("./"): + if isinstance(reference, str) and reference.startswith(("./", "$/.github/actions/")): continue if not isinstance(reference, str) or "@" not in reference or not VERSION.fullmatch(reference.rsplit("@", 1)[-1]): errors.append(f"{path.relative_to(root)}: {reference}: use a version tag such as @v1") diff --git a/.github/scripts/tests/test_flutter_profile.py b/.github/scripts/tests/test_flutter_profile.py new file mode 100644 index 00000000..39a3032b --- /dev/null +++ b/.github/scripts/tests/test_flutter_profile.py @@ -0,0 +1,123 @@ +"""Use real Dart token listing to verify private CI configuration without uploads.""" + +import json +import os +from pathlib import Path +import platform +import shlex +import shutil +import subprocess +import tempfile +import textwrap +import unittest + + +ROOT = Path(__file__).resolve().parents[3] +DART = shutil.which('dart') +if DART: + candidates = [Path(os.environ.get('FLUTTER_ROOT', '')) / 'bin/cache/dart-sdk/bin/dart', + Path(DART).resolve().parent / 'cache/dart-sdk/bin/dart'] + DART = next((str(path) for path in candidates if path.is_file()), DART) + + +@unittest.skipUnless(DART and os.name == 'posix', 'Unix Dart SDK is required') +class FlutterProfileTests(unittest.TestCase): + def setUp(self): + self.folder = tempfile.TemporaryDirectory(prefix='zstd-sdk-profile-') + self.addCleanup(self.folder.cleanup) + self.root = Path(self.folder.name).resolve() + self.operator = self.root / 'operator home' + self.operator.mkdir() + sdk = self.root / 'sdk with spaces' + (sdk / 'bin').mkdir(parents=True) + (sdk / 'bin/dart').write_text('#!/bin/bash\nexec ' + shlex.quote(DART) + ' "$@"\n') + (sdk / 'bin/dart').chmod(0o700) + (sdk / 'bin/flutter').write_text('#!/bin/bash\nprintf "original-flutter\\n"\n') + (sdk / 'bin/flutter').chmod(0o700) + self.environment = {'PATH': os.environ['PATH'], 'HOME': str(self.operator), + 'XDG_CONFIG_HOME': str(self.operator / '.config'), + 'PUB_CACHE': str(self.root / 'shared package cache'), + 'FLUTTER_ROOT': str(sdk), 'RUNNER_TEMP': str(self.root), + 'GITHUB_ENV': str(self.root / 'github-env'), + 'GITHUB_PATH': str(self.root / 'github-path')} + for name in ['GITHUB_ENV', 'GITHUB_PATH']: + Path(self.environment[name]).touch() + self.original_store = self.token_file(self.operator, self.environment['XDG_CONFIG_HOME']) + self.write_reference(self.original_store, 'https://operator-fixture.invalid') + self.original_bytes = self.original_store.read_bytes() + self.assertIn('operator-fixture.invalid', self.list_tokens(DART, self.environment)) + + @staticmethod + def token_file(home, config): + base = home / 'Library/Application Support' if platform.system() == 'Darwin' else Path(config) + return base / 'dart/pub-tokens.json' + + @staticmethod + def write_reference(path, url): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({'version': 1, 'hosted': [{'url': url, 'env': 'MISSING_FIXTURE_TOKEN'}]})) + + @staticmethod + def list_tokens(executable, environment): + result = subprocess.run([executable, 'pub', 'token', 'list'], env=environment, + capture_output=True, text=True, timeout=15) + if result.returncode: + raise AssertionError(result.stdout + result.stderr) + return result.stdout + + def prepare(self): + source = (ROOT / '.github/actions/setup-flutter/action.yml').read_text() + self.assertEqual(source.count(' run: |\n'), 1) + script = textwrap.dedent(source.split(' run: |\n', 1)[1]) + subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', script], + env=self.environment, check=True, timeout=15) + environment = dict(self.environment) + environment.update(dict(line.split('=', 1) for line in Path(environment['GITHUB_ENV']).read_text().splitlines())) + wrapper = Path(Path(environment['GITHUB_PATH']).read_text().strip()) / 'dart' + return wrapper, environment + + def test_dart_wrapper_uses_a_private_profile_and_preserves_the_shared_archive_cache(self): + wrapper, environment = self.prepare() + self.assertNotIn('operator-fixture.invalid', self.list_tokens(str(wrapper), environment)) + private_home = Path(environment['ZSTANDARD_FLUTTER_SETTINGS_HOME']) + self.write_reference(self.token_file(private_home, environment['XDG_CONFIG_HOME']), + 'https://private-fixture.invalid') + self.assertIn('private-fixture.invalid', self.list_tokens(str(wrapper), environment)) + # Configured repository checks deliberately retain PATH only. + self.assertIn('private-fixture.invalid', self.list_tokens(str(wrapper), {'PATH': os.environ['PATH']})) + self.assertEqual(environment['PUB_CACHE'], self.environment['PUB_CACHE']) + self.assertEqual(self.original_store.read_bytes(), self.original_bytes) + + def test_disposable_ios_sdk_override_is_honored_with_a_path_only_fallback(self): + wrapper, environment = self.prepare() + patched = self.root / 'patched SDK with spaces' + (patched / 'bin').mkdir(parents=True) + for tool in ['flutter', 'dart']: + executable = patched / 'bin' / tool + executable.write_text('#!/bin/bash\nprintf "patched-' + tool + '\\n"\n') + executable.chmod(0o700) + result = subprocess.run([str(wrapper.with_name(tool)), '--version'], + env={**environment, 'FLUTTER_ROOT': str(patched)}, + capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), 'patched-' + tool) + fallback = subprocess.run([str(wrapper.with_name('flutter')), '--version'], + env={'PATH': os.environ['PATH']}, + capture_output=True, text=True, timeout=10) + self.assertEqual(fallback.returncode, 0, fallback.stderr) + self.assertEqual(fallback.stdout.strip(), 'original-flutter') + + @unittest.skipUnless(platform.system() == 'Linux', 'The publisher uses Linux XDG configuration') + def test_direct_sdk_invocation_uses_the_same_private_token_store(self): + wrapper, environment = self.prepare() + private_home = Path(environment['ZSTANDARD_FLUTTER_SETTINGS_HOME']) + self.write_reference(self.token_file(private_home, environment['XDG_CONFIG_HOME']), + 'https://private-fixture.invalid') + direct = self.list_tokens(DART, environment) + self.assertIn('private-fixture.invalid', direct) + self.assertNotIn('operator-fixture.invalid', direct) + self.assertEqual(self.original_store.read_bytes(), self.original_bytes) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/scripts/tests/test_flutter_project_files.py b/.github/scripts/tests/test_flutter_project_files.py new file mode 100644 index 00000000..57ba3b67 --- /dev/null +++ b/.github/scripts/tests/test_flutter_project_files.py @@ -0,0 +1,117 @@ +"""Run real save/restore steps against tracked Apple host fixtures without Flutter.""" + +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +import yaml + +ROOT = Path(__file__).resolve().parents[3] +ACTION = './.github/actions/preserve-flutter-project-files' + + +class FlutterProjectFilesTests(unittest.TestCase): + def setUp(self): + self.folder = tempfile.TemporaryDirectory(prefix='zstd-project-files-') + self.addCleanup(self.folder.cleanup) + self.root = Path(self.folder.name) / 'workspace with spaces' + self.root.mkdir() + self.temporary = Path(self.folder.name) / 'private state' + self.temporary.mkdir() + self.environment = {key: value for key, value in os.environ.items() + if not key.startswith('GIT_')} + self.environment.update(RUNNER_TEMP=str(self.temporary), BASH_ENV=os.devnull, + GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_NOSYSTEM='1') + self.git('init', '-q') + action = yaml.safe_load((ROOT / ACTION.removeprefix('./') / 'action.yml').read_text()) + self.script = action['runs']['steps'][0]['run'] + + def git(self, *args): + subprocess.run(['git', *args], cwd=self.root, env=self.environment, + capture_output=True, check=True, timeout=10) + + def write(self, name, content): + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + return path + + def run_action(self, package, mode): + result = subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', self.script], + cwd=self.root, env={**self.environment, 'PACKAGE': package, + 'MODE': mode}, capture_output=True, + text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_apple_migrations_restore_exact_starting_files_and_preserve_other_work(self): + for platform in ['ios', 'macos']: + with self.subTest(platform=platform): + package = 'zstandard_' + platform + project = f'{package}/example/{platform}/Runner.xcodeproj' + paths = [f'{package}/analysis_options.yaml', + f'{package}/example/analysis_options.yaml', + project + '/project.pbxproj', + project + '/xcshareddata/xcschemes/Runner.xcscheme', + project + '/xcshareddata/settings with spaces.xml'] + for name in paths: + self.write(name, 'original ' + name) + self.git('add', '--', package) + # Preserve the actual starting worktree, including authored edits, + # rather than replacing it with an assumed HEAD/index baseline. + self.write(paths[2], 'authored baseline before Flutter ran') + originals = {name: (self.root / name).read_bytes() for name in paths} + self.run_action(package, 'save') + for name in paths: + self.write(name, 'Flutter migration changed ' + name) + scheme = self.root / paths[3] + scheme.unlink() + scheme.parent.rmdir() + untracked = self.write(project + '/untracked-note.txt', 'retain untracked work') + unrelated = self.write(package + '/lib/authored.dart', 'retain unrelated work') + self.run_action(package, 'restore') + for name, content in originals.items(): + self.assertEqual((self.root / name).read_bytes(), content) + self.assertEqual(untracked.read_text(), 'retain untracked work') + self.assertEqual(unrelated.read_text(), 'retain unrelated work') + + def test_restore_without_a_snapshot_leaves_source_untouched(self): + path = self.write('zstandard_ios/analysis_options.yaml', 'keep me') + self.git('add', '--', 'zstandard_ios') + self.run_action('zstandard_ios', 'restore') + self.assertEqual(path.read_text(), 'keep me') + self.assertFalse((self.temporary / 'flutter-project-files').exists()) + + def test_non_apple_packages_keep_the_existing_analysis_only_scope(self): + analysis = self.write('zstandard_web/analysis_options.yaml', 'original analysis') + source = self.write('zstandard_web/lib/source.dart', 'original source') + self.git('add', '--', 'zstandard_web') + self.run_action('zstandard_web', 'save') + analysis.write_text('Flutter analysis rewrite') + source.write_text('authored source') + self.run_action('zstandard_web', 'restore') + self.assertEqual(analysis.read_text(), 'original analysis') + self.assertEqual(source.read_text(), 'authored source') + + def test_apple_jobs_restore_after_integration_even_when_a_step_fails(self): + for workflow, job, integration in [ + ('pr_check_ios.yml', 'check_ios_test', 'Run integration tests'), + ('pr_check_macos.yml', 'check_macos_test', + 'Launch macOS example and run integration tests'), + ('release_workflow.yml', 'candidate_apple', 'Run Apple integration tests'), + ]: + with self.subTest(workflow=workflow): + data = yaml.safe_load((ROOT / '.github/workflows' / workflow).read_text()) + steps = data['jobs'][job]['steps'] + integration_index = next(i for i, step in enumerate(steps) + if step.get('name') == integration) + restore_index = max(i for i, step in enumerate(steps) + if step.get('uses') == ACTION + and step['with']['mode'] == 'restore') + self.assertGreater(restore_index, integration_index) + self.assertEqual(steps[restore_index]['if'], 'always()') + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/scripts/tests/test_package_publication.py b/.github/scripts/tests/test_package_publication.py new file mode 100644 index 00000000..4ef3159b --- /dev/null +++ b/.github/scripts/tests/test_package_publication.py @@ -0,0 +1,172 @@ +"""Run publication shell steps with a clocked registry/SDK fixture, without uploads.""" + +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +import yaml + + +ROOT = Path(__file__).resolve().parents[3] +WORKFLOW = '.github/workflows/publish_workflow.yml' +COMPOSITE = '.github/actions/publish-package/action.yml' +ORDER = ['zstandard_platform_interface', 'zstandard_native', 'zstandard_android', + 'zstandard_ios', 'zstandard_linux', 'zstandard_macos', 'zstandard_web', + 'zstandard_windows', 'zstandard_cli', 'zstandard'] + + +class PackagePublicationTests(unittest.TestCase): + def setUp(self): + self.folder = tempfile.TemporaryDirectory(prefix='zstd-publisher-test-') + self.addCleanup(self.folder.cleanup) + self.root = Path(self.folder.name) / 'workspace with spaces' + self.root.mkdir() + for package in ORDER: + (self.root / package).mkdir() + self.state = self.root / 'registry.json' + self.state.write_text(json.dumps({'clock': 0, 'issued': None, 'events': []})) + tools = self.root / 'tools' + tools.mkdir() + source = '''#!/usr/bin/env python3 +import json, os, sys +from pathlib import Path +path = Path(os.environ['REGISTRY_STATE']) +state = json.loads(path.read_text()) +args = sys.argv[1:] +tool = Path(sys.argv[0]).name +package = Path.cwd().name +code = 0 +if tool == 'setup-sdk': + state['issued'] = state['clock'] + state['events'].append(['credential', state['clock']]) + # setup-dart inherits the composite's action path. Its optional matcher + # command fails in the runner when that parent has no analyzer asset. + if os.environ.get('INPUT_PROBLEM-MATCHER', 'true') != 'false': + matcher = Path(os.environ['GITHUB_ACTION_PATH']) / 'dart-analyzer.json' + state['events'].append(['matcher', str(matcher)]) + if not matcher.is_file(): + code = 66 +elif args[:2] == ['pub', 'get']: + state['events'].append(['resolve', package, tool]) + if os.environ.get('FAIL_RESOLVE') == package: + code = 43 + elif tool == 'flutter' and package == 'zstandard_web' and '--no-example' not in args: + # Flutter otherwise resolves the Web example's conflicting path/hosted sources. + code = 42 +elif args[:2] == ['run', 'scripts/publish_if_missing.dart']: + package = args[2] + if package in os.environ.get('ALREADY_PUBLISHED', '').split(','): + state['events'].append(['skip', package]) + elif state['issued'] is None or state['clock'] - state['issued'] >= 300: + code = 65 + else: + state['events'].append(['publish', package]) +elif args[:2] == ['run', 'scripts/wait_for_pub_version.dart']: + state['events'].append(['indexed', args[2]]) + state['clock'] += 120 +else: + code = 99 +path.write_text(json.dumps(state)) +sys.exit(code) +''' + for name in ['dart', 'flutter', 'setup-sdk']: + executable = tools / name + executable.write_text(source) + executable.chmod(0o700) + # setup-python's Linux interpreter needs its shared-library search path + # even when fixture commands run with an otherwise isolated environment. + self.env = {key: value for key, value in os.environ.items() + if key in ['PATH', 'LD_LIBRARY_PATH', 'SYSTEMROOT']} + self.env.update(PATH=str(tools) + os.pathsep + self.env['PATH'], + HOME=self.folder.name, BASH_ENV=os.devnull, + REGISTRY_STATE=str(self.state)) + self.action_path = self.root / 'composite action with spaces' + self.action_path.mkdir() + self.env['GITHUB_ACTION_PATH'] = str(self.action_path) + self.action = yaml.safe_load((ROOT / COMPOSITE).read_text()) + workflow = yaml.safe_load((ROOT / WORKFLOW).read_text()) + self.calls = [step for step in workflow['jobs']['publish']['steps'] + if step.get('uses') == './.github/actions/publish-package'] + + def run_package(self, package): + inputs = {'package': package, 'version': '9.8.7', 'dart-version': '3.12.0'} + for step in self.action['runs']['steps']: + if 'uses' in step: + self.assertEqual(step['uses'], 'dart-lang/setup-dart@v1') + self.assertEqual(step['with']['sdk'], '${{ inputs.dart-version }}') + sdk_env = {**self.env, 'INPUT_PROBLEM-MATCHER': str( + step.get('with', {}).get('problem-matcher', 'true')).lower()} + result = subprocess.run(['setup-sdk'], env=sdk_env, capture_output=True, + text=True, timeout=10) + if result.returncode: + return result + continue + env = dict(self.env) + for key, value in step.get('env', {}).items(): + for name, content in inputs.items(): + value = value.replace('${{ inputs.' + name + ' }}', content) + self.assertNotIn('${{', value) + env[key] = value + result = subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', step['run']], + cwd=self.root, env=env, capture_output=True, + text=True, timeout=10) + if result.returncode: + return result + return result + + def run_release(self): + for call in self.calls: + result = self.run_package(call['with']['package']) + if result.returncode: + return result + return result + + def events(self, kind): + return [event[1:] for event in json.loads(self.state.read_text())['events'] + if event[0] == kind] + + def test_all_packages_publish_in_order_across_expiring_credentials(self): + self.assertEqual([call['with']['package'] for call in self.calls], ORDER) + result = self.run_release() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.events('publish'), [[package] for package in ORDER]) + self.assertEqual(self.events('indexed'), [[package] for package in ORDER]) + self.assertEqual(self.events('credential'), [[120 * i] for i in range(10)]) + self.assertEqual([event[0] for event in self.events('resolve')], ORDER) + + def test_retry_skips_immutable_versions_and_continues_remaining_packages(self): + self.env['ALREADY_PUBLISHED'] = ','.join(ORDER[:6]) + result = self.run_release() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.events('skip'), [[package] for package in ORDER[:6]]) + self.assertEqual(self.events('publish'), [[package] for package in ORDER[6:]]) + + def test_nested_sdk_matcher_failure_stops_before_any_package_upload(self): + self.action['runs']['steps'][0]['with'].pop('problem-matcher', None) + result = self.run_release() + self.assertEqual(result.returncode, 66) + self.assertEqual(self.events('credential'), [[0]]) + self.assertEqual(self.events('matcher'), [ + [str(self.action_path / 'dart-analyzer.json')]]) + self.assertEqual(self.events('resolve'), []) + self.assertEqual(self.events('publish'), []) + + def test_resolution_failure_stops_before_uploading_or_starting_next_package(self): + self.env['FAIL_RESOLVE'] = 'zstandard_web' + result = self.run_release() + self.assertEqual(result.returncode, 43) + self.assertEqual(self.events('publish'), [[package] for package in ORDER[:6]]) + self.assertEqual([event[0] for event in self.events('resolve')], ORDER[:7]) + + def test_invalid_package_cannot_reach_dependency_resolution(self): + result = self.run_package('../outside') + self.assertEqual(result.returncode, 2) + self.assertEqual(self.events('resolve'), []) + self.assertEqual(self.events('publish'), []) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/scripts/tests/test_release_builds.py b/.github/scripts/tests/test_release_builds.py index deda1288..a363ebc3 100644 --- a/.github/scripts/tests/test_release_builds.py +++ b/.github/scripts/tests/test_release_builds.py @@ -20,7 +20,7 @@ class ReleaseBuildTests(unittest.TestCase): def setUp(self): self.folder = tempfile.TemporaryDirectory(prefix='zstd-release-build-') self.addCleanup(self.folder.cleanup) - self.root = Path(self.folder.name) / 'workspace with spaces' + self.root = Path(self.folder.name).resolve() / 'workspace with spaces' self.root.mkdir() template = Path(self.folder.name) / 'template' template.mkdir() @@ -112,6 +112,151 @@ def test_linux_native_tests_reconfigure_the_existing_flutter_build_directory(sel self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertTrue((existing / 'CMakeFiles').is_dir()) + def static_fixture(self): + packages = ['zstandard_platform_interface', 'zstandard_native', 'zstandard_android', + 'zstandard_ios', 'zstandard_linux', 'zstandard_macos', 'zstandard_web', + 'zstandard_windows', 'zstandard_cli', 'zstandard'] + for package in packages: + (self.root / package).mkdir(exist_ok=True) + self.git('init', '-q') + tools = self.root / 'tools' + tools.mkdir() + tool = '''#!/usr/bin/env python3 +import json,os,sys +from pathlib import Path +package=Path.cwd().name +args=sys.argv[1:] +if args[:2]==['pub','get']: + with open(os.environ['TRACE'],'a') as trace: + trace.write(json.dumps(['get',package])+'\\n') + if package==os.environ.get('FAIL_RESOLVE'): + sys.exit(23) + Path('.resolved').touch() +elif args and args[0]=='format': + with open(os.environ['TRACE'],'a') as trace: + trace.write(json.dumps(['format',package])+'\\n') + if not Path('.resolved').exists(): + sys.exit(31) +''' + for name in ['dart', 'flutter']: + (tools / name).write_text(tool) + (tools / name).chmod(0o700) + trace = self.root / 'trace' + trace.touch() + self.env.update(PATH=str(tools) + os.pathsep + self.env['PATH'], + VERSION='1.6.0', TRACE=str(trace)) + script = next(step['run'] for step in WORKFLOW['jobs']['candidate_static_checks']['steps'] + if step.get('name') == 'Verify all release metadata and package tests') + return packages, trace, script + + @unittest.skipUnless(shutil.which('cmake') and shutil.which('ctest'), + 'CMake and CTest are needed for real test discovery') + def test_native_ctest_requires_discovered_passing_tests(self): + build = next(step['run'] for step in WORKFLOW['jobs']['candidate_linux']['steps'] + if step.get('name') == 'Build, run native tests and run integration tests') + command = next(line for line in build.splitlines() if line.startswith('ctest ')) + for outcome in ['passing', 'failing', 'empty']: + with self.subTest(outcome=outcome): + source = self.root / outcome + plugin = source / 'plugins/zstandard_linux' + plugin.mkdir(parents=True) + # The empty case reproduces enable_testing() being called only + # inside Flutter's plugin subdirectory rather than at the root. + top_level = 'enable_testing()\n' if outcome != 'empty' else '' + (source / 'CMakeLists.txt').write_text( + 'cmake_minimum_required(VERSION 3.16)\nproject(ctest_contract NONE)\n' + + top_level + 'add_subdirectory(plugins/zstandard_linux)\n') + (plugin / 'CMakeLists.txt').write_text( + 'enable_testing()\nadd_test(NAME native_contract ' + 'COMMAND "${CMAKE_COMMAND}" -E ' + + ('false' if outcome == 'failing' else 'true') + ')\n') + directory = source / 'build' + subprocess.run(['cmake', '-S', str(source), '-B', str(directory)], + env=self.env, check=True, capture_output=True, timeout=15) + result = self.run_command(f'cmake_build_dir="{directory}"\n' + command) + if outcome == 'passing': + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn('1/1 Test', result.stdout) + else: + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_static_checks_resolve_each_package_before_formatting(self): + packages, trace, script = self.static_fixture() + result = self.run_command(script) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + calls = [json.loads(line) for line in trace.read_text().splitlines()] + self.assertEqual(sorted(package for kind, package in calls if kind == 'format'), + sorted(packages)) + + def test_static_checks_do_not_hide_dependency_resolution_failure(self): + _, trace, script = self.static_fixture() + self.env['FAIL_RESOLVE'] = 'zstandard_android' + result = self.run_command(script) + self.assertNotEqual(result.returncode, 0) + calls = [json.loads(line) for line in trace.read_text().splitlines()] + self.assertIn(['get', 'zstandard_android'], calls) + self.assertNotIn(['format', 'zstandard_android'], calls) + + def test_linux_vm_tests_load_the_freshly_built_plugin(self): + self.git('init', '-q') + (self.root / 'zstandard_linux/example').mkdir(parents=True) + tools = self.root / 'tools' + tools.mkdir() + flutter = tools / 'flutter' + flutter.write_text('''#!/usr/bin/env python3 +import json,os,sys +from pathlib import Path +args=sys.argv[1:] +if args[:2]==['build','linux']: + build=Path('build/linux/x64/release') + (build/'bundle/lib').mkdir(parents=True) + (build/'bundle/lib/libzstandard_linux_plugin.so').write_bytes(b'fixture plugin') + (build/'CMakeCache.txt').touch() + Path('main.dart').touch() + Path('flutter-target').write_text(str(Path('main.dart').resolve())) +elif args and args[0]=='test': + search=os.environ.get('LD_LIBRARY_PATH','').split(os.pathsep) + with open(os.environ['TRACE'],'a') as trace: + trace.write(json.dumps({'kind':'vm-test','search':search})+'\\n') + if not any((Path(path)/'libzstandard_linux_plugin.so').is_file() for path in search): + sys.exit(32) +''') + flutter.chmod(0o700) + (tools / 'cmake').write_text('''#!/usr/bin/env python3 +import json,os,sys +from pathlib import Path +if '--build' in sys.argv: + target=Path(os.environ['NATIVE_TARGET']).read_text() + if not Path(target).is_file(): + sys.exit(33) + with open(os.environ['TRACE'],'a') as trace: + trace.write(json.dumps({'kind':'native-build'})+'\\n') +''') + (tools / 'cmake').chmod(0o700) + (tools / 'ctest').write_text('#!/bin/sh\nexit 0\n') + (tools / 'ctest').chmod(0o700) + (self.root / 'scripts').mkdir() + # Flutter's integration driver leaves generated_config.cmake pointing + # at a test listener that it removes on exit. A subsequent native build + # must not try to assemble that retired target. + (self.root / 'scripts/test_linux_integration.sh').write_text( + 'printf "%s\\n" "$PWD/zstandard_linux/example/retired-listener.dart" ' + '> zstandard_linux/example/flutter-target\n') + trace = self.root / 'trace' + self.env.update(PATH=str(tools) + os.pathsep + self.env['PATH'], TRACE=str(trace), + NATIVE_TARGET=str(self.root / 'zstandard_linux/example/flutter-target')) + script = next(step['run'] for step in WORKFLOW['jobs']['candidate_linux']['steps'] + if step.get('name') == 'Build, run native tests and run integration tests') + result = self.run_command(script) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + calls = [json.loads(line) for line in trace.read_text().splitlines()] + vm_tests = [call for call in calls if call['kind'] == 'vm-test'] + self.assertEqual(len(vm_tests), 1) + self.assertEqual(vm_tests[0]['search'][0], + str(self.root / 'zstandard_linux/example/build/linux/x64/release/bundle/lib')) + self.assertEqual(sum(call['kind'] == 'native-build' for call in calls), 1) + self.assertTrue(Path(self.env['NATIVE_TARGET']).read_text().endswith('/retired-listener.dart\n')) + def test_release_admission_rejects_other_branches_before_checkout(self): script = WORKFLOW['jobs']['validate']['steps'][0]['with']['script'] cases = [(prefix, ref, expected) diff --git a/.github/scripts/tests/test_release_checkout.py b/.github/scripts/tests/test_release_checkout.py new file mode 100644 index 00000000..aea99e33 --- /dev/null +++ b/.github/scripts/tests/test_release_checkout.py @@ -0,0 +1,170 @@ +"""Exercise release cache retirement with real Git repositories, without uploads.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +import yaml + +ROOT = Path(__file__).resolve().parents[3] +ACTION = ROOT / '.github/actions/checkout-release-source' +USE = '$/.github/actions/checkout-release-source' +NODE = ''' +try { + const result = require(process.argv[1])({core: {notice() {}}}); + console.log(JSON.stringify(result)); +} catch (error) { console.error(error.message); process.exitCode = 1; } +''' + + +class ReleaseCheckoutTests(unittest.TestCase): + def setUp(self): + self.folder = tempfile.TemporaryDirectory(prefix='zstandard checkout tests ') + self.addCleanup(self.folder.cleanup) + self.root = Path(self.folder.name).resolve() + self.environment = {k: v for k, v in os.environ.items() if not k.startswith('GIT_')} + self.environment.update({'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_NOSYSTEM': '1', + 'GIT_AUTHOR_NAME': 'Fixture', 'GIT_COMMITTER_NAME': 'Fixture', + 'GIT_AUTHOR_EMAIL': 'fixture@example.invalid', + 'GIT_COMMITTER_EMAIL': 'fixture@example.invalid'}) + self.source = self.root / 'source' + self.source.mkdir() + self.git(self.source, 'init', '-q', '-b', 'main') + for text in ['first', 'second']: + (self.source / 'file.txt').write_text(text) + self.git(self.source, 'add', 'file.txt') + self.git(self.source, 'commit', '-qm', text) + self.case_number = 0 + + def git(self, directory, *arguments): + return subprocess.run(['git', '-C', str(directory), *arguments], check=True, + capture_output=True, text=True, env=self.environment).stdout.strip() + + def case(self, shallow=True): + self.case_number += 1 + work = self.root / f'runner {self.case_number}' / 'work' + temporary = work / '_temp' + temporary.mkdir(parents=True) + workspace = work / 'zstandard/zstandard' + workspace.parent.mkdir() + command = ['clone', '-q'] + (['--depth=1'] if shallow else []) + self.git(self.root, *command, self.source.as_uri(), str(workspace)) + self.git(workspace, 'remote', 'set-url', 'origin', 'https://github.com/vypdev/zstandard') + environment = {**self.environment, 'GITHUB_ACTIONS': 'true', + 'GITHUB_REPOSITORY': 'vypdev/zstandard', 'GITHUB_RUN_ID': '123', + 'GITHUB_RUN_ATTEMPT': '1', 'GITHUB_WORKSPACE': str(workspace), + 'RUNNER_TEMP': str(temporary)} + return workspace, temporary, environment + + def retire(self, environment): + return subprocess.run(['node', '-e', NODE, str(ACTION / 'retire-shallow-cache.cjs')], + env=environment, capture_output=True, text=True, timeout=20) + + def test_shallow_cache_is_preserved_and_a_fresh_fetch_has_complete_history(self): + workspace, temporary, environment = self.case() + original_head = self.git(workspace, 'rev-parse', 'HEAD') + self.assertEqual(self.git(workspace, 'rev-list', '--count', 'HEAD'), '1') + result = self.retire(environment) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(json.loads(result.stdout)['retired']) + self.assertFalse((workspace / '.git').exists()) + self.assertEqual((workspace / 'file.txt').read_text(), 'second') + retired = next(temporary.glob('zstandard-shallow-git-*')) + self.assertEqual((retired / 'git/HEAD').read_text().strip().split()[-1], 'refs/heads/main') + self.assertEqual(json.loads((retired / '.owner').read_text())['head'], original_head) + self.assertTrue((retired / 'git/shallow').is_file()) + # The next official checkout creates a new store, retaining full ancestry. + self.git(workspace, 'init', '-q') + self.git(workspace, 'remote', 'add', 'origin', self.source.as_uri()) + self.git(workspace, 'fetch', '-q', 'origin', '+refs/heads/*:refs/remotes/origin/*') + self.git(workspace, 'checkout', '-qf', '-B', 'main', 'origin/main') + self.assertEqual(self.git(workspace, 'rev-parse', '--is-shallow-repository'), 'false') + self.assertEqual(self.git(workspace, 'rev-list', '--count', 'HEAD'), '2') + self.git(workspace, 'fsck', '--connectivity-only', '--no-dangling') + + def test_complete_or_new_checkout_does_not_retire_metadata(self): + workspace, temporary, environment = self.case(shallow=False) + inode = (workspace / '.git').stat().st_ino + result = self.retire(environment) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(json.loads(result.stdout)['retired']) + self.assertEqual((workspace / '.git').stat().st_ino, inode) + self.assertEqual(list(temporary.iterdir()), []) + (workspace / '.git').rename(self.root / 'complete-fixture-metadata') + result = self.retire(environment) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(json.loads(result.stdout)['retired']) + + def test_authored_changes_stashes_and_unpublished_branches_are_retained(self): + for state in ['dirty', 'untracked', 'stash', 'unpublished']: + with self.subTest(state=state): + workspace, temporary, environment = self.case() + if state == 'untracked': + (workspace / 'authored.txt').write_text('retain me') + else: + (workspace / 'file.txt').write_text('retain me') + if state == 'stash': + self.git(workspace, 'stash', 'push', '-q') + if state == 'unpublished': + self.git(workspace, 'add', 'file.txt') + self.git(workspace, 'commit', '-qm', 'unpublished authored work') + inode = (workspace / '.git').stat().st_ino + head = self.git(workspace, 'rev-parse', 'HEAD') + result = self.retire(environment) + self.assertNotEqual(result.returncode, 0) + self.assertEqual((workspace / '.git').stat().st_ino, inode) + self.assertEqual(self.git(workspace, 'rev-parse', 'HEAD'), head) + self.assertEqual(list(temporary.iterdir()), []) + + def test_unrelated_workspace_repository_links_and_locks_are_rejected(self): + for state in ['outside', 'wrong-origin', 'linked', 'lock', 'worktree']: + with self.subTest(state=state): + workspace, temporary, environment = self.case() + if state == 'outside': + environment['RUNNER_TEMP'] = str(self.root) + elif state == 'wrong-origin': + self.git(workspace, 'remote', 'set-url', 'origin', 'https://github.com/other/repository') + elif state == 'linked': + (workspace / '.git').rename(temporary / 'original-git') + (workspace / '.git').symlink_to(temporary / 'original-git', target_is_directory=True) + elif state == 'lock': + (workspace / '.git/index.lock').touch() + elif state == 'worktree': + (workspace / '.git/worktrees/active').mkdir(parents=True) + result = self.retire(environment) + self.assertNotEqual(result.returncode, 0) + self.assertTrue((workspace / '.git').exists()) + self.assertEqual(list(temporary.glob('zstandard-shallow-git-*')), []) + + def test_release_jobs_keep_exact_refs_and_complete_history(self): + action = yaml.safe_load((ACTION / 'action.yml').read_text()) + checkout = action['runs']['steps'][-1] + self.assertEqual(checkout['uses'], 'actions/checkout@v6') + self.assertEqual(action['inputs']['fetch-depth']['default'], '0') + for name, count in [('release_workflow.yml', 11), ('publish_workflow.yml', 2)]: + workflow = yaml.safe_load((ROOT / '.github/workflows' / name).read_text()) + calls = [step for job in workflow['jobs'].values() for step in job.get('steps', []) + if step.get('uses') == USE] + self.assertEqual(len(calls), count) + self.assertTrue(all(str(call['with']['fetch-depth']) == '0' for call in calls)) + workflow = yaml.safe_load((ROOT / '.github/workflows/publish_workflow.yml').read_text()) + self.assertEqual(workflow['jobs']['publish']['steps'][0]['with']['ref'], + '${{ needs.validate_tag.outputs.commit }}') + + def test_checkout_free_local_action_does_not_allow_remote_commit_pins(self): + folder = self.root / 'policy/.github/workflows' + folder.mkdir(parents=True) + path = folder / 'fixture.yml' + command = [sys.executable, str(ROOT / '.github/scripts/check_action_versions.py'), str(folder.parents[1])] + path.write_text(f'jobs:\n fixture:\n steps:\n - uses: {USE}\n - uses: actions/checkout@v6\n') + self.assertEqual(subprocess.run(command, capture_output=True).returncode, 0) + path.write_text('jobs:\n fixture:\n steps:\n - uses: actions/checkout@' + 'a' * 40 + '\n') + self.assertNotEqual(subprocess.run(command, capture_output=True).returncode, 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/scripts/tests/test_verification_runtime.py b/.github/scripts/tests/test_verification_runtime.py new file mode 100644 index 00000000..acac3431 --- /dev/null +++ b/.github/scripts/tests/test_verification_runtime.py @@ -0,0 +1,64 @@ +"""Verify native tool settings survive only through reviewed PATH launchers.""" + +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[3] + + +class VerificationRuntimeTests(unittest.TestCase): + def test_private_launchers_keep_native_paths_and_arguments_without_capturing_secrets(self): + with tempfile.TemporaryDirectory(prefix='zstd-verification-env-') as temporary: + root = Path(temporary).resolve() + profile = root / 'profile with spaces' + profile.mkdir() + sdk = root / 'sdk with spaces' + sdk.mkdir() + trace = root / 'trace.json' + for tool in ['flutter', 'dart']: + executable = sdk / tool + executable.write_text('''#!/usr/bin/env python3 +import json,os,sys +from pathlib import Path +keys=['CPATH','LD_LIBRARY_PATH','PKG_CONFIG_SYSROOT_DIR','PUB_CACHE','XDG_CONFIG_HOME','UNRELATED_SECRET'] +Path(os.environ['TRACE']).write_text(json.dumps({'args':sys.argv[1:],'env':{key:os.environ.get(key) for key in keys}})) +''') + executable.chmod(0o700) + paths = root / 'github-path' + paths.touch() + header_path = str(root / 'headers $(touch unexpected)') + environment = {'PATH': str(sdk) + os.pathsep + os.environ['PATH'], + 'HOME': str(profile), 'GITHUB_PATH': str(paths), + 'CPATH': header_path, 'LD_LIBRARY_PATH': str(root / 'private libraries'), + 'PKG_CONFIG_SYSROOT_DIR': str(root / 'sysroot with spaces'), + 'PUB_CACHE': str(profile / 'private cache'), + 'XDG_CONFIG_HOME': str(profile / '.config'), + 'UNRELATED_SECRET': 'must-not-be-captured'} + subprocess.run(['bash', str(ROOT / '.github/actions/copilot-verification-setup/runtime-launchers.sh')], + env=environment, check=True, timeout=10) + launcher = Path(paths.read_text().strip()) + self.assertTrue(launcher.is_relative_to(profile)) + for tool in ['flutter', 'dart']: + probe = {'PATH': os.environ['PATH'], 'HOME': str(profile), + 'TRACE': str(trace), 'LD_LIBRARY_PATH': str(root / 'actual test plugin')} + subprocess.run([str(launcher / tool), 'literal $(touch unexpected)', 'two words'], + cwd=root, env=probe, check=True, timeout=10) + result = json.loads(trace.read_text()) + self.assertEqual(result['args'], ['literal $(touch unexpected)', 'two words']) + self.assertEqual(result['env']['CPATH'], header_path) + self.assertEqual(result['env']['PKG_CONFIG_SYSROOT_DIR'], environment['PKG_CONFIG_SYSROOT_DIR']) + self.assertEqual(result['env']['PUB_CACHE'], environment['PUB_CACHE']) + self.assertEqual(result['env']['XDG_CONFIG_HOME'], environment['XDG_CONFIG_HOME']) + self.assertEqual(result['env']['LD_LIBRARY_PATH'].split(os.pathsep), + [environment['LD_LIBRARY_PATH'], probe['LD_LIBRARY_PATH']]) + self.assertIsNone(result['env']['UNRELATED_SECRET']) + self.assertFalse((root / 'unexpected').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/workflows/android_sdk_contract.yml b/.github/workflows/android_sdk_contract.yml index 1ca187ed..ee3b7d39 100644 --- a/.github/workflows/android_sdk_contract.yml +++ b/.github/workflows/android_sdk_contract.yml @@ -3,7 +3,9 @@ on: pull_request: paths: - scripts/run_android_emulator_ci.sh + - scripts/android_ci_session.py - scripts/tests/test_android_sdk_provisioning.py + - scripts/tests/test_android_ci_session.py - .github/workflows/android_sdk_contract.yml permissions: contents: read @@ -15,8 +17,11 @@ jobs: group: vypdev labels: [self-hosted, codex, Linux] timeout-minutes: 2 + env: + PYTHONDONTWRITEBYTECODE: '1' steps: - uses: actions/checkout@v6 with: persist-credentials: false - run: python3 scripts/tests/test_android_sdk_provisioning.py + - run: python3 scripts/tests/test_android_ci_session.py diff --git a/.github/workflows/linux_dependency_contract.yml b/.github/workflows/linux_dependency_contract.yml index 66e1f0fd..27e785d3 100644 --- a/.github/workflows/linux_dependency_contract.yml +++ b/.github/workflows/linux_dependency_contract.yml @@ -3,6 +3,7 @@ on: pull_request: paths: - .github/actions/setup-linux-dependencies/** + - .github/actions/copilot-verification-setup/linux-dependencies.sh - .github/scripts/tests/verify_linux_prefix.sh - .github/workflows/linux_dependency_contract.yml permissions: @@ -29,14 +30,7 @@ jobs: run: | set -euo pipefail state="$(mktemp -d "$RUNNER_TEMP/zstandard-linux-contract.XXXXXX")" - python3 -m venv "$state/venv" - "$state/venv/bin/python" -m pip --isolated install --index-url https://pypi.org/simple PyYAML==6.0.3 - "$state/venv/bin/python" - "$state" <<'PY' - from pathlib import Path - import sys, yaml - action = yaml.safe_load(Path('.github/actions/setup-linux-dependencies/action.yml').read_text()) - Path(sys.argv[1], 'action.sh').write_text(action['runs']['steps'][0]['run']) - PY + cp .github/actions/copilot-verification-setup/linux-dependencies.sh "$state/action.sh" cp .github/scripts/tests/verify_linux_prefix.sh "$state/check.sh" echo "LINUX_CONTRACT_STATE=$state" >> "$GITHUB_ENV" - name: Execute without privileges or a host Docker socket diff --git a/.github/workflows/package_preparation_contract.yml b/.github/workflows/package_preparation_contract.yml index 7da85bd8..f4df2dcb 100644 --- a/.github/workflows/package_preparation_contract.yml +++ b/.github/workflows/package_preparation_contract.yml @@ -7,6 +7,9 @@ on: - '.github/scripts/check_action_versions.py' - '.github/scripts/update_versions.dart' - '.github/scripts/tests/test_package_preparation.py' + - '.github/scripts/tests/test_package_publication.py' + - '.github/scripts/tests/test_release_checkout.py' + - '.github/scripts/tests/test_flutter_project_files.py' - '.github/scripts/tests/test_hotfix_ref.py' - 'scripts/check_pub_metadata.dart' - 'scripts/check_release_tag.sh' @@ -27,6 +30,9 @@ jobs: group: vypdev labels: [self-hosted, codex, Linux] timeout-minutes: 10 + env: + # Keep imported test modules out of subsequent clean-workspace checkouts. + PYTHONDONTWRITEBYTECODE: '1' steps: - uses: actions/checkout@v6 with: @@ -46,8 +52,14 @@ jobs: "$RUNNER_TEMP/preparation-contract/bin/python" -m pip install PyYAML==6.0.3 - name: Require version tags for workflow and composite Actions run: '"$RUNNER_TEMP/preparation-contract/bin/python" .github/scripts/check_action_versions.py' + - name: Verify safe replacement of inherited shallow checkout metadata + run: '"$RUNNER_TEMP/preparation-contract/bin/python" -m unittest discover -s .github/scripts/tests -p test_release_checkout.py -v' + - name: Verify Apple fixture restoration after Flutter migrations + run: '"$RUNNER_TEMP/preparation-contract/bin/python" -m unittest discover -s .github/scripts/tests -p test_flutter_project_files.py -v' - name: Verify real metadata preparation and failure behavior run: '"$RUNNER_TEMP/preparation-contract/bin/python" -m unittest discover -s .github/scripts/tests -p test_package_preparation.py -v' + - name: Verify publication ordering, credential renewal and failure recovery without uploads + run: '"$RUNNER_TEMP/preparation-contract/bin/python" -m unittest discover -s .github/scripts/tests -p test_package_publication.py -v' - name: Reject preparation outside the matching hotfix branch run: '"$RUNNER_TEMP/preparation-contract/bin/python" -m unittest discover -s .github/scripts/tests -p test_hotfix_ref.py -v' - name: Verify release and hotfix tag provenance without publication diff --git a/.github/workflows/pr_check_android.yml b/.github/workflows/pr_check_android.yml index e8c5dff9..cca2e3fb 100644 --- a/.github/workflows/pr_check_android.yml +++ b/.github/workflows/pr_check_android.yml @@ -74,7 +74,7 @@ jobs: runs-on: group: vypdev labels: [self-hosted, codex, Linux] - timeout-minutes: 60 + timeout-minutes: 120 strategy: fail-fast: false # The self-hosted runner shares one ADB daemon and one emulator port. diff --git a/.github/workflows/pr_check_ios.yml b/.github/workflows/pr_check_ios.yml index 7d286bba..8cba6aeb 100644 --- a/.github/workflows/pr_check_ios.yml +++ b/.github/workflows/pr_check_ios.yml @@ -45,6 +45,7 @@ jobs: run: flutter pub get - uses: ./.github/actions/preserve-flutter-project-files + if: always() with: package: zstandard_ios mode: restore @@ -112,6 +113,7 @@ jobs: run: flutter pub get - uses: ./.github/actions/preserve-flutter-project-files + if: always() with: package: zstandard_ios mode: restore @@ -215,6 +217,13 @@ jobs: timeout-minutes: 3 run: bash .github/scripts/prepare_ios_flutter.sh --cleanup + - name: Restore Apple host files after Flutter migrations + if: always() + uses: ./.github/actions/preserve-flutter-project-files + with: + package: zstandard_ios + mode: restore + check_ios_publish_dry_run: name: Dry Run Publish iOS if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} @@ -243,6 +252,7 @@ jobs: run: flutter pub get - uses: ./.github/actions/preserve-flutter-project-files + if: always() with: package: zstandard_ios mode: restore diff --git a/.github/workflows/pr_check_linux.yml b/.github/workflows/pr_check_linux.yml index 610a1890..9b8e98a8 100644 --- a/.github/workflows/pr_check_linux.yml +++ b/.github/workflows/pr_check_linux.yml @@ -91,7 +91,7 @@ jobs: test -n "$cmake_build_dir" ZSTANDARD_LINUX_ENABLE_TESTS=1 cmake -S linux -B "$cmake_build_dir" cmake --build "$cmake_build_dir" --target zstandard_linux_test --parallel - ctest --test-dir "$cmake_build_dir" --output-on-failure + ctest --test-dir "$cmake_build_dir" --output-on-failure --no-tests=error - name: Run integration tests working-directory: zstandard_linux/example @@ -145,7 +145,7 @@ jobs: test -n "$cmake_build_dir" ZSTANDARD_LINUX_ENABLE_TESTS=1 cmake -S linux -B "$cmake_build_dir" cmake --build "$cmake_build_dir" --target zstandard_linux_test --parallel - ctest --test-dir "$cmake_build_dir" --output-on-failure + ctest --test-dir "$cmake_build_dir" --output-on-failure --no-tests=error - name: Run integration tests (C sources from pub cache) working-directory: zstandard_linux/example diff --git a/.github/workflows/pr_check_macos.yml b/.github/workflows/pr_check_macos.yml index b678df73..6cb2384d 100644 --- a/.github/workflows/pr_check_macos.yml +++ b/.github/workflows/pr_check_macos.yml @@ -45,6 +45,7 @@ jobs: run: flutter pub get - uses: ./.github/actions/preserve-flutter-project-files + if: always() with: package: zstandard_macos mode: restore @@ -108,6 +109,7 @@ jobs: run: flutter pub get - uses: ./.github/actions/preserve-flutter-project-files + if: always() with: package: zstandard_macos mode: restore @@ -163,6 +165,13 @@ jobs: test ! -e zstandard_native && test ! -L zstandard_native mv "$ZSTANDARD_NATIVE_ORIGINAL_PATH" zstandard_native + - name: Restore Apple host files after Flutter migrations + if: always() + uses: ./.github/actions/preserve-flutter-project-files + with: + package: zstandard_macos + mode: restore + check_macos_publish_dry_run: name: Dry Run Publish macOS if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} @@ -191,6 +200,7 @@ jobs: run: flutter pub get - uses: ./.github/actions/preserve-flutter-project-files + if: always() with: package: zstandard_macos mode: restore diff --git a/.github/workflows/project_verification_contract.yml b/.github/workflows/project_verification_contract.yml new file mode 100644 index 00000000..35ff0292 --- /dev/null +++ b/.github/workflows/project_verification_contract.yml @@ -0,0 +1,69 @@ +name: Project verification setup contract + +on: + pull_request: + paths: + - '.github/actions/copilot-verification-setup/**' + - '.github/actions/setup-linux-dependencies/**' + - '.github/actions/setup-flutter/**' + - '.github/scripts/tests/test_flutter_profile.py' + - '.github/workflows/project_verification_contract.yml' + - 'scripts/test_all.sh' + - 'scripts/run_unit_tests.py' + - 'scripts/tests/test_unit_test_runner.py' + - '.github/scripts/tests/test_verification_runtime.py' + - 'scripts/create_local_overrides.dart' + +permissions: + contents: read + +jobs: + project-verification: + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: + group: vypdev + labels: [self-hosted, codex, Linux] + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Create a private dependency profile + id: profile + shell: bash + run: | + set -euo pipefail + profile="$(mktemp -d "$RUNNER_TEMP/zstandard-verification-contract.XXXXXX")" + mkdir "$profile/home" + printf '%s\n' 'zstandard-verification-contract' > "$profile/.owner" + printf 'directory=%s\n' "$profile" >> "$GITHUB_OUTPUT" + printf 'HOME=%s\n' "$profile/home" >> "$GITHUB_ENV" + - uses: ./.github/actions/copilot-verification-setup + - name: Verify isolated native launchers + run: python3 -B -m unittest discover -s .github/scripts/tests -p test_verification_runtime.py -v + - name: Verify Dart configuration stays private when the SDK is invoked directly + run: python3 -B -m unittest discover -s .github/scripts/tests -p test_flutter_profile.py -v + - name: Verify local dependency preparation, failure handling and source restoration + run: python3 -B -m unittest discover -s scripts/tests -p test_unit_test_runner.py -v + - name: Run the configured project verification without source changes + shell: bash + run: | + set -euo pipefail + bash scripts/test_all.sh + git diff --check + git diff --exit-code + test -z "$(git ls-files --others --exclude-standard)" + - name: Retire the owned dependency profile + if: always() && steps.profile.outputs.directory != '' + shell: bash + env: + ZSTANDARD_PROFILE: ${{ steps.profile.outputs.directory }} + run: | + set -euo pipefail + case "$ZSTANDARD_PROFILE" in + "$RUNNER_TEMP"/zstandard-verification-contract.*) ;; + *) echo 'Unexpected private profile path.' >&2; exit 1 ;; + esac + test ! -L "$ZSTANDARD_PROFILE" + test "$(cat "$ZSTANDARD_PROFILE/.owner")" = zstandard-verification-contract + rm -r -- "$ZSTANDARD_PROFILE" diff --git a/.github/workflows/publish_workflow.yml b/.github/workflows/publish_workflow.yml index e8b23134..e3acc5dd 100644 --- a/.github/workflows/publish_workflow.yml +++ b/.github/workflows/publish_workflow.yml @@ -34,7 +34,7 @@ jobs: issue: ${{ steps.candidate.outputs.issue }} operation-id: ${{ steps.candidate.outputs.operation-id }} steps: - - uses: actions/checkout@v6 + - uses: $/.github/actions/checkout-release-source with: fetch-depth: 0 @@ -101,45 +101,80 @@ jobs: contents: read id-token: write steps: - - uses: actions/checkout@v6 + - uses: $/.github/actions/checkout-release-source with: ref: ${{ needs.validate_tag.outputs.commit }} fetch-depth: 0 - - # setup-dart provisions the short-lived pub.dev credential from GitHub's - # OIDC token. Flutter is installed afterwards so its bundled Dart stays - # first on PATH while using the same Pub credentials. - - uses: dart-lang/setup-dart@v1 - - uses: ./.github/actions/setup-flutter with: flutter-version: ${{ env.FLUTTER_VERSION }} - - - name: Publish in dependency order, safely skipping immutable versions - env: - VERSION: ${{ needs.validate_tag.outputs.version }} + - name: Read the tested Dart SDK version + id: dart run: | set -euo pipefail - publish_and_wait() { - local package="$1" - if [[ "$package" == zstandard_native || "$package" == zstandard_cli ]]; then - (cd "$package" && dart pub get) - else - (cd "$package" && flutter pub get) - fi - dart run scripts/publish_if_missing.dart "$package" "$VERSION" - dart run scripts/wait_for_pub_version.dart "$package" "$VERSION" 1200 - } - - publish_and_wait zstandard_platform_interface - publish_and_wait zstandard_native - for package in \ - zstandard_android zstandard_ios zstandard_linux zstandard_macos \ - zstandard_web zstandard_windows; do - publish_and_wait "$package" - done - publish_and_wait zstandard_cli - publish_and_wait zstandard + version="$(dart --version 2>&1 | sed -nE 's/^Dart SDK version: ([0-9]+\.[0-9]+\.[0-9]+).*/\1/p')" + test -n "$version" + echo "version=$version" >> "$GITHUB_OUTPUT" + - name: Publish zstandard_platform_interface + uses: ./.github/actions/publish-package + with: + package: zstandard_platform_interface + version: ${{ needs.validate_tag.outputs.version }} + dart-version: ${{ steps.dart.outputs.version }} + - name: Publish zstandard_native + uses: ./.github/actions/publish-package + with: + package: zstandard_native + version: ${{ needs.validate_tag.outputs.version }} + dart-version: ${{ steps.dart.outputs.version }} + - name: Publish zstandard_android + uses: ./.github/actions/publish-package + with: + package: zstandard_android + version: ${{ needs.validate_tag.outputs.version }} + dart-version: ${{ steps.dart.outputs.version }} + - name: Publish zstandard_ios + uses: ./.github/actions/publish-package + with: + package: zstandard_ios + version: ${{ needs.validate_tag.outputs.version }} + dart-version: ${{ steps.dart.outputs.version }} + - name: Publish zstandard_linux + uses: ./.github/actions/publish-package + with: + package: zstandard_linux + version: ${{ needs.validate_tag.outputs.version }} + dart-version: ${{ steps.dart.outputs.version }} + - name: Publish zstandard_macos + uses: ./.github/actions/publish-package + with: + package: zstandard_macos + version: ${{ needs.validate_tag.outputs.version }} + dart-version: ${{ steps.dart.outputs.version }} + - name: Publish zstandard_web + uses: ./.github/actions/publish-package + with: + package: zstandard_web + version: ${{ needs.validate_tag.outputs.version }} + dart-version: ${{ steps.dart.outputs.version }} + - name: Publish zstandard_windows + uses: ./.github/actions/publish-package + with: + package: zstandard_windows + version: ${{ needs.validate_tag.outputs.version }} + dart-version: ${{ steps.dart.outputs.version }} + - name: Publish zstandard_cli + uses: ./.github/actions/publish-package + with: + package: zstandard_cli + version: ${{ needs.validate_tag.outputs.version }} + dart-version: ${{ steps.dart.outputs.version }} + - name: Publish zstandard + uses: ./.github/actions/publish-package + with: + package: zstandard + version: ${{ needs.validate_tag.outputs.version }} + dart-version: ${{ steps.dart.outputs.version }} verify: name: Verify all published packages diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 1cd69c7b..de9b6808 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -170,6 +170,9 @@ jobs: - name: Test candidate source isolation run: python3 -m unittest discover -s scripts/tests -p test_native_candidate_source.py -v + - name: Test Web integration launcher + run: python3 -m unittest discover -s scripts/tests -p test_web_integration_runner.py -v + - name: Validate shell scripts run: bash -n scripts/*.sh zstandard_ios/scripts/*.sh zstandard_macos/scripts/*.sh diff --git a/.github/workflows/release_workflow.yml b/.github/workflows/release_workflow.yml index bb35ba86..3aa56924 100644 --- a/.github/workflows/release_workflow.yml +++ b/.github/workflows/release_workflow.yml @@ -108,7 +108,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v6 + - uses: $/.github/actions/checkout-release-source with: ref: ${{ github.ref }} fetch-depth: 0 @@ -197,7 +197,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v6 + - uses: $/.github/actions/checkout-release-source with: ref: ${{ needs.update_files.outputs.commit }} fetch-depth: 0 @@ -252,7 +252,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v6 + - uses: $/.github/actions/checkout-release-source with: ref: ${{ needs.cli_build_macos_precompiled_libs.outputs.commit }} fetch-depth: 0 @@ -305,9 +305,8 @@ jobs: cli_build_windows_precompiled_libs: name: Build CLI libraries - Windows - runs-on: - group: vypdev - labels: [self-hosted, Windows, X64] + # Both MSVC architectures are preinstalled on this standard image. + runs-on: windows-2025 timeout-minutes: 60 needs: [update_files, cli_build_linux_precompiled_libs] outputs: @@ -315,38 +314,39 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v6 + - uses: $/.github/actions/checkout-release-source with: ref: ${{ needs.cli_build_linux_precompiled_libs.outputs.commit }} fetch-depth: 0 - - uses: ./.github/actions/setup-flutter-windows - with: - flutter-version: ${{ env.FLUTTER_VERSION }} - name: Build x64 and ARM64 libraries with the installed Visual Studio - shell: pwsh + shell: powershell run: | $ErrorActionPreference = 'Stop' - $generator = @((cmake --help | Select-String -AllMatches -Pattern 'Visual Studio \d+ \d{4}').Matches | ForEach-Object Value | Sort-Object -Unique | Sort-Object { [int](($_ -split ' ')[-1]) } -Descending | Select-Object -First 1) - if (-not $generator) { throw 'No Visual Studio CMake generator was found.' } + $vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio/Installer/vswhere.exe' + $installations = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 Microsoft.VisualStudio.Component.VC.Tools.ARM64 -format json | ConvertFrom-Json + $installation = $installations | Select-Object -First 1 + if (-not $installation) { throw 'Visual Studio with both x64 and ARM64 C++ tools is required.' } + $vsPath = $installation.installationPath + $vsMajor = ([Version]$installation.installationVersion).Major + $generator = (cmake --help | Select-String -AllMatches -Pattern "Visual Studio $vsMajor \d{4}").Matches | ForEach-Object Value | Sort-Object -Unique | Select-Object -First 1 + if (-not $generator) { throw "CMake has no generator for the installed Visual Studio $vsMajor." } Write-Host "Using CMake generator: $generator" $root = (Get-Location).Path $x64Build = Join-Path $root 'zstandard_cli/builders/windows_x64/build' $armBuild = Join-Path $root 'zstandard_cli/builders/windows_arm/build' Remove-Item $x64Build,$armBuild -Recurse -Force -ErrorAction SilentlyContinue - cmake -S zstandard_cli/builders/windows_x64 -B $x64Build -G $generator -A x64 + cmake -S zstandard_cli/builders/windows_x64 -B $x64Build -G $generator -A x64 "-DCMAKE_GENERATOR_INSTANCE=$vsPath" if ($LASTEXITCODE -ne 0) { throw 'Windows x64 CMake configuration failed.' } cmake --build $x64Build --config Release --parallel if ($LASTEXITCODE -ne 0) { throw 'Windows x64 build failed.' } - cmake -S zstandard_cli/builders/windows_arm -B $armBuild -G $generator -A ARM64 + cmake -S zstandard_cli/builders/windows_arm -B $armBuild -G $generator -A ARM64 "-DCMAKE_GENERATOR_INSTANCE=$vsPath" if ($LASTEXITCODE -ne 0) { throw 'Windows ARM64 CMake configuration failed.' } cmake --build $armBuild --config Release --parallel if ($LASTEXITCODE -ne 0) { throw 'Windows ARM64 build failed.' } New-Item -ItemType Directory -Force zstandard_cli/lib/src/bin | Out-Null Copy-Item (Join-Path $x64Build 'Release/zstandard_windows.dll') zstandard_cli/lib/src/bin/zstandard_windows_x64.dll Copy-Item (Join-Path $armBuild 'Release/zstandard_windows.dll') zstandard_cli/lib/src/bin/zstandard_windows_arm64.dll - $vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio/Installer/vswhere.exe' - $vsPath = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath - $dumpbin = Get-ChildItem $vsPath -Filter dumpbin.exe -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1 + $dumpbin = Get-ChildItem $vsPath -Filter dumpbin.exe -Recurse -ErrorAction SilentlyContinue | Where-Object { $_.FullName -match '[\\/]bin[\\/]Hostx64[\\/]x64[\\/]dumpbin\.exe$' } | Select-Object -First 1 if (-not $dumpbin) { throw 'dumpbin.exe was not found in the installed Visual Studio toolchain.' } $x64Headers = (& $dumpbin.FullName /headers zstandard_cli/lib/src/bin/zstandard_windows_x64.dll | Out-String) $armHeaders = (& $dumpbin.FullName /headers zstandard_cli/lib/src/bin/zstandard_windows_arm64.dll | Out-String) @@ -355,7 +355,7 @@ jobs: Write-Host 'Windows x64 and ARM64 DLL architectures verified.' - id: candidate name: Commit and push Windows libraries - shell: pwsh + shell: powershell env: RELEASE_BRANCH: ${{ github.ref_name }} run: | @@ -371,8 +371,9 @@ jobs: } elseif ($LASTEXITCODE -ne 0) { throw 'Inspecting Windows library changes failed.' } git push origin "HEAD:$env:RELEASE_BRANCH" if ($LASTEXITCODE -ne 0) { throw 'Pushing Windows libraries failed.' } - "commit=$(git rev-parse HEAD)" >> $env:GITHUB_OUTPUT + $candidateCommit = git rev-parse HEAD if ($LASTEXITCODE -ne 0) { throw 'Reading Windows candidate commit failed.' } + "commit=$candidateCommit" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append candidate_static_checks: name: Candidate static and package checks @@ -382,29 +383,36 @@ jobs: timeout-minutes: 45 needs: [update_files, cli_build_windows_precompiled_libs] steps: - - uses: actions/checkout@v6 + - uses: $/.github/actions/checkout-release-source with: ref: ${{ needs.cli_build_windows_precompiled_libs.outputs.commit }} fetch-depth: 0 - uses: ./.github/actions/setup-flutter with: flutter-version: ${{ env.FLUTTER_VERSION }} + - uses: ./.github/actions/prepare-candidate-dependencies + - name: Test Web integration launcher + run: python3 -m unittest discover -s scripts/tests -p test_web_integration_runner.py -v - name: Verify all release metadata and package tests env: VERSION: ${{ inputs.version }} run: | set -euo pipefail - printf 'dependency_overrides:\n zstandard_native:\n path: ../zstandard_native\n' > zstandard_cli/pubspec_overrides.yaml - printf 'dependency_overrides:\n zstandard_android:\n path: ../zstandard_android\n zstandard_ios:\n path: ../zstandard_ios\n zstandard_linux:\n path: ../zstandard_linux\n zstandard_macos:\n path: ../zstandard_macos\n zstandard_native:\n path: ../zstandard_native\n zstandard_platform_interface:\n path: ../zstandard_platform_interface\n zstandard_web:\n path: ../zstandard_web\n zstandard_windows:\n path: ../zstandard_windows\n' > zstandard/pubspec_overrides.yaml - trap 'rm -f zstandard_cli/pubspec_overrides.yaml zstandard/pubspec_overrides.yaml' EXIT dart scripts/check_pub_metadata.dart --release-version "$VERSION" - (cd zstandard_platform_interface && flutter pub get && flutter analyze && flutter test) - (cd zstandard_native && dart pub get && dart analyze) - (cd zstandard_cli && dart pub get && dart analyze && dart test) - (cd zstandard && flutter pub get && flutter analyze && flutter test) for package in zstandard_platform_interface zstandard_native zstandard_android zstandard_ios zstandard_linux zstandard_macos zstandard_web zstandard_windows zstandard_cli zstandard; do - (cd "$package" && dart format --output=none --set-exit-if-changed .) + ( + cd "$package" + case "$package" in + zstandard_native|zstandard_cli) dart pub get ;; + *) flutter pub get ;; + esac + dart format --output=none --set-exit-if-changed . + ) done + (cd zstandard_platform_interface && flutter analyze && flutter test) + (cd zstandard_native && dart analyze) + (cd zstandard_cli && dart analyze && dart test) + (cd zstandard && flutter analyze && flutter test) git diff --check candidate_android: @@ -415,7 +423,7 @@ jobs: timeout-minutes: 120 needs: [update_files, cli_build_windows_precompiled_libs] steps: - - uses: actions/checkout@v6 + - uses: $/.github/actions/checkout-release-source with: ref: ${{ needs.cli_build_windows_precompiled_libs.outputs.commit }} fetch-depth: 0 @@ -477,7 +485,7 @@ jobs: timeout-minutes: 60 needs: [update_files, cli_build_windows_precompiled_libs] steps: - - uses: actions/checkout@v6 + - uses: $/.github/actions/checkout-release-source with: ref: ${{ needs.cli_build_windows_precompiled_libs.outputs.commit }} fetch-depth: 0 @@ -493,16 +501,22 @@ jobs: printf 'dependency_overrides:\n zstandard_native:\n path: ../../zstandard_native\n zstandard_platform_interface:\n path: ../../zstandard_platform_interface\n' > zstandard_linux/example/pubspec_overrides.yaml printf 'dependency_overrides:\n zstandard_native:\n path: ../zstandard_native\n zstandard_platform_interface:\n path: ../zstandard_platform_interface\n' > zstandard_linux/pubspec_overrides.yaml trap 'rm -f zstandard_linux/pubspec_overrides.yaml zstandard_linux/example/pubspec_overrides.yaml' EXIT - (cd zstandard_linux && flutter pub get && flutter analyze && flutter test) + (cd zstandard_linux && flutter pub get && flutter analyze) (cd zstandard_linux/example && flutter pub get) (cd zstandard_linux/example && flutter build linux --release) - bash scripts/test_linux_integration.sh + plugin_library=$(find zstandard_linux/example/build/linux -type f -path '*/release/bundle/lib/libzstandard_linux_plugin.so' -print -quit) + test -n "$plugin_library" + plugin_directory="$(cd "$(dirname "$plugin_library")" && pwd)" + (cd zstandard_linux && LD_LIBRARY_PATH="$plugin_directory${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" flutter test) cmake_build_dir=$(find zstandard_linux/example/build/linux -type f -name CMakeCache.txt -not -path '*/_deps/*' -print | sed 's#/CMakeCache.txt$##' | sort | tail -n 1) test -n "$cmake_build_dir" cmake_build_dir="$(cd "$cmake_build_dir" && pwd)" (cd zstandard_linux/example && ZSTANDARD_LINUX_ENABLE_TESTS=1 cmake -S linux -B "$cmake_build_dir") cmake --build "$cmake_build_dir" --target zstandard_linux_test --parallel - ctest --test-dir "$cmake_build_dir" --output-on-failure + ctest --test-dir "$cmake_build_dir" --output-on-failure --no-tests=error + # Flutter integration leaves generated_config.cmake pointing at a + # temporary test listener; finish native builds before it is retired. + bash scripts/test_linux_integration.sh candidate_web: name: Candidate Web build and integration @@ -512,7 +526,7 @@ jobs: timeout-minutes: 90 needs: [update_files, cli_build_windows_precompiled_libs] steps: - - uses: actions/checkout@v6 + - uses: $/.github/actions/checkout-release-source with: ref: ${{ needs.cli_build_windows_precompiled_libs.outputs.commit }} fetch-depth: 0 @@ -550,7 +564,7 @@ jobs: timeout-minutes: 75 needs: [update_files, cli_build_windows_precompiled_libs] steps: - - uses: actions/checkout@v6 + - uses: $/.github/actions/checkout-release-source with: ref: ${{ needs.cli_build_windows_precompiled_libs.outputs.commit }} fetch-depth: 0 @@ -558,7 +572,7 @@ jobs: with: flutter-version: ${{ env.FLUTTER_VERSION }} - name: Build and run Windows tests - shell: pwsh + shell: powershell run: | $ErrorActionPreference = 'Stop' @" @@ -626,7 +640,7 @@ jobs: platform: [ios, macos] dependency_manager: [swiftpm, cocoapods] steps: - - uses: actions/checkout@v6 + - uses: $/.github/actions/checkout-release-source with: ref: ${{ needs.cli_build_windows_precompiled_libs.outputs.commit }} fetch-depth: 0 @@ -738,7 +752,7 @@ jobs: - candidate_windows - candidate_apple steps: - - uses: actions/checkout@v6 + - uses: $/.github/actions/checkout-release-source with: ref: ${{ needs.cli_build_windows_precompiled_libs.outputs.commit }} fetch-depth: 0 diff --git a/CHANGELOG.md b/CHANGELOG.md index fcc08b63..25493a8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,33 @@ +## 1.5.3 - Repair nested SDK setup before pub.dev publication + +- Fix nested publication SDK setup by disabling its optional analyzer problem matcher. Preserve fresh OIDC credentials for every package and all publication dry runs. +- Add regression coverage for an inherited composite action path with no matcher asset, including failure before any package upload. +- Retain the accepted publisher, checkout, Apple host cleanup and Android emulator-session fixes from production 1.5.2; regenerate all distributable outputs during deployment preparation. +- Document recovery from a tagged workflow that failed before its first package upload. +- Minimum SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; validation uses Flutter 3.47.2. + +## 1.5.2 - Release recover package publication safely + +- Fix package publication so Flutter resolves only the distributable package, without resolving example apps that mix local and hosted dependencies. +- Renew pub.dev OIDC credentials for each package after the previous package has finished indexing. +- Retain all release-preparation fixes and generated artifacts accepted in production by PR #169; regenerate every distributable artifact in the new deployment preparation phase. +- Document recovery from a partial immutable publication. +- Minimum supported SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; release validation uses Flutter 3.47.2. + +- Make release verification reproducible with job-local Flutter/Dart and native test dependencies, and isolate publication token references from ordinary CI. + +## 1.5.1 - Prepare patch release with complete deployment built artifacts + +- Fix iOS simulator startup by waiting for its log stream before launching the test application; preserve both SwiftPM and CocoaPods support and workspace/pub-cache native-source resolution. +- Isolate Flutter settings and Android Gradle state per CI job, select Java 17 explicitly, and retain both modern AGP 9 and legacy Android consumers. +- Add bounded native/Web/CLI decompression, including unknown-size and concatenated frames, with a 256 MiB default output limit and valid empty-input frames. +- Use byte-safe native isolate transfers and a dedicated WebAssembly worker; improve CLI argument parsing, stdin/stdout handling, collision protection and native-library resolution. +- Use the canonical reviewed zstd source across all native platforms and keep Apple dependency-manager source sets aligned. +- Generate matching WebAssembly/JavaScript assets and rebuild the complete macOS universal, Linux x64/arm64 and Windows x64/ARM64 CLI library set during deployment preparation. +- Validate one complete retained candidate before production promotion and publish the ten packages in dependency order using GitHub OIDC and the protected pub.dev environment. Copilot owns promotion, tag/release creation, recovery and reconciliation. +- Document artifact retention, exact-commit handoff, runner requirements and the distinction between validation builds and publication outputs. +- Compatibility: requires Dart ^3.12.0 and Flutter >=3.44.0; CI is validated with Flutter 3.47.2. These minimums are higher than the previously published 1.5.0 requirements. + ## Unreleased - Added bounded decompression (256 MiB default) with support for unknown-size diff --git a/docs/deployment/RUNBOOK.md b/docs/deployment/RUNBOOK.md index 00400e7e..49ce34fd 100644 --- a/docs/deployment/RUNBOOK.md +++ b/docs/deployment/RUNBOOK.md @@ -14,7 +14,7 @@ 5. Run release contract tests, metadata checks and PR platform CI. No tag or registry credential is needed for these checks. 6. Record the release type and SDK compatibility in the Release issue. For a - patch, select `Patch` and `Automatic`; from `v1.5.0`, Copilot resolves `1.5.1`. + patch, select `Patch` and `Automatic`; from `v1.5.1`, Copilot resolves `1.5.2`. Announce Dart 3.12 / Flutter 3.44 minimums in the release notes. Maintenance changes alone do not authorize deployment. @@ -53,7 +53,9 @@ uses GitHub OIDC; no persistent pub token is required. No platform is published during the native build chain. Once all builds and candidate gates pass, the ten package uploads run in dependency order and wait -for indexing. This is not an atomic registry transaction: if publication stops, +for indexing. Each package renews its OIDC credential after the previous wait. +Resolve only the distributable package with `pub get --no-example`; example +apps remain covered by candidate/CI tests. This is not an atomic registry transaction: if publication stops, use Copilot recovery for the same accepted candidate/tag to continue missing versions, rather than rebuilding and uploading platforms independently. @@ -63,10 +65,23 @@ create independent integration PRs or delete managed branches yourself. ## Recovery -- Preparation failure before promotion: fix the linked branch and rerun. - Native commit steps accept unchanged rebuilt outputs. +- Preparation failure before promotion: fix the linked branch and start a new + preparation through Copilot's authorized launcher when workflow code changed. + GitHub reruns retain the original workflow revision; they suit transient + failures with unchanged code. Native commit steps accept unchanged outputs. +- Checkout reports `pack has unresolved deltas`: deployment checkouts retire + inherited shallow Git metadata in the verified runner workspace before a fresh + full-history fetch. Inspect a refusal rather than deleting worktrees, stashes, + authored changes or another checkout. See + [complete-history checkout](../development/self-hosted-ci.md#complete-history-for-deployment-checkouts). - Remote tip moved: preparation stops rather than silently testing a newer commit. Inspect the concurrent change and start a consistent candidate attempt. +- Promotion-PR CI requires a source correction before acceptance: Copilot has + already frozen the candidate SHA. Disable that PR's auto-merge and use an + explicitly approved recovery for the unpublished candidate; the current + limitation is tracked in [Copilot #732](https://github.com/vypdev/copilot/issues/732). + A replacement needs a complete new preparation and its own evidence. Never + reuse the old attestation or rewrite its stored `sourceSha` in place. - Production content differs: reconcile the production change before preparing a new candidate; do not bypass evidence validation. - Publication interrupted or indexing delayed: inspect the issue failure and @@ -74,6 +89,19 @@ create independent integration PRs or delete managed branches yourself. versions are safely skipped. - Existing tag on recovery: the continuation retries its failed publication workflow, preserving the tag and operation identity. +- Publisher code defect after tagging: a retry runs the same frozen workflow. + Prepare an authorized new patch with the fix, leaving the old tag and public + package versions unchanged. Record the old operation's partial publication; + do not report it as complete. Preserve production fixes in the new candidate + through Copilot's supported branch synchronization before preparation. +- SDK setup reports a missing `dart-analyzer.json`: the package publisher runs + the SDK action inside a composite, whose inherited action path contains no + analyzer matcher. Publication SDK setup disables only that optional matcher + with `problem-matcher: false`; dry runs and required CI stay enabled. For an + already tagged workflow, prepare a new patch with this correction. +- Copilot synchronization verification reports `flutter: command not found`: + provision Flutter for the command's own job, then retry the same supported + command. Do not remove configured tests or manually merge the managed branch. - GitHub Release/checksum finalization failed: rerun the same tag workflow. Existing packages are reused; Copilot verifies its release receipt. - Expired evidence: publication stops. Retention is 90 days; plan a new verified diff --git a/docs/development/project-verification.md b/docs/development/project-verification.md new file mode 100644 index 00000000..dc1dbda8 --- /dev/null +++ b/docs/development/project-verification.md @@ -0,0 +1,59 @@ +# Project dependencies for Copilot verification + +Application CI and Copilot commands run in separate jobs. The Flutter setup in +an Android, Linux, Web or Apple build does not prepare the job that handles an +issue comment. A branch synchronization can therefore fail with +`flutter: command not found` even when platform CI is passing. + +The repository uses Copilot's supported project dependency hook at +`.github/actions/copilot-verification-setup/action.yml`. This composite belongs +to this project. It provisions Flutter 3.47.2 and its bundled Dart SDK inside +the verification job, reusing the same versioned runner tool cache as normal +CI. Linux compiler/GTK dependencies use the existing rootless private package +bootstrap, shared with platform CI. Reviewed launchers carry native include and +library settings through Copilot's supported PATH handoff. The preparation +does not require global Flutter or package installation in the runner service. +All remote Action references use version tags. + +The installed comment workflows already call Copilot's reusable coordinator. +They do not need modification to use this hook. The Copilot engine, approval +policy, configured verification commands, managed branches and deployment +lifecycle remain unchanged. + +Ordinary application jobs also isolate Dart's configuration, including commands +that invoke a freshly installed Dart SDK directly. The package archive cache +can be reused, while pub.dev token references stay in the job's private profile. +A publication job must not leave later public dependency reads dependent on its +short-lived `PUB_TOKEN` environment variable. + +## Review and activation + +1. Keep `VERIFICATION_SETUP=false` while preparing and reviewing the hook. + Enabling the generated placeholder does not install dependencies; it fails. +2. Run **Project verification setup contract** on the maintenance PR. It installs + the project hook in the same Linux job as `bash scripts/test_all.sh`, with + a private dependency profile. The check requires the tests to pass without + tracked changes or new publishable files. Platform CI remains required. + The test runner resolves local candidate packages, builds the actual Linux + example plugin before its FFI unit tests and restores existing local + dependency overrides after success or failure. These builds are validation + hosts; distributable generation remains in release preparation. +3. Review and merge the dependency configuration onto trusted `master` before + enabling `VERIFICATION_SETUP=true`. Copilot reads the trusted main snapshot; + a change only on a candidate branch cannot replace that setup. +4. Retry the original supported command, such as + `/copilot sync-branch --from master --no-agent`, and inspect its actual native + setup and verification steps. Successful product CI validates the project + hook, while the command run verifies its integration with Copilot isolation. + +Copilot provides a fresh private dependency profile for each configured check. +Setup may install dependencies but must leave tracked and publishable source +unchanged. It runs with a five-minute setup deadline and fails closed if setup +or verification fails. Keep configured tests intact and diagnose missing native +test prerequisites through the project CI contract. + +To disable the project hook, set `VERIFICATION_SETUP=false`. Future commands +then need their dependencies provisioned in their own job. Disabling the hook +does not alter any accepted candidate, stable tag or published package. + +See the primary [Copilot project setup contract](https://github.com/vypdev/copilot/blob/v3/docs/security-operations/operations/verification.mdx#repository-owned-setup). diff --git a/docs/development/release-process.md b/docs/development/release-process.md index 5cdea1aa..5703b93e 100644 --- a/docs/development/release-process.md +++ b/docs/development/release-process.md @@ -31,7 +31,7 @@ pub.dev publication steps. Do not create a competing branch or merge flow. For a patch release, use the installed Release form with **Release Type: Patch** and **Release Version: Automatic**. Copilot resolves the next patch -from the latest stable tag; for example, `v1.5.0` advances to `1.5.1`. Record +from the latest stable tag; for example, `v1.5.1` advances to `1.5.2`. Record the resolved version in the issue's control center before starting deployment. The release notes must announce the current minimums: Dart `^3.12.0` and @@ -55,8 +55,19 @@ preparation; do not bump or generate them locally before starting that phase. release branch and passes its exact commit to the next job. Unchanged outputs can be retried; a moved remote rejects a push. No package is published while this build chain is running. + The Windows CLI producer uses GitHub's standard `windows-2025` image with + both MSVC toolsets; it does not need Flutter to compile the C libraries. + Windows application validation runs on the trusted self-hosted desktops. 5. All candidate tests check out the final native-build commit. Static, formatting, package, platform release-build and integration gates must pass. + Resolve all ten packages against the local candidate before formatting, + so each formatter uses its declared Dart language version. Linux package + tests load the plugin from the freshly built release bundle, and native + CTest runs before integration retires Flutter's temporary test entry point. + Native test discovery is enabled at the example's CMake root; an empty + CTest suite fails both release and normal Linux CI. + Web integration starts its driver inside Xvfb without relying on exported + Bash functions. The final gate rejects a moved release tip and uploads `release-candidate-X.Y.Z`, recording commit/tree, issue and run/attempt. 6. Only then does `prepare_deployment_action` open Copilot's normal production @@ -124,7 +135,13 @@ zstandard_platform_interface → zstandard_native ``` Each unpublished package must pass `dart pub publish --dry-run`, and the -workflow waits for indexing between packages. Only after all ten versions are +workflow waits for indexing between packages. Each package receives a fresh +OIDC credential after the preceding indexing wait; a token issued once for the +whole sequence can expire before later uploads. Dependency resolution uses +`pub get --no-example`, so publication checks the distributable package without +resolving example applications that intentionally use local path dependencies. +The Dart SDK matches the version bundled with the tested Flutter SDK. +Only after all ten versions are independently verified does Copilot create/verify the GitHub Release, receive the checksum attachment and run `published_deployment_action`. Copilot then performs its configured protected reconciliation. The old independent @@ -144,6 +161,18 @@ moving it. The tag workflow can also be rerun directly after inspecting the failure. Published versions are skipped; reconciliation starts only after all ten packages are verified. +Rerunning a tag workflow uses that tag's original workflow code. An environment +or transient registry failure can be corrected and retried, but a defect in the +frozen publisher requires a corrected new release/version. Keep existing tags +and already published versions unchanged, record the old release as partial, +and obtain authorization for the next patch. Before preparing its candidate, +retain accepted production fixes through Copilot's supported synchronization +command in the new release issue, for example +`/copilot sync-branch --from master --no-agent`. The command must pass the +configured verification, with Flutter available in its own job; a separate +platform CI run does not provision that job. Do not synchronize managed remote +branches manually or mark a partial operation as published. + Evidence is retained for 90 days. Missing, expired or mismatched evidence blocks publication and cannot be replaced by an operator assertion. Prepare a new candidate/version when an old frozen operation cannot be verified. diff --git a/docs/development/self-hosted-ci.md b/docs/development/self-hosted-ci.md index a3a2682d..fe12ecd0 100644 --- a/docs/development/self-hosted-ci.md +++ b/docs/development/self-hosted-ci.md @@ -1,5 +1,9 @@ # Self-hosted CI contract +Copilot command verification prepares its project SDK in its own Action job; +platform job setup does not carry over. See +[project dependency setup and activation](project-verification.md). + The repository uses self-hosted runners for platform checks that cannot run reliably on a generic hosted image. The automated scope covers Android, Linux, Web, Apple Silicon macOS/iOS, and Windows X64. Apple and Windows require their corresponding hardware and toolchain. ## Linux runner labels @@ -84,6 +88,10 @@ marked directory. The shared SDK stays unchanged; see the Windows jobs target `[self-hosted, Windows, X64]`. The workflow installs and caches the pinned Flutter SDK on each runner, enables Windows desktop support, verifies Git and CMake, and checks that the runner can create symbolic links. The runner must provide a Visual Studio C++ desktop toolchain and Windows Developer Mode (or equivalent `SeCreateSymbolicLinkPrivilege` for the runner service account). Integration tests launch a real Windows Flutter application, so the runner must have an interactive desktop session; a headless service session is not sufficient for this job. +Windows CI and release jobs use the built-in Windows PowerShell (`powershell`), including runners without PowerShell 7 (`pwsh`). Release preparation selects a Visual Studio instance with both x64 and ARM64 tools, passes that exact instance to CMake and writes GitHub output files explicitly as UTF-8; Windows PowerShell 5.1 otherwise redirects output as UTF-16. The CLI library producer uses GitHub's standard `windows-2025` image, which includes both toolsets; Windows application integration stays on the self-hosted desktop runners described here. + +Windows setup resolves CMake and CTest from the job's current `PATH`, a standalone CMake installation, or Visual Studio's bundled CMake tools, then adds their directory to the job's `PATH`. A machine-wide `PATH` change is not required; both executables must be installed and runnable. + The workflow builds `zstandard_windows/example` in Debug mode, runs the generated native test target (including a zstd compression/decompression round trip), runs the Windows package tests with the built DLL on `PATH`, and launches the example for Flutter integration tests. The same sequence is repeated with the current `zstandard_native` package staged outside the checkout and selected through `package_config.json`. A separate federated-example job builds `zstandard/example` and runs its integration suite. ## Web job requirements @@ -109,3 +117,46 @@ These CI builds produce disposable validation hosts. Publishable WebAssembly and precompiled CLI libraries are regenerated by the Copilot deployment workflow and retained in its tested candidate. See [artifact storage and handoff](release-process.md#artifact-storage-and-handoff). + +## Android sessions on shared Linux hosts + +Several runner services on one Linux machine share the Android SDK and ADB +daemon. The Android launcher leases the whole SDK/emulator/test session through +a private, user-scoped host lock, including cleanup. A workflow's matrix limit +alone cannot coordinate another PR or release workflow. Waiting for this lease +is bounded by `ANDROID_CI_LOCK_TIMEOUT_SECONDS` (3600 seconds by default). +Android PR test jobs allow 120 minutes for provisioning, waiting and tests. + +By default, each session uses a job-local AVD home and a unique AVD name. The launcher +exports the selected `ANDROID_SERIAL`, refuses an already active device on its +port and terminates only the emulator process it started. It never kills an +existing device just to make room. The lease stays in a parent process, so an +ADB daemon cannot retain it; cancellation is forwarded to the worker and its +cleanup finishes before the lease is released. + +## Complete history for deployment checkouts + +Release preparation and tag validation use the project-owned +`checkout-release-source` composite. GitHub's [checkout-free local Action +syntax](https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/find-and-customize-actions) +(`$/`) loads it from the workflow's exact revision before the workspace checkout. +External Actions still use version tags. + +An inherited shallow checkout can fail a full-history fetch with +`pack has unresolved deltas`, even after checkout's built-in retries. Before +fetching release history, the composite verifies the runner workspace hierarchy, +repository identity and clean source state. It moves only shallow Git metadata +to private job temporary storage, retaining the old cache for that job, and lets +the official checkout action fetch a new complete repository. It leaves complete +checkouts alone and refuses linked worktrees, locks, stashes, unexpected files +or local branches absent from cached remote refs. It does not modify the runner's +global Git configuration or any remote branch or tag. The runner's normal job +temporary-directory cleanup retires the cached metadata. + +Apple validation jobs save the tracked Xcode project and scheme files before +Flutter runs and restore their exact starting contents in an `always()` step +after integration tests. Flutter's SwiftPM migrations are test-host changes; +leaving them in a shared checkout can block the next release's clean-source +check. If an older job left those migrations behind, run the corrected Apple +PR checks before preparing again. Their ordinary checkout and final restoration +clean the CI host without weakening the deployment checkout's source guards. diff --git a/docs/development/testing.md b/docs/development/testing.md index e44f7fa5..15b209ec 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -105,6 +105,13 @@ dart test ./scripts/test_all.sh ``` +This command requires Python 3 and Flutter/Dart. It resolves the matching local +packages before testing, preserves existing local dependency overrides and +restores them when it finishes. On Linux it builds the example's actual plugin +before running FFI unit tests, so the native compiler and GTK development tools +must be available. The resulting debug host is a test output; release artifacts +are generated separately by deployment preparation. + Use `./scripts/test_all_integration.sh` for full coverage without skips (see above). ### Integration tests (main plugin) diff --git a/scripts/android_ci_session.py b/scripts/android_ci_session.py new file mode 100644 index 00000000..e66d9df3 --- /dev/null +++ b/scripts/android_ci_session.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Lease the host's shared Android SDK/ADB/emulator for one complete CI session.""" + +import fcntl +import os +from pathlib import Path +import signal +import stat +import subprocess +import sys +import time + +HANDOFF = 'ZSTANDARD_ANDROID_SESSION_PARENT' + + +def run_session(command, lock_path, wait_seconds=3600): + flags = os.O_CREAT | os.O_RDWR | getattr(os, 'O_NOFOLLOW', 0) + descriptor = os.open(lock_path, flags, 0o600) + with os.fdopen(descriptor, 'a') as lease: + information = os.fstat(lease.fileno()) + if not stat.S_ISREG(information.st_mode) or information.st_uid != os.getuid(): + raise PermissionError('Android CI lease must be an owned regular file.') + deadline = time.monotonic() + wait_seconds + waiting = False + while True: + try: + fcntl.flock(lease, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + if time.monotonic() >= deadline: + print('Timed out waiting for the shared Android CI session.', file=sys.stderr) + return 75 + if not waiting: + print('Waiting for the shared Android CI session.', flush=True) + waiting = True + time.sleep(min(0.1, max(0, deadline - time.monotonic()))) + print('Acquired the shared Android CI session.', flush=True) + child = None + pending = [] + + def forward(signum, _frame): + if child is None: + pending.append(signum) + return + try: + os.killpg(child.pid, signum) + except ProcessLookupError: + pass + + signals = [signal.SIGINT, signal.SIGTERM, signal.SIGHUP] + previous = {number: signal.signal(number, forward) for number in signals} + try: + # The lease stays in this parent. SDK daemons must never inherit it. + child = subprocess.Popen(command, start_new_session=True, close_fds=True, + env={**os.environ, HANDOFF: str(os.getpid())}) + for number in pending: + forward(number, None) + result = child.wait() + return result if result >= 0 else 128 - result + finally: + if child is not None and child.poll() is None: + forward(signal.SIGTERM, None) + try: + child.wait(timeout=30) + except subprocess.TimeoutExpired: + forward(signal.SIGKILL, None) + child.wait() + for number, handler in previous.items(): + signal.signal(number, handler) + + +def main(arguments): + if not arguments: + raise ValueError('An Android CI session command is required.') + wait_seconds = int(os.environ.get('ANDROID_CI_LOCK_TIMEOUT_SECONDS', '3600')) + if not 1 <= wait_seconds <= 7200: + raise ValueError('ANDROID_CI_LOCK_TIMEOUT_SECONDS must be between 1 and 7200.') + # Runner services share this user-level host resource, regardless of repo, + # workflow, branch or their separate RUNNER_TEMP directories. + directory = Path('/tmp') / f'zstandard-android-ci-{os.getuid()}' + directory.mkdir(mode=0o700, exist_ok=True) + information = directory.lstat() + if not stat.S_ISDIR(information.st_mode) or information.st_uid != os.getuid() \ + or stat.S_IMODE(information.st_mode) & 0o077: + raise PermissionError('Android CI lease directory must be private and owned.') + return run_session(arguments, directory / 'session.lock', wait_seconds) + + +if __name__ == '__main__': + try: + raise SystemExit(main(sys.argv[1:])) + except (OSError, ValueError) as error: + print(f'Android CI session failed: {error}', file=sys.stderr) + raise SystemExit(1) diff --git a/scripts/run_android_emulator_ci.sh b/scripts/run_android_emulator_ci.sh index ee97e847..40d151d8 100644 --- a/scripts/run_android_emulator_ci.sh +++ b/scripts/run_android_emulator_ci.sh @@ -10,7 +10,7 @@ fi api_level="${ANDROID_API_LEVEL:-30}" target="${ANDROID_SYSTEM_IMAGE_TARGET:-aosp_atd}" arch="${ANDROID_SYSTEM_IMAGE_ARCH:-x86_64}" -avd_name="${ANDROID_AVD_NAME:-zstandard_ci_test}" +avd_name="${ANDROID_AVD_NAME:-zstandard_ci_test-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-$$}" emulator_port="${EMULATOR_PORT:-5554}" sdk_root="${ANDROID_SDK_ROOT:-${ANDROID_HOME:-}}" @@ -19,6 +19,15 @@ if [[ -z "$sdk_root" || ! -d "$sdk_root" ]]; then exit 1 fi +# Multiple runner services on one machine share the SDK, ADB daemon and port. +# The Python parent owns the lease until this worker and its cleanup finish. +if [[ "${ZSTANDARD_ANDROID_SESSION_PARENT:-}" != "$PPID" ]]; then + script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + exec python3 "$script_directory/android_ci_session.py" \ + bash "$script_directory/run_android_emulator_ci.sh" "$@" +fi +unset ZSTANDARD_ANDROID_SESSION_PARENT + find_sdk_tool() { local tool_name="$1" local candidate @@ -49,7 +58,7 @@ export ANDROID_SDK_ROOT="$sdk_root" export PATH="$sdk_root/platform-tools:$sdk_root/emulator:$PATH" android_user_home="${ANDROID_USER_HOME:-$HOME/.android}" -avd_home="${ANDROID_AVD_HOME:-$android_user_home/avd}" +avd_home="${ANDROID_AVD_HOME:-${RUNNER_TEMP:-$android_user_home}/zstandard-android-avd}" mkdir -p "$avd_home" export ANDROID_USER_HOME="$android_user_home" export ANDROID_AVD_HOME="$avd_home" @@ -93,13 +102,19 @@ printf 'hw.cpu.ncore=2\nhw.gps=no\n' >> "$avd_config" device="emulator-${emulator_port}" export EMULATOR_PORT="$emulator_port" +export ANDROID_SERIAL="$device" emulator_log="${RUNNER_TEMP:-/tmp}/${avd_name}-${emulator_port}.log" rm -f "$emulator_log" adb_timeout_seconds="${ANDROID_ADB_TIMEOUT_SECONDS:-30}" timeout "$adb_timeout_seconds" "$adb_bin" start-server >/dev/null -timeout "$adb_timeout_seconds" "$adb_bin" -s "$device" emu kill >/dev/null 2>&1 || true +connected_devices="$(timeout "$adb_timeout_seconds" "$adb_bin" devices)" +if printf '%s\n' "$connected_devices" | awk -v serial="$device" \ + '$1 == serial { found = 1 } END { exit !found }'; then + echo "Android device $device is already in use; refusing to replace another session." >&2 + exit 1 +fi echo "Starting software Android emulator on port ${emulator_port}..." "$emulator_bin" \ @@ -116,11 +131,12 @@ echo "Starting software Android emulator on port ${emulator_port}..." -camera-back none \ -accel off > "$emulator_log" 2>&1 & emulator_pid=$! +owned_device=false cleanup() { local exit_code=$? - if (( exit_code != 0 )); then + if (( exit_code != 0 )) && [[ "$owned_device" == true ]]; then local diagnostics_file="${RUNNER_TEMP:-/tmp}/${avd_name}-${emulator_port}-diagnostics.txt" { echo "Android integration diagnostics for ${device}" @@ -150,7 +166,6 @@ cleanup() { tail -n 350 "$diagnostics_file" >&2 || true fi - timeout "$adb_timeout_seconds" "$adb_bin" -s "$device" emu kill >/dev/null 2>&1 || true if kill -0 "$emulator_pid" >/dev/null 2>&1; then kill "$emulator_pid" >/dev/null 2>&1 || true wait "$emulator_pid" >/dev/null 2>&1 || true @@ -169,8 +184,18 @@ echo "Waiting for Android emulator boot..." boot_timeout_seconds=900 boot_started_at="$(date +%s)" while true; do + if ! kill -0 "$emulator_pid" >/dev/null 2>&1; then + echo "Android emulator exited before boot completed." >&2 + exit 1 + fi boot_state="$( timeout "$adb_timeout_seconds" "$adb_bin" -s "$device" shell getprop sys.boot_completed 2>/dev/null || true )" if [[ "$boot_state" == *"1"* ]]; then + registered_avd="$(timeout "$adb_timeout_seconds" "$adb_bin" -s "$device" emu avd name 2>/dev/null | sed -n '1{s/\r$//;p;}' || true)" + if [[ "$registered_avd" != "$avd_name" ]]; then + echo "Android port $emulator_port belongs to a different AVD; refusing to run tests." >&2 + exit 1 + fi + owned_device=true echo "Android emulator boot completed." break fi @@ -179,10 +204,6 @@ while true; do echo "Android emulator did not boot within ${boot_timeout_seconds} seconds." >&2 exit 1 fi - if ! kill -0 "$emulator_pid" >/dev/null 2>&1; then - echo "Android emulator exited before boot completed." >&2 - exit 1 - fi sleep 2 done diff --git a/scripts/run_unit_tests.py b/scripts/run_unit_tests.py new file mode 100644 index 00000000..28acdbf8 --- /dev/null +++ b/scripts/run_unit_tests.py @@ -0,0 +1,112 @@ +"""Run the existing unit suites against local packages and a real Linux plugin.""" + +import os +from pathlib import Path +import platform +import stat +import subprocess +import sys +import tempfile + + +PACKAGES = ['zstandard', 'zstandard_platform_interface', 'zstandard_android', + 'zstandard_ios', 'zstandard_macos', 'zstandard_linux', + 'zstandard_windows', 'zstandard_web', 'zstandard_cli'] +OVERRIDDEN = [package for package in PACKAGES + if package != 'zstandard_platform_interface'] + + +def run(command, directory, environment): + return subprocess.run(command, cwd=directory, env=environment, + check=True).returncode + + +def run_suite(root): + root = root.resolve() + environment = dict(os.environ) + snapshots = {} + generated = {} + failed = False + try: + # Preserve authored local overrides; the helper also prepares examples. + for package in OVERRIDDEN: + folders = [root / package] + if package not in ['zstandard', 'zstandard_cli']: + folders.append(root / package / 'example') + if package == 'zstandard_android': + folders.append(root / package / 'example_legacy') + for folder in folders: + if not folder.is_dir(): + continue + path = folder / 'pubspec_overrides.yaml' + if not folder.resolve().is_relative_to(root) or path.is_symlink(): + raise ValueError('Unsafe local override path; no source was changed.') + if path.exists() and not path.is_file(): + raise ValueError('Local override must be a regular file.') + snapshots[path] = (path.read_bytes(), stat.S_IMODE(path.stat().st_mode)) if path.exists() else None + + for package in OVERRIDDEN: + if not (root / package).is_dir(): + continue + try: + run(['dart', str(root / 'scripts/create_local_overrides.dart'), package], root, environment) + finally: + # Capture helper output even when it fails after a partial write. + for path in snapshots: + if path.relative_to(root).parts[0] == package and path.is_file() and not path.is_symlink(): + generated[path] = path.read_bytes() + + for package in PACKAGES: + folder = root / package + if not folder.is_dir(): + print(f'Skip {package} (not found)', flush=True) + continue + print(f'---- Testing {package} ----', flush=True) + sdk = 'dart' if package == 'zstandard_cli' else 'flutter' + try: + run([sdk, 'pub', 'get', '--no-example'], folder, environment) + if package == 'zstandard_linux' and platform.system() == 'Linux': + example = folder / 'example' + run(['flutter', 'pub', 'get', '--no-example'], example, environment) + run(['flutter', 'build', 'linux', '--debug', '--no-pub'], example, environment) + architecture = {'x86_64': 'x64', 'aarch64': 'arm64', 'arm64': 'arm64'}.get(platform.machine()) + if architecture is None: + raise ValueError('Unsupported Linux test architecture.') + libraries = example / 'build/linux' / architecture / 'debug/bundle/lib' + if not (libraries / 'libzstandard_linux_plugin.so').is_file(): + raise ValueError('The Linux build did not produce its actual test plugin.') + environment['LD_LIBRARY_PATH'] = str(libraries) + (os.pathsep + environment['LD_LIBRARY_PATH'] if environment.get('LD_LIBRARY_PATH') else '') + command = [sdk, 'test'] + ([] if sdk == 'dart' else ['--no-pub']) + run(command, folder, environment) + print(f'OK {package}', flush=True) + except (subprocess.CalledProcessError, OSError, ValueError) as error: + print(f'FAILED {package}: {error}', file=sys.stderr, flush=True) + failed = True + except (subprocess.CalledProcessError, OSError, ValueError) as error: + print(f'Unit test preparation failed: {error}', file=sys.stderr, flush=True) + failed = True + finally: + for index, (path, original) in enumerate(snapshots.items()): + expected = generated.get(path) + if expected is None: + continue + if path.is_symlink() or not path.is_file() or path.read_bytes() != expected: + failed = True + print(f'An override changed during testing; retaining it: {path}', file=sys.stderr) + if original is not None: + backup = Path(tempfile.mkdtemp(prefix='zstandard-unit-override-recovery-')) / f'{index}.yaml' + backup.write_bytes(original[0]) + backup.chmod(original[1]) + print(f'Original override retained at {backup}', file=sys.stderr) + continue + if original is None: + path.unlink() + else: + path.write_bytes(original[0]) + path.chmod(original[1]) + print('One or more packages had test failures.' if failed else 'All tests passed.', flush=True) + return int(failed) + + +if __name__ == '__main__': + sys.exit(run_suite(Path(__file__).resolve().parents[1])) diff --git a/scripts/test_all.sh b/scripts/test_all.sh index aa1a2786..f19800ed 100755 --- a/scripts/test_all.sh +++ b/scripts/test_all.sh @@ -1,41 +1,8 @@ #!/usr/bin/env bash # Run all unit tests across zstandard packages. # Usage: from repo root, run: ./scripts/test_all.sh -# Requires: Flutter SDK (for Flutter packages), Dart SDK (for CLI). +# Requires: Python 3, Flutter/Dart and the platform's native build tools. -set -e +set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT" -FAILED=0 - -run_test() { - local dir="$1" - local runner="$2" - if [[ ! -d "$dir" ]]; then - echo "Skip $dir (not found)" - return 0 - fi - echo "---- Testing $dir ----" - if (cd "$dir" && $runner); then - echo "OK $dir" - return 0 - else - echo "FAILED $dir" - FAILED=1 - return 1 - fi -} - -# Flutter packages -for pkg in zstandard zstandard_platform_interface zstandard_android zstandard_ios zstandard_macos zstandard_linux zstandard_windows zstandard_web; do - run_test "$pkg" "flutter test" || true -done - -# CLI is pure Dart -run_test "zstandard_cli" "dart test" || true - -if [[ $FAILED -eq 1 ]]; then - echo "One or more packages had test failures." - exit 1 -fi -echo "All tests passed." +exec python3 "$ROOT/scripts/run_unit_tests.py" diff --git a/scripts/test_web_integration.sh b/scripts/test_web_integration.sh index e5199108..04f3103f 100755 --- a/scripts/test_web_integration.sh +++ b/scripts/test_web_integration.sh @@ -62,7 +62,7 @@ elif [[ -d "$ROOT/zstandard_web/example/integration_test" ]] && [[ -d "$ROOT/zst run_web_driver() { # These must outlive the function body because the EXIT trap runs when - # the exported-function subshell exits (Bash 3.2 included). + # the browser-driver subshell exits (Bash 3.2 included). driver_pid="" driver_log="" @@ -120,13 +120,14 @@ elif [[ -d "$ROOT/zstandard_web/example/integration_test" ]] && [[ -d "$ROOT/zst fi } - export -f chrome_driver_ready run_web_driver export CHROMEDRIVER_PORT export WEB_SERVER_PORT DRIVE_OUTPUT=$(mktemp -t flutter_drive_XXXXXX.txt) if [[ "$(uname -s)" == "Linux" ]] && command -v xvfb-run >/dev/null 2>&1; then - if (cd "$ROOT/zstandard_web/example" && xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' bash -c 'run_web_driver') > "$DRIVE_OUTPUT" 2>&1; then + # Xvfb's /bin/sh wrapper can discard Bash's exported-function variables. + # Pass the trusted definitions as an argument to the child Bash instead. + if (cd "$ROOT/zstandard_web/example" && xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' bash -euo pipefail -c "$(declare -f chrome_driver_ready run_web_driver); run_web_driver") > "$DRIVE_OUTPUT" 2>&1; then DRIVE_EXIT=0 else DRIVE_EXIT=$? diff --git a/scripts/tests/test_android_ci_session.py b/scripts/tests/test_android_ci_session.py new file mode 100644 index 00000000..a360fbde --- /dev/null +++ b/scripts/tests/test_android_ci_session.py @@ -0,0 +1,150 @@ +"""Exercise real process locks and cancellation without using a host Android SDK.""" + +import contextlib +import fcntl +import importlib.util +import io +import json +import os +from pathlib import Path +import select +import signal +import subprocess +import sys +import tempfile +import time +import unittest + +sys.dont_write_bytecode = True +MODULE_PATH = Path(__file__).resolve().parents[1] / 'android_ci_session.py' +SPEC = importlib.util.spec_from_file_location('android_ci_session', MODULE_PATH) +SESSION = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(SESSION) +ENTRY = ''' +import importlib.util,json,sys +from pathlib import Path +sys.dont_write_bytecode=True +spec=importlib.util.spec_from_file_location('session',sys.argv[1]) +module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module) +raise SystemExit(module.run_session(json.loads(sys.argv[4]),Path(sys.argv[2]),float(sys.argv[3]))) +''' + + +class AndroidSessionTests(unittest.TestCase): + def setUp(self): + self.folder = tempfile.TemporaryDirectory(prefix='zstd-android-lock-') + self.addCleanup(self.folder.cleanup) + self.root = Path(self.folder.name) + self.lock = self.root / 'shared host lease' + + def stop(self, process): + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=3) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=3) + process.stdout.close() + process.stderr.close() + + def start(self, command, timeout=5): + process = subprocess.Popen([sys.executable, '-c', ENTRY, str(MODULE_PATH), + str(self.lock), str(timeout), json.dumps(command)], + env={**os.environ, 'PYTHONDONTWRITEBYTECODE': '1'}, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + self.addCleanup(self.stop, process) + return process + + def wait_for(self, path): + deadline = time.monotonic() + 5 + while not path.exists(): + if time.monotonic() >= deadline: + self.fail('The fixture command did not reach its expected barrier.') + time.sleep(0.01) + + def run_now(self, command, timeout=1): + with contextlib.redirect_stdout(io.StringIO()): + return SESSION.run_session(command, self.lock, timeout) + + def test_concurrent_runner_requests_serialize_the_complete_command(self): + first_entered = self.root / 'first entered' + release = self.root / 'release first' + second_entered = self.root / 'second entered' + first = self.start([sys.executable, '-c', + 'from pathlib import Path;import sys,time;' + 'Path(sys.argv[1]).touch();' + 'exec("while not Path(sys.argv[2]).exists(): time.sleep(0.01)")', + str(first_entered), str(release)]) + self.wait_for(first_entered) + second = self.start([sys.executable, '-c', + 'from pathlib import Path;import sys;Path(sys.argv[1]).touch()', + str(second_entered)]) + self.assertTrue(select.select([second.stdout], [], [], 5)[0]) + self.assertEqual(second.stdout.readline().strip(), + 'Waiting for the shared Android CI session.') + self.assertFalse(second_entered.exists()) + release.touch() + first.communicate(timeout=5) + second.communicate(timeout=5) + self.assertEqual(first.returncode, 0) + self.assertEqual(second.returncode, 0) + self.assertTrue(second_entered.exists()) + + def test_command_failure_and_spawn_error_release_the_lease(self): + self.assertEqual(self.run_now([sys.executable, '-c', 'raise SystemExit(43)']), 43) + with self.assertRaises(FileNotFoundError): + self.run_now([str(self.root / 'missing executable')]) + self.assertEqual(self.run_now([sys.executable, '-c', 'pass']), 0) + + def test_lock_timeout_never_runs_the_waiting_command(self): + entered = self.root / 'must not enter' + with self.lock.open('w') as held: + fcntl.flock(held, fcntl.LOCK_EX) + request = self.start([sys.executable, '-c', + 'from pathlib import Path;import sys;Path(sys.argv[1]).touch()', + str(entered)], timeout=0.2) + _, error = request.communicate(timeout=5) + self.assertEqual(request.returncode, 75, error) + self.assertFalse(entered.exists()) + + def test_cancellation_reaches_the_worker_and_releases_the_lease(self): + entered = self.root / 'worker entered' + request = self.start([sys.executable, '-c', + 'from pathlib import Path;import sys,time;' + 'Path(sys.argv[1]).touch();time.sleep(60)', str(entered)]) + self.wait_for(entered) + request.terminate() + request.communicate(timeout=5) + self.assertEqual(request.returncode, 128 + signal.SIGTERM) + self.assertEqual(self.run_now([sys.executable, '-c', 'pass']), 0) + + def test_sdk_daemons_cannot_retain_the_parent_lease(self): + pid_file = self.root / 'daemon pid' + daemon = ('import subprocess,sys;from pathlib import Path;' + 'p=subprocess.Popen([sys.executable,"-c","import time;time.sleep(60)"],' + 'start_new_session=True,close_fds=False,stdout=subprocess.DEVNULL,' + 'stderr=subprocess.DEVNULL);Path(sys.argv[1]).write_text(str(p.pid))') + request = self.start([sys.executable, '-c', daemon, str(pid_file)]) + request.communicate(timeout=5) + self.assertEqual(request.returncode, 0) + pid = int(pid_file.read_text()) + try: + self.assertEqual(self.run_now([sys.executable, '-c', 'pass'], timeout=0.3), 0) + finally: + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass + + def test_symlink_lease_cannot_modify_an_unrelated_file(self): + original = self.root / 'authored source' + original.write_text('retain me') + self.lock.symlink_to(original) + with self.assertRaises(OSError): + self.run_now([sys.executable, '-c', 'raise SystemExit(99)']) + self.assertEqual(original.read_text(), 'retain me') + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/test_android_sdk_provisioning.py b/scripts/tests/test_android_sdk_provisioning.py index 9b368b7e..b8818010 100644 --- a/scripts/tests/test_android_sdk_provisioning.py +++ b/scripts/tests/test_android_sdk_provisioning.py @@ -2,6 +2,7 @@ import os from pathlib import Path import subprocess +import sys import tempfile import unittest @@ -9,12 +10,15 @@ class AndroidProvisioningTests(unittest.TestCase): - def exercise(self, install_fails): + def exercise(self, install_fails, scenario='avd-fails'): with tempfile.TemporaryDirectory() as folder: root = Path(folder) sdk = root / 'sdk' tools = sdk / 'cmdline-tools/latest/bin' tools.mkdir(parents=True) + timeout_fixture = tools / 'timeout' + timeout_fixture.write_text('#!/bin/bash\nshift\nexec "$@"\n') + timeout_fixture.chmod(0o755) trace = root / 'trace' manager = tools / 'sdkmanager' manager.write_text('''#!/bin/bash @@ -23,19 +27,67 @@ def exercise(self, install_fails): printf 'install\\n' >> "$TRACE" if [[ "$INSTALL_FAILS" == true ]]; then exit 42; fi mkdir -p "$ANDROID_SDK_ROOT/platform-tools" "$ANDROID_SDK_ROOT/emulator" -printf '#!/bin/bash\\nexit 0\\n' > "$ANDROID_SDK_ROOT/platform-tools/adb" -printf '#!/bin/bash\\nexit 0\\n' > "$ANDROID_SDK_ROOT/emulator/emulator" +cp "$ADB_FIXTURE" "$ANDROID_SDK_ROOT/platform-tools/adb" +cp "$EMULATOR_FIXTURE" "$ANDROID_SDK_ROOT/emulator/emulator" chmod +x "$ANDROID_SDK_ROOT/platform-tools/adb" "$ANDROID_SDK_ROOT/emulator/emulator" ''') avd = tools / 'avdmanager' - avd.write_text('#!/bin/bash\nprintf "avd\\n" >> "$TRACE"\nexit 43\n') + avd.write_text('''#!/bin/bash +printf 'avd\\n' >> "$TRACE" +if [[ "$SCENARIO" == avd-fails ]]; then exit 43; fi +while [[ $# -gt 0 ]]; do + if [[ "$1" == --name ]]; then shift; name="$1"; fi + shift +done +mkdir -p "$ANDROID_AVD_HOME/$name.avd" +printf 'fixture\\n' > "$ANDROID_AVD_HOME/$name.avd/config.ini" +''') + adb_fixture = root / 'adb fixture' + adb_fixture.write_text('''#!/bin/bash +printf 'adb:%s\\n' "$*" >> "$TRACE" +if [[ "$1" == devices ]]; then + printf 'List of devices attached\\n' + if [[ "$SCENARIO" == occupied ]]; then printf 'emulator-5554\\toffline\\n'; fi +elif [[ "$*" == *'shell getprop'* ]]; then + while [[ ! -e "$EMULATOR_STARTED" ]]; do sleep 0.01; done + printf '1\\n' +elif [[ "$*" == *'emu avd name'* ]]; then + printf 'foreign_avd\\nOK\\n' +fi +''') + emulator_fixture = root / 'emulator fixture' + emulator_fixture.write_text('''#!/bin/bash +printf 'emulator\\n' >> "$TRACE" +touch "$EMULATOR_STARTED" +trap 'exit 0' TERM +while true; do sleep 0.1; done +''') manager.chmod(0o755) avd.chmod(0o755) env = os.environ.copy() + env.pop('ANDROID_USER_HOME', None) + env.pop('ANDROID_AVD_HOME', None) env.update(HOME=str(root), ANDROID_HOME=str(sdk), ANDROID_SDK_ROOT=str(sdk), + RUNNER_TEMP=str(root / 'runner temp'), PATH=str(tools) + ':/usr/bin:/bin', TRACE=str(trace), + SCENARIO=scenario, ADB_FIXTURE=str(adb_fixture), + EMULATOR_FIXTURE=str(emulator_fixture), + EMULATOR_STARTED=str(root / 'emulator started'), + PYTHONDONTWRITEBYTECODE='1', INSTALL_FAILS=str(install_fails).lower()) - result = subprocess.run(['bash', str(SCRIPT), 'true'], env=env, + # Compose the real coordinator/worker with a private fixture lock; + # tests must never wait on or provision the real host session. + entry = ''' +import importlib.util,sys +from pathlib import Path +sys.dont_write_bytecode=True +spec=importlib.util.spec_from_file_location('session',sys.argv[1]) +module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module) +raise SystemExit(module.run_session(['bash',sys.argv[3],'true'],Path(sys.argv[2]),1)) +''' + result = subprocess.run([sys.executable, '-c', entry, + str(SCRIPT.parent / 'android_ci_session.py'), + str(root / 'private lease'), str(SCRIPT)], env=env, text=True, capture_output=True, timeout=10) return result, trace.read_text() if trace.exists() else '' @@ -50,6 +102,21 @@ def test_install_failure_stops_before_avd_or_device_creation(self): self.assertEqual(result.returncode, 42, result.stdout + result.stderr) self.assertEqual(trace, 'install\n') + def test_an_existing_offline_device_is_never_replaced(self): + result, trace = self.exercise(False, scenario='occupied') + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn('already in use', result.stderr) + self.assertNotIn('\nemulator\n', trace) + self.assertNotIn('emu kill', trace) + + def test_a_foreign_booted_avd_cannot_supply_false_readiness(self): + result, trace = self.exercise(False, scenario='foreign') + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn('different AVD', result.stderr) + self.assertIn('\nemulator\n', trace) + self.assertNotIn('emu kill', trace) + self.assertNotIn('dumpsys', trace) + if __name__ == '__main__': unittest.main() diff --git a/scripts/tests/test_unit_test_runner.py b/scripts/tests/test_unit_test_runner.py new file mode 100644 index 00000000..9670cf6c --- /dev/null +++ b/scripts/tests/test_unit_test_runner.py @@ -0,0 +1,134 @@ +"""Exercise candidate resolution, native test ordering and authored-state recovery.""" + +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location('unit_runner', ROOT / 'scripts/run_unit_tests.py') +RUNNER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(RUNNER) +DART = shutil.which('dart') +if DART: + embedded = Path(DART).resolve().parent / 'cache/dart-sdk/bin/dart' + if embedded.is_file(): + DART = str(embedded) + + +@unittest.skipUnless(DART, 'Dart is required for the actual override generator') +class UnitTestRunnerTests(unittest.TestCase): + def setUp(self): + self.folder = tempfile.TemporaryDirectory(prefix='zstd-unit-runner-') + self.addCleanup(self.folder.cleanup) + self.root = Path(self.folder.name).resolve() / 'workspace with spaces' + (self.root / 'scripts').mkdir(parents=True) + shutil.copyfile(ROOT / 'scripts/create_local_overrides.dart', self.root / 'scripts/create_local_overrides.dart') + for name in ['test_all.sh', 'run_unit_tests.py']: + shutil.copyfile(ROOT / 'scripts' / name, self.root / 'scripts' / name) + for package in RUNNER.PACKAGES + ['zstandard_native']: + (self.root / package).mkdir() + if package not in ['zstandard_cli', 'zstandard_platform_interface', 'zstandard_native']: + (self.root / package / 'example').mkdir() + (self.root / 'zstandard_android/example_legacy').mkdir() + tools = self.root / 'tools' + tools.mkdir() + self.trace = self.root / 'trace.jsonl' + tool = '''#!/usr/bin/env python3 +import json, os, sys +from pathlib import Path +args=sys.argv[1:] +sdk=Path(sys.argv[0]).name +folder=Path.cwd() +package=folder.name +if sdk=='dart' and args and args[0].endswith('create_local_overrides.dart'): + os.execv(os.environ['REAL_DART'], [os.environ['REAL_DART'], *args]) +with open(os.environ['TRACE'], 'a') as trace: + trace.write(json.dumps({'sdk':sdk,'args':args,'package':package,'ld':os.environ.get('LD_LIBRARY_PATH')})+'\\n') +if args[:2]==['pub','get']: + assert '--no-example' in args + if package!='zstandard_platform_interface': + assert (folder/'pubspec_overrides.yaml').is_file() +elif args[:2]==['build','linux']: + if os.environ.get('FAIL_BUILD')=='true': sys.exit(7) + library=folder/'build/linux/x64/debug/bundle/lib/libzstandard_linux_plugin.so' + library.parent.mkdir(parents=True,exist_ok=True) + library.write_bytes(b'unit-fixture; actual plugin build is checked by Linux CI') +elif args and args[0]=='test': + if sdk=='flutter': assert '--no-pub' in args + if package=='zstandard_linux' and (sys.platform=='linux' or os.environ.get('EXPECT_LINUX_PLUGIN')=='true'): + library=folder/'example/build/linux/x64/debug/bundle/lib/libzstandard_linux_plugin.so' + assert library.is_file() + assert str(library.parent) in os.environ.get('LD_LIBRARY_PATH','').split(os.pathsep) + if os.environ.get('FAIL_PACKAGE')==package: sys.exit(8) +else: + sys.exit(99) +''' + for name in ['dart', 'flutter']: + executable = tools / name + executable.write_text(tool) + executable.chmod(0o700) + self.environment = {'PATH': str(tools) + os.pathsep + os.environ['PATH'], + 'HOME': self.folder.name, 'REAL_DART': DART, + 'TRACE': str(self.trace)} + + def run_suite(self, **settings): + with patch.dict(os.environ, {**self.environment, 'EXPECT_LINUX_PLUGIN': 'true', **settings}, clear=True), \ + patch.object(RUNNER.platform, 'system', return_value='Linux'), \ + patch.object(RUNNER.platform, 'machine', return_value='x86_64'): + return RUNNER.run_suite(self.root) + + def events(self): + return [json.loads(line) for line in self.trace.read_text().splitlines()] if self.trace.exists() else [] + + def test_runs_every_existing_suite_after_local_resolution_and_real_build_step(self): + self.assertEqual(self.run_suite(), 0) + events = self.events() + self.assertEqual([event['package'] for event in events if event['args'][0] == 'test'], RUNNER.PACKAGES) + build = next(i for i, event in enumerate(events) if event['args'][:2] == ['build', 'linux']) + linux = next(i for i, event in enumerate(events) if event['package'] == 'zstandard_linux' and event['args'][0] == 'test') + self.assertLess(build, linux) + self.assertEqual(list(self.root.glob('**/pubspec_overrides.yaml')), []) + + def test_configured_shell_entry_point_runs_the_same_suites(self): + result = subprocess.run(['bash', str(self.root / 'scripts/test_all.sh')], + cwd=self.folder.name, env=self.environment, + capture_output=True, text=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual([event['package'] for event in self.events() if event['args'][0] == 'test'], RUNNER.PACKAGES) + + def test_original_local_overrides_are_restored_after_a_failed_suite(self): + original = self.root / 'zstandard/pubspec_overrides.yaml' + content = b'dependency_overrides:\n authored_package:\n path: /authored path\n' + original.write_bytes(content) + original.chmod(0o640) + self.assertEqual(self.run_suite(FAIL_PACKAGE='zstandard_android'), 1) + self.assertEqual(original.read_bytes(), content) + self.assertEqual(original.stat().st_mode & 0o777, 0o640) + self.assertEqual([event['package'] for event in self.events() if event['args'][0] == 'test'], RUNNER.PACKAGES) + self.assertEqual(list(self.root.glob('**/pubspec_overrides.yaml')), [original]) + + def test_failed_native_build_cannot_run_linux_tests_but_keeps_other_suites(self): + self.assertEqual(self.run_suite(FAIL_BUILD='true'), 1) + packages = [event['package'] for event in self.events() if event['args'][0] == 'test'] + self.assertNotIn('zstandard_linux', packages) + self.assertEqual(packages, [package for package in RUNNER.PACKAGES if package != 'zstandard_linux']) + self.assertEqual(list(self.root.glob('**/pubspec_overrides.yaml')), []) + + def test_override_symlink_is_rejected_before_any_helper_or_test(self): + outside = Path(self.folder.name) / 'authored-external.yaml' + outside.write_text('authored sentinel\n') + (self.root / 'zstandard/pubspec_overrides.yaml').symlink_to(outside) + self.assertEqual(self.run_suite(), 1) + self.assertEqual(outside.read_text(), 'authored sentinel\n') + self.assertEqual(self.events(), []) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/test_web_integration_runner.py b/scripts/tests/test_web_integration_runner.py new file mode 100644 index 00000000..ce5cdc12 --- /dev/null +++ b/scripts/tests/test_web_integration_runner.py @@ -0,0 +1,90 @@ +"""Exercise the Web launcher across a shell that strips exported functions.""" + +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] + + +class WebIntegrationRunnerTests(unittest.TestCase): + def setUp(self): + self.folder = tempfile.TemporaryDirectory(prefix='zstd-web-runner-') + self.addCleanup(self.folder.cleanup) + self.root = Path(self.folder.name).resolve() / 'workspace with spaces' + (self.root / 'scripts').mkdir(parents=True) + for directory in ['test', 'example/integration_test', 'example/test_driver']: + (self.root / 'zstandard_web' / directory).mkdir(parents=True) + shutil.copyfile(ROOT / 'scripts/test_web_integration.sh', + self.root / 'scripts/test_web_integration.sh') + tools = self.root / 'tools' + tools.mkdir() + self.write_tool(tools / 'uname', '#!/bin/sh\necho Linux\n') + self.write_tool(tools / 'curl', '#!/bin/sh\nexit 0\n') + self.write_tool(tools / 'sleep', '#!/bin/sh\nexit 0\n') + self.write_tool(tools / 'chromedriver', '#!/bin/sh\nexit 99\n') + self.write_tool(tools / 'xvfb-run', '''#!/usr/bin/env python3 +import os,sys +args=sys.argv[1:] +while args and args[0].startswith('--'): + args.pop(0) +env={key:value for key,value in os.environ.items() + if not key.startswith('BASH_FUNC_') and not value.startswith('() {')} +os.execvpe(args[0],args,env) +''') + self.write_tool(tools / 'flutter', '''#!/usr/bin/env python3 +import json,os,sys +from pathlib import Path +Path(os.environ['TRACE']).write_text(json.dumps({'args':sys.argv[1:],'cwd':os.getcwd()})) +mode=os.environ['DRIVER_RESULT'] +if mode=='missing-marker': + print('Driver returned without executing tests.') +else: + print('All tests passed!') +if mode=='compile-failure': + print('Failed to compile application.') +sys.exit(7 if mode=='failed-process' else 0) +''') + self.trace = self.root / 'trace.json' + self.env = {**os.environ, 'PATH': str(tools) + os.pathsep + os.environ['PATH'], + 'BASH_ENV': os.devnull, 'TRACE': str(self.trace), + 'CHROME_EXECUTABLE': '/fixture browser/Chrome Browser', + 'ZSTANDARD_WEB_PORT': '12345'} + self.env.pop('ZSTANDARD_SKIP_WEB', None) + + @staticmethod + def write_tool(path, content): + path.write_text(content) + path.chmod(0o700) + + def run_driver(self, result): + return subprocess.run(['bash', str(self.root / 'scripts/test_web_integration.sh')], + cwd=self.root, env={**self.env, 'DRIVER_RESULT': result}, + capture_output=True, text=True, timeout=20) + + def test_driver_survives_a_wrapper_that_strips_exported_functions(self): + result = self.run_driver('success') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + trace = json.loads(self.trace.read_text()) + self.assertEqual(trace['cwd'], str(self.root / 'zstandard_web/example')) + self.assertEqual(trace['args'][0], 'drive') + self.assertIn('--chrome-binary=/fixture browser/Chrome Browser', trace['args']) + self.assertIn('--driver-port=4444', trace['args']) + self.assertIn('--web-port=12345', trace['args']) + + def test_driver_requires_process_success_and_positive_test_evidence(self): + for mode in ['missing-marker', 'compile-failure', 'failed-process']: + with self.subTest(mode=mode): + self.trace.unlink(missing_ok=True) + result = self.run_driver(mode) + self.assertTrue(self.trace.exists(), result.stdout + result.stderr) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + + +if __name__ == '__main__': + unittest.main() diff --git a/zstandard/CHANGELOG.md b/zstandard/CHANGELOG.md index 8dc6c83c..25493a8a 100644 --- a/zstandard/CHANGELOG.md +++ b/zstandard/CHANGELOG.md @@ -1,13 +1,52 @@ +## 1.5.3 - Repair nested SDK setup before pub.dev publication + +- Fix nested publication SDK setup by disabling its optional analyzer problem matcher. Preserve fresh OIDC credentials for every package and all publication dry runs. +- Add regression coverage for an inherited composite action path with no matcher asset, including failure before any package upload. +- Retain the accepted publisher, checkout, Apple host cleanup and Android emulator-session fixes from production 1.5.2; regenerate all distributable outputs during deployment preparation. +- Document recovery from a tagged workflow that failed before its first package upload. +- Minimum SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; validation uses Flutter 3.47.2. + +## 1.5.2 - Release recover package publication safely + +- Fix package publication so Flutter resolves only the distributable package, without resolving example apps that mix local and hosted dependencies. +- Renew pub.dev OIDC credentials for each package after the previous package has finished indexing. +- Retain all release-preparation fixes and generated artifacts accepted in production by PR #169; regenerate every distributable artifact in the new deployment preparation phase. +- Document recovery from a partial immutable publication. +- Minimum supported SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; release validation uses Flutter 3.47.2. + +- Make release verification reproducible with job-local Flutter/Dart and native test dependencies, and isolate publication token references from ordinary CI. + +## 1.5.1 - Prepare patch release with complete deployment built artifacts + +- Fix iOS simulator startup by waiting for its log stream before launching the test application; preserve both SwiftPM and CocoaPods support and workspace/pub-cache native-source resolution. +- Isolate Flutter settings and Android Gradle state per CI job, select Java 17 explicitly, and retain both modern AGP 9 and legacy Android consumers. +- Add bounded native/Web/CLI decompression, including unknown-size and concatenated frames, with a 256 MiB default output limit and valid empty-input frames. +- Use byte-safe native isolate transfers and a dedicated WebAssembly worker; improve CLI argument parsing, stdin/stdout handling, collision protection and native-library resolution. +- Use the canonical reviewed zstd source across all native platforms and keep Apple dependency-manager source sets aligned. +- Generate matching WebAssembly/JavaScript assets and rebuild the complete macOS universal, Linux x64/arm64 and Windows x64/ARM64 CLI library set during deployment preparation. +- Validate one complete retained candidate before production promotion and publish the ten packages in dependency order using GitHub OIDC and the protected pub.dev environment. Copilot owns promotion, tag/release creation, recovery and reconciliation. +- Document artifact retention, exact-commit handoff, runner requirements and the distinction between validation builds and publication outputs. +- Compatibility: requires Dart ^3.12.0 and Flutter >=3.44.0; CI is validated with Flutter 3.47.2. These minimums are higher than the previously published 1.5.0 requirements. + ## Unreleased -- Added `maxOutputSize` to decompression and extension APIs with a 256 MiB - default, while preserving third-party platform-interface compatibility. -- Standardized null failure semantics and documented unknown-size, - concatenated, and empty-frame behavior across platforms. -- Improved pub.dev package metadata and added regression checks for publication - requirements, including Apple Swift Package Manager manifests. -- Removed the placeholder `WIP` test; the package now reports only meaningful - unit and property checks. +- Added bounded decompression (256 MiB default) with support for unknown-size + and concatenated frames across native, Web, CLI, and extension APIs. +- Moved native work to byte-safe isolates and WebAssembly work to a dedicated + Web Worker; empty input now always produces a valid zstd frame. +- Reworked CLI argument parsing, stdin/stdout support, collision protection, + library resolution, exported-symbol tests, and finite benchmark reporting. +- Pinned native provenance, dependency sources, Apple versions, Emscripten, + third-party Actions, and aligned SwiftPM/CocoaPods native source sets. +- Split release preparation from immutable tag-triggered OIDC publishing and + added a GitHub-hosted, read-only safety gate for untrusted pull requests and + a separate reviewed release-integration workflow. +- Fixed intermittent iOS and macOS build failures caused by deleting synced zstd sources during compilation. +- Updated the minimum supported SDK version to Flutter 3.44/Dart 3.12 and migrated Android builds to Built-in Kotlin on AGP 9+. +- Removed obsolete Android Jetifier settings, stale workflow paths, and 58 MiB of unreferenced repository media. +- Removed placeholder `WIP` tests, isolated the legacy Android example's analysis, + and migrated the CLI away from deprecated platform APIs. +- Added Swift Package Manager support for iOS and macOS while retaining CocoaPods compatibility. ## 1.5.0 - Dependencies Updated diff --git a/zstandard/pubspec.yaml b/zstandard/pubspec.yaml index 55d733e9..807d429f 100644 --- a/zstandard/pubspec.yaml +++ b/zstandard/pubspec.yaml @@ -1,6 +1,6 @@ name: zstandard description: "Flutter plugin for fast Zstandard compression and decompression across native and WebAssembly platforms." -version: 1.5.0 +version: 1.5.3 homepage: https://vyp.dev repository: https://github.com/vypdev/zstandard @@ -28,13 +28,13 @@ flutter: dependencies: flutter: sdk: flutter - zstandard_android: ^1.5.0 - zstandard_ios: ^1.5.0 - zstandard_linux: ^1.5.0 - zstandard_macos: ^1.5.0 - zstandard_platform_interface: ^1.5.0 - zstandard_web: ^1.5.0 - zstandard_windows: ^1.5.0 + zstandard_android: ^1.5.3 + zstandard_ios: ^1.5.3 + zstandard_linux: ^1.5.3 + zstandard_macos: ^1.5.3 + zstandard_platform_interface: ^1.5.3 + zstandard_web: ^1.5.3 + zstandard_windows: ^1.5.3 dev_dependencies: flutter_lints: ^6.0.0 diff --git a/zstandard_android/CHANGELOG.md b/zstandard_android/CHANGELOG.md index 9b62e4ab..25493a8a 100644 --- a/zstandard_android/CHANGELOG.md +++ b/zstandard_android/CHANGELOG.md @@ -1,21 +1,52 @@ +## 1.5.3 - Repair nested SDK setup before pub.dev publication + +- Fix nested publication SDK setup by disabling its optional analyzer problem matcher. Preserve fresh OIDC credentials for every package and all publication dry runs. +- Add regression coverage for an inherited composite action path with no matcher asset, including failure before any package upload. +- Retain the accepted publisher, checkout, Apple host cleanup and Android emulator-session fixes from production 1.5.2; regenerate all distributable outputs during deployment preparation. +- Document recovery from a tagged workflow that failed before its first package upload. +- Minimum SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; validation uses Flutter 3.47.2. + +## 1.5.2 - Release recover package publication safely + +- Fix package publication so Flutter resolves only the distributable package, without resolving example apps that mix local and hosted dependencies. +- Renew pub.dev OIDC credentials for each package after the previous package has finished indexing. +- Retain all release-preparation fixes and generated artifacts accepted in production by PR #169; regenerate every distributable artifact in the new deployment preparation phase. +- Document recovery from a partial immutable publication. +- Minimum supported SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; release validation uses Flutter 3.47.2. + +- Make release verification reproducible with job-local Flutter/Dart and native test dependencies, and isolate publication token references from ordinary CI. + +## 1.5.1 - Prepare patch release with complete deployment built artifacts + +- Fix iOS simulator startup by waiting for its log stream before launching the test application; preserve both SwiftPM and CocoaPods support and workspace/pub-cache native-source resolution. +- Isolate Flutter settings and Android Gradle state per CI job, select Java 17 explicitly, and retain both modern AGP 9 and legacy Android consumers. +- Add bounded native/Web/CLI decompression, including unknown-size and concatenated frames, with a 256 MiB default output limit and valid empty-input frames. +- Use byte-safe native isolate transfers and a dedicated WebAssembly worker; improve CLI argument parsing, stdin/stdout handling, collision protection and native-library resolution. +- Use the canonical reviewed zstd source across all native platforms and keep Apple dependency-manager source sets aligned. +- Generate matching WebAssembly/JavaScript assets and rebuild the complete macOS universal, Linux x64/arm64 and Windows x64/ARM64 CLI library set during deployment preparation. +- Validate one complete retained candidate before production promotion and publish the ten packages in dependency order using GitHub OIDC and the protected pub.dev environment. Copilot owns promotion, tag/release creation, recovery and reconciliation. +- Document artifact retention, exact-commit handoff, runner requirements and the distinction between validation builds and publication outputs. +- Compatibility: requires Dart ^3.12.0 and Flutter >=3.44.0; CI is validated with Flutter 3.47.2. These minimums are higher than the previously published 1.5.0 requirements. + ## Unreleased -- Added byte-safe worker-isolate execution and bounded streaming decompression - for unknown-size and concatenated frames. -- Updated the minimum supported SDK version to Flutter 3.44/Dart 3.12. -- Migrated Android builds to Built-in Kotlin on AGP 9+. -- Removed legacy Kotlin task configuration so the consuming app owns Kotlin - compiler settings. -- Added an AGP 8.11.1/Gradle 8.14 legacy example alongside the AGP 9.1.0 - example. -- Removed obsolete Jetifier configuration and gave the legacy example and its - instrumentation sources a distinct, consistent package identity. -- Restricted the instrumentation-only JNI bridge to Debug builds so test - symbols are absent from production binaries. -- Analyze the independent legacy consumer in its own package context instead - of treating it as part of the plugin package. -- Test debug and release APKs for armeabi-v7a, arm64-v8a, and x86_64, and run - both Dart integration and native JNI instrumentation round-trip tests in CI. +- Added bounded decompression (256 MiB default) with support for unknown-size + and concatenated frames across native, Web, CLI, and extension APIs. +- Moved native work to byte-safe isolates and WebAssembly work to a dedicated + Web Worker; empty input now always produces a valid zstd frame. +- Reworked CLI argument parsing, stdin/stdout support, collision protection, + library resolution, exported-symbol tests, and finite benchmark reporting. +- Pinned native provenance, dependency sources, Apple versions, Emscripten, + third-party Actions, and aligned SwiftPM/CocoaPods native source sets. +- Split release preparation from immutable tag-triggered OIDC publishing and + added a GitHub-hosted, read-only safety gate for untrusted pull requests and + a separate reviewed release-integration workflow. +- Fixed intermittent iOS and macOS build failures caused by deleting synced zstd sources during compilation. +- Updated the minimum supported SDK version to Flutter 3.44/Dart 3.12 and migrated Android builds to Built-in Kotlin on AGP 9+. +- Removed obsolete Android Jetifier settings, stale workflow paths, and 58 MiB of unreferenced repository media. +- Removed placeholder `WIP` tests, isolated the legacy Android example's analysis, + and migrated the CLI away from deprecated platform APIs. +- Added Swift Package Manager support for iOS and macOS while retaining CocoaPods compatibility. ## 1.5.0 - Dependencies Updated diff --git a/zstandard_android/pubspec.yaml b/zstandard_android/pubspec.yaml index 9077ce43..8eb7c5b8 100644 --- a/zstandard_android/pubspec.yaml +++ b/zstandard_android/pubspec.yaml @@ -1,6 +1,6 @@ name: zstandard_android description: Android implementation of the Zstandard compression plugin for Flutter applications. -version: 1.5.0 +version: 1.5.3 homepage: https://vyp.dev repository: https://github.com/vypdev/zstandard/tree/master/zstandard_android @@ -12,8 +12,8 @@ environment: dependencies: flutter: sdk: flutter - zstandard_platform_interface: ^1.5.0 - zstandard_native: ^1.5.0 + zstandard_platform_interface: ^1.5.3 + zstandard_native: ^1.5.3 ffi: ^2.2.0 dev_dependencies: diff --git a/zstandard_cli/CHANGELOG.md b/zstandard_cli/CHANGELOG.md index 5ed19bff..25493a8a 100644 --- a/zstandard_cli/CHANGELOG.md +++ b/zstandard_cli/CHANGELOG.md @@ -1,13 +1,52 @@ +## 1.5.3 - Repair nested SDK setup before pub.dev publication + +- Fix nested publication SDK setup by disabling its optional analyzer problem matcher. Preserve fresh OIDC credentials for every package and all publication dry runs. +- Add regression coverage for an inherited composite action path with no matcher asset, including failure before any package upload. +- Retain the accepted publisher, checkout, Apple host cleanup and Android emulator-session fixes from production 1.5.2; regenerate all distributable outputs during deployment preparation. +- Document recovery from a tagged workflow that failed before its first package upload. +- Minimum SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; validation uses Flutter 3.47.2. + +## 1.5.2 - Release recover package publication safely + +- Fix package publication so Flutter resolves only the distributable package, without resolving example apps that mix local and hosted dependencies. +- Renew pub.dev OIDC credentials for each package after the previous package has finished indexing. +- Retain all release-preparation fixes and generated artifacts accepted in production by PR #169; regenerate every distributable artifact in the new deployment preparation phase. +- Document recovery from a partial immutable publication. +- Minimum supported SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; release validation uses Flutter 3.47.2. + +- Make release verification reproducible with job-local Flutter/Dart and native test dependencies, and isolate publication token references from ordinary CI. + +## 1.5.1 - Prepare patch release with complete deployment built artifacts + +- Fix iOS simulator startup by waiting for its log stream before launching the test application; preserve both SwiftPM and CocoaPods support and workspace/pub-cache native-source resolution. +- Isolate Flutter settings and Android Gradle state per CI job, select Java 17 explicitly, and retain both modern AGP 9 and legacy Android consumers. +- Add bounded native/Web/CLI decompression, including unknown-size and concatenated frames, with a 256 MiB default output limit and valid empty-input frames. +- Use byte-safe native isolate transfers and a dedicated WebAssembly worker; improve CLI argument parsing, stdin/stdout handling, collision protection and native-library resolution. +- Use the canonical reviewed zstd source across all native platforms and keep Apple dependency-manager source sets aligned. +- Generate matching WebAssembly/JavaScript assets and rebuild the complete macOS universal, Linux x64/arm64 and Windows x64/ARM64 CLI library set during deployment preparation. +- Validate one complete retained candidate before production promotion and publish the ten packages in dependency order using GitHub OIDC and the protected pub.dev environment. Copilot owns promotion, tag/release creation, recovery and reconciliation. +- Document artifact retention, exact-commit handoff, runner requirements and the distinction between validation builds and publication outputs. +- Compatibility: requires Dart ^3.12.0 and Flutter >=3.44.0; CI is validated with Flutter 3.47.2. These minimums are higher than the previously published 1.5.0 requirements. + ## Unreleased -- Added bounded streaming decompression for unknown-size and concatenated - frames, and valid compression of empty input. -- Added structured CLI options, stdin/stdout pipelines, safe output naming and - overwrite refusal, stable exit codes, and CWD-independent library loading. -- Added complete native-symbol validation, finite microsecond benchmarks, and - static MSVC runtime linkage for bundled Windows libraries. -- Migrated platform-version detection from deprecated `LocalPlatform` APIs to - `NativePlatform.current`. +- Added bounded decompression (256 MiB default) with support for unknown-size + and concatenated frames across native, Web, CLI, and extension APIs. +- Moved native work to byte-safe isolates and WebAssembly work to a dedicated + Web Worker; empty input now always produces a valid zstd frame. +- Reworked CLI argument parsing, stdin/stdout support, collision protection, + library resolution, exported-symbol tests, and finite benchmark reporting. +- Pinned native provenance, dependency sources, Apple versions, Emscripten, + third-party Actions, and aligned SwiftPM/CocoaPods native source sets. +- Split release preparation from immutable tag-triggered OIDC publishing and + added a GitHub-hosted, read-only safety gate for untrusted pull requests and + a separate reviewed release-integration workflow. +- Fixed intermittent iOS and macOS build failures caused by deleting synced zstd sources during compilation. +- Updated the minimum supported SDK version to Flutter 3.44/Dart 3.12 and migrated Android builds to Built-in Kotlin on AGP 9+. +- Removed obsolete Android Jetifier settings, stale workflow paths, and 58 MiB of unreferenced repository media. +- Removed placeholder `WIP` tests, isolated the legacy Android example's analysis, + and migrated the CLI away from deprecated platform APIs. +- Added Swift Package Manager support for iOS and macOS while retaining CocoaPods compatibility. ## 1.5.0 - Dependencies Updated diff --git a/zstandard_cli/lib/src/bin/libzstandard_linux_arm64.so b/zstandard_cli/lib/src/bin/libzstandard_linux_arm64.so index 33b33b50..4de3c1c3 100755 Binary files a/zstandard_cli/lib/src/bin/libzstandard_linux_arm64.so and b/zstandard_cli/lib/src/bin/libzstandard_linux_arm64.so differ diff --git a/zstandard_cli/lib/src/bin/libzstandard_linux_x64.so b/zstandard_cli/lib/src/bin/libzstandard_linux_x64.so index 2faafc85..9ad2d1c1 100755 Binary files a/zstandard_cli/lib/src/bin/libzstandard_linux_x64.so and b/zstandard_cli/lib/src/bin/libzstandard_linux_x64.so differ diff --git a/zstandard_cli/lib/src/bin/libzstandard_macos.dylib b/zstandard_cli/lib/src/bin/libzstandard_macos.dylib index e950d078..6cfb1e6c 100755 Binary files a/zstandard_cli/lib/src/bin/libzstandard_macos.dylib and b/zstandard_cli/lib/src/bin/libzstandard_macos.dylib differ diff --git a/zstandard_cli/lib/src/bin/zstandard_windows_arm64.dll b/zstandard_cli/lib/src/bin/zstandard_windows_arm64.dll index dd7f41ec..65590bac 100644 Binary files a/zstandard_cli/lib/src/bin/zstandard_windows_arm64.dll and b/zstandard_cli/lib/src/bin/zstandard_windows_arm64.dll differ diff --git a/zstandard_cli/lib/src/bin/zstandard_windows_x64.dll b/zstandard_cli/lib/src/bin/zstandard_windows_x64.dll index e03b0778..b74ad305 100644 Binary files a/zstandard_cli/lib/src/bin/zstandard_windows_x64.dll and b/zstandard_cli/lib/src/bin/zstandard_windows_x64.dll differ diff --git a/zstandard_cli/lib/src/cli_runner.dart b/zstandard_cli/lib/src/cli_runner.dart index 6292f09d..0d538d32 100644 --- a/zstandard_cli/lib/src/cli_runner.dart +++ b/zstandard_cli/lib/src/cli_runner.dart @@ -8,7 +8,7 @@ import 'utils/constants.dart'; import 'zstandard_cli_base.dart'; /// Version of the command-line interface. -const String zstandardCliVersion = '1.5.0'; +const String zstandardCliVersion = '1.5.3'; /// Runs the compression command and returns a process exit code. Future runCompressCommand( diff --git a/zstandard_cli/pubspec.yaml b/zstandard_cli/pubspec.yaml index 489f0fe3..d864e95e 100644 --- a/zstandard_cli/pubspec.yaml +++ b/zstandard_cli/pubspec.yaml @@ -1,6 +1,6 @@ name: zstandard_cli description: Command-line interface for compressing and decompressing files with Zstandard. -version: 1.5.0 +version: 1.5.3 executables: zstandard-compress: compress @@ -16,7 +16,7 @@ dependencies: ffi: ^2.2.0 path: ^1.9.1 platform: ^3.1.6 - zstandard_native: ^1.5.0 + zstandard_native: ^1.5.3 dev_dependencies: coverage: ^1.15.0 diff --git a/zstandard_ios/CHANGELOG.md b/zstandard_ios/CHANGELOG.md index 433461d1..25493a8a 100644 --- a/zstandard_ios/CHANGELOG.md +++ b/zstandard_ios/CHANGELOG.md @@ -1,9 +1,52 @@ +## 1.5.3 - Repair nested SDK setup before pub.dev publication + +- Fix nested publication SDK setup by disabling its optional analyzer problem matcher. Preserve fresh OIDC credentials for every package and all publication dry runs. +- Add regression coverage for an inherited composite action path with no matcher asset, including failure before any package upload. +- Retain the accepted publisher, checkout, Apple host cleanup and Android emulator-session fixes from production 1.5.2; regenerate all distributable outputs during deployment preparation. +- Document recovery from a tagged workflow that failed before its first package upload. +- Minimum SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; validation uses Flutter 3.47.2. + +## 1.5.2 - Release recover package publication safely + +- Fix package publication so Flutter resolves only the distributable package, without resolving example apps that mix local and hosted dependencies. +- Renew pub.dev OIDC credentials for each package after the previous package has finished indexing. +- Retain all release-preparation fixes and generated artifacts accepted in production by PR #169; regenerate every distributable artifact in the new deployment preparation phase. +- Document recovery from a partial immutable publication. +- Minimum supported SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; release validation uses Flutter 3.47.2. + +- Make release verification reproducible with job-local Flutter/Dart and native test dependencies, and isolate publication token references from ordinary CI. + +## 1.5.1 - Prepare patch release with complete deployment built artifacts + +- Fix iOS simulator startup by waiting for its log stream before launching the test application; preserve both SwiftPM and CocoaPods support and workspace/pub-cache native-source resolution. +- Isolate Flutter settings and Android Gradle state per CI job, select Java 17 explicitly, and retain both modern AGP 9 and legacy Android consumers. +- Add bounded native/Web/CLI decompression, including unknown-size and concatenated frames, with a 256 MiB default output limit and valid empty-input frames. +- Use byte-safe native isolate transfers and a dedicated WebAssembly worker; improve CLI argument parsing, stdin/stdout handling, collision protection and native-library resolution. +- Use the canonical reviewed zstd source across all native platforms and keep Apple dependency-manager source sets aligned. +- Generate matching WebAssembly/JavaScript assets and rebuild the complete macOS universal, Linux x64/arm64 and Windows x64/ARM64 CLI library set during deployment preparation. +- Validate one complete retained candidate before production promotion and publish the ten packages in dependency order using GitHub OIDC and the protected pub.dev environment. Copilot owns promotion, tag/release creation, recovery and reconciliation. +- Document artifact retention, exact-commit handoff, runner requirements and the distinction between validation builds and publication outputs. +- Compatibility: requires Dart ^3.12.0 and Flutter >=3.44.0; CI is validated with Flutter 3.47.2. These minimums are higher than the previously published 1.5.0 requirements. + ## Unreleased -- Added byte-safe worker-isolate execution and bounded streaming decompression; - aligned CocoaPods and SwiftPM native sources and pinned SwiftPM releases. -- Fixed intermittent iOS build failures caused by deleting synced zstd sources during compilation. -- Added Swift Package Manager support while retaining CocoaPods compatibility. +- Added bounded decompression (256 MiB default) with support for unknown-size + and concatenated frames across native, Web, CLI, and extension APIs. +- Moved native work to byte-safe isolates and WebAssembly work to a dedicated + Web Worker; empty input now always produces a valid zstd frame. +- Reworked CLI argument parsing, stdin/stdout support, collision protection, + library resolution, exported-symbol tests, and finite benchmark reporting. +- Pinned native provenance, dependency sources, Apple versions, Emscripten, + third-party Actions, and aligned SwiftPM/CocoaPods native source sets. +- Split release preparation from immutable tag-triggered OIDC publishing and + added a GitHub-hosted, read-only safety gate for untrusted pull requests and + a separate reviewed release-integration workflow. +- Fixed intermittent iOS and macOS build failures caused by deleting synced zstd sources during compilation. +- Updated the minimum supported SDK version to Flutter 3.44/Dart 3.12 and migrated Android builds to Built-in Kotlin on AGP 9+. +- Removed obsolete Android Jetifier settings, stale workflow paths, and 58 MiB of unreferenced repository media. +- Removed placeholder `WIP` tests, isolated the legacy Android example's analysis, + and migrated the CLI away from deprecated platform APIs. +- Added Swift Package Manager support for iOS and macOS while retaining CocoaPods compatibility. ## 1.5.0 - Dependencies Updated diff --git a/zstandard_ios/ios/zstandard_ios.podspec b/zstandard_ios/ios/zstandard_ios.podspec index d1014b5b..d52b382b 100644 --- a/zstandard_ios/ios/zstandard_ios.podspec +++ b/zstandard_ios/ios/zstandard_ios.podspec @@ -5,7 +5,7 @@ Pod::Spec.new do |s| s.cocoapods_version = '>= 1.11.0' # for script_phase :before_headers s.name = 'zstandard_ios' - s.version = '1.5.0' + s.version = '1.5.3' s.summary = 'iOS implementation of the Zstandard Flutter plugin.' s.description = <<-DESC Native Zstandard compression and bounded decompression for Flutter on iOS. diff --git a/zstandard_ios/ios/zstandard_ios/Package.swift b/zstandard_ios/ios/zstandard_ios/Package.swift index d655ba5f..08972443 100644 --- a/zstandard_ios/ios/zstandard_ios/Package.swift +++ b/zstandard_ios/ios/zstandard_ios/Package.swift @@ -10,7 +10,7 @@ let nativePackageDependency: Package.Dependency = { // Match the immutable repository tag carrying this plugin release. return .package( url: "https://github.com/vypdev/zstandard.git", - exact: "1.5.0" + exact: "1.5.3" ) }() diff --git a/zstandard_ios/pubspec.yaml b/zstandard_ios/pubspec.yaml index a19febdd..905a5784 100644 --- a/zstandard_ios/pubspec.yaml +++ b/zstandard_ios/pubspec.yaml @@ -1,6 +1,6 @@ name: zstandard_ios description: iOS implementation of the Zstandard compression plugin for Flutter applications. -version: 1.5.0 +version: 1.5.3 homepage: https://vyp.dev repository: https://github.com/vypdev/zstandard/tree/master/zstandard_ios @@ -12,8 +12,8 @@ environment: dependencies: flutter: sdk: flutter - zstandard_platform_interface: ^1.5.0 - zstandard_native: ^1.5.0 + zstandard_platform_interface: ^1.5.3 + zstandard_native: ^1.5.3 ffi: ^2.2.0 dev_dependencies: diff --git a/zstandard_linux/CHANGELOG.md b/zstandard_linux/CHANGELOG.md index 69100c37..25493a8a 100644 --- a/zstandard_linux/CHANGELOG.md +++ b/zstandard_linux/CHANGELOG.md @@ -1,12 +1,52 @@ +## 1.5.3 - Repair nested SDK setup before pub.dev publication + +- Fix nested publication SDK setup by disabling its optional analyzer problem matcher. Preserve fresh OIDC credentials for every package and all publication dry runs. +- Add regression coverage for an inherited composite action path with no matcher asset, including failure before any package upload. +- Retain the accepted publisher, checkout, Apple host cleanup and Android emulator-session fixes from production 1.5.2; regenerate all distributable outputs during deployment preparation. +- Document recovery from a tagged workflow that failed before its first package upload. +- Minimum SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; validation uses Flutter 3.47.2. + +## 1.5.2 - Release recover package publication safely + +- Fix package publication so Flutter resolves only the distributable package, without resolving example apps that mix local and hosted dependencies. +- Renew pub.dev OIDC credentials for each package after the previous package has finished indexing. +- Retain all release-preparation fixes and generated artifacts accepted in production by PR #169; regenerate every distributable artifact in the new deployment preparation phase. +- Document recovery from a partial immutable publication. +- Minimum supported SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; release validation uses Flutter 3.47.2. + +- Make release verification reproducible with job-local Flutter/Dart and native test dependencies, and isolate publication token references from ordinary CI. + +## 1.5.1 - Prepare patch release with complete deployment built artifacts + +- Fix iOS simulator startup by waiting for its log stream before launching the test application; preserve both SwiftPM and CocoaPods support and workspace/pub-cache native-source resolution. +- Isolate Flutter settings and Android Gradle state per CI job, select Java 17 explicitly, and retain both modern AGP 9 and legacy Android consumers. +- Add bounded native/Web/CLI decompression, including unknown-size and concatenated frames, with a 256 MiB default output limit and valid empty-input frames. +- Use byte-safe native isolate transfers and a dedicated WebAssembly worker; improve CLI argument parsing, stdin/stdout handling, collision protection and native-library resolution. +- Use the canonical reviewed zstd source across all native platforms and keep Apple dependency-manager source sets aligned. +- Generate matching WebAssembly/JavaScript assets and rebuild the complete macOS universal, Linux x64/arm64 and Windows x64/ARM64 CLI library set during deployment preparation. +- Validate one complete retained candidate before production promotion and publish the ten packages in dependency order using GitHub OIDC and the protected pub.dev environment. Copilot owns promotion, tag/release creation, recovery and reconciliation. +- Document artifact retention, exact-commit handoff, runner requirements and the distinction between validation builds and publication outputs. +- Compatibility: requires Dart ^3.12.0 and Flutter >=3.44.0; CI is validated with Flutter 3.47.2. These minimums are higher than the previously published 1.5.0 requirements. + ## Unreleased -- Added byte-safe worker-isolate execution and bounded streaming decompression - for unknown-size and concatenated frames. -- Build and run the native Linux CMake test target in CI. -- Extend Linux integration coverage to assert non-empty compression output and - exact byte-for-byte decompression round-trips. -- Removed the placeholder `WIP` test so skipped macOS-hosted runs cannot be - mistaken for executed Linux behavior. +- Added bounded decompression (256 MiB default) with support for unknown-size + and concatenated frames across native, Web, CLI, and extension APIs. +- Moved native work to byte-safe isolates and WebAssembly work to a dedicated + Web Worker; empty input now always produces a valid zstd frame. +- Reworked CLI argument parsing, stdin/stdout support, collision protection, + library resolution, exported-symbol tests, and finite benchmark reporting. +- Pinned native provenance, dependency sources, Apple versions, Emscripten, + third-party Actions, and aligned SwiftPM/CocoaPods native source sets. +- Split release preparation from immutable tag-triggered OIDC publishing and + added a GitHub-hosted, read-only safety gate for untrusted pull requests and + a separate reviewed release-integration workflow. +- Fixed intermittent iOS and macOS build failures caused by deleting synced zstd sources during compilation. +- Updated the minimum supported SDK version to Flutter 3.44/Dart 3.12 and migrated Android builds to Built-in Kotlin on AGP 9+. +- Removed obsolete Android Jetifier settings, stale workflow paths, and 58 MiB of unreferenced repository media. +- Removed placeholder `WIP` tests, isolated the legacy Android example's analysis, + and migrated the CLI away from deprecated platform APIs. +- Added Swift Package Manager support for iOS and macOS while retaining CocoaPods compatibility. ## 1.5.0 - Dependencies Updated diff --git a/zstandard_linux/example/linux/CMakeLists.txt b/zstandard_linux/example/linux/CMakeLists.txt index af714c93..44c480b4 100644 --- a/zstandard_linux/example/linux/CMakeLists.txt +++ b/zstandard_linux/example/linux/CMakeLists.txt @@ -86,7 +86,8 @@ set_target_properties(${BINARY_NAME} RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/intermediates_do_not_run" ) -# Enable the test target. +# Enable discovery from the example's build root and opt in to native tests. +enable_testing() set(include_zstandard_linux_tests TRUE) # Generated plugin build rules, which manage building the plugins and adding diff --git a/zstandard_linux/pubspec.yaml b/zstandard_linux/pubspec.yaml index 9c928aac..672fac31 100644 --- a/zstandard_linux/pubspec.yaml +++ b/zstandard_linux/pubspec.yaml @@ -1,6 +1,6 @@ name: zstandard_linux description: Linux implementation of the Zstandard compression plugin for Flutter applications. -version: 1.5.0 +version: 1.5.3 homepage: https://vyp.dev repository: https://github.com/vypdev/zstandard/tree/master/zstandard_linux @@ -12,8 +12,8 @@ environment: dependencies: flutter: sdk: flutter - zstandard_platform_interface: ^1.5.0 - zstandard_native: ^1.5.0 + zstandard_platform_interface: ^1.5.3 + zstandard_native: ^1.5.3 ffi: ^2.2.0 dev_dependencies: diff --git a/zstandard_macos/CHANGELOG.md b/zstandard_macos/CHANGELOG.md index 4e5b6ee1..25493a8a 100644 --- a/zstandard_macos/CHANGELOG.md +++ b/zstandard_macos/CHANGELOG.md @@ -1,9 +1,52 @@ +## 1.5.3 - Repair nested SDK setup before pub.dev publication + +- Fix nested publication SDK setup by disabling its optional analyzer problem matcher. Preserve fresh OIDC credentials for every package and all publication dry runs. +- Add regression coverage for an inherited composite action path with no matcher asset, including failure before any package upload. +- Retain the accepted publisher, checkout, Apple host cleanup and Android emulator-session fixes from production 1.5.2; regenerate all distributable outputs during deployment preparation. +- Document recovery from a tagged workflow that failed before its first package upload. +- Minimum SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; validation uses Flutter 3.47.2. + +## 1.5.2 - Release recover package publication safely + +- Fix package publication so Flutter resolves only the distributable package, without resolving example apps that mix local and hosted dependencies. +- Renew pub.dev OIDC credentials for each package after the previous package has finished indexing. +- Retain all release-preparation fixes and generated artifacts accepted in production by PR #169; regenerate every distributable artifact in the new deployment preparation phase. +- Document recovery from a partial immutable publication. +- Minimum supported SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; release validation uses Flutter 3.47.2. + +- Make release verification reproducible with job-local Flutter/Dart and native test dependencies, and isolate publication token references from ordinary CI. + +## 1.5.1 - Prepare patch release with complete deployment built artifacts + +- Fix iOS simulator startup by waiting for its log stream before launching the test application; preserve both SwiftPM and CocoaPods support and workspace/pub-cache native-source resolution. +- Isolate Flutter settings and Android Gradle state per CI job, select Java 17 explicitly, and retain both modern AGP 9 and legacy Android consumers. +- Add bounded native/Web/CLI decompression, including unknown-size and concatenated frames, with a 256 MiB default output limit and valid empty-input frames. +- Use byte-safe native isolate transfers and a dedicated WebAssembly worker; improve CLI argument parsing, stdin/stdout handling, collision protection and native-library resolution. +- Use the canonical reviewed zstd source across all native platforms and keep Apple dependency-manager source sets aligned. +- Generate matching WebAssembly/JavaScript assets and rebuild the complete macOS universal, Linux x64/arm64 and Windows x64/ARM64 CLI library set during deployment preparation. +- Validate one complete retained candidate before production promotion and publish the ten packages in dependency order using GitHub OIDC and the protected pub.dev environment. Copilot owns promotion, tag/release creation, recovery and reconciliation. +- Document artifact retention, exact-commit handoff, runner requirements and the distinction between validation builds and publication outputs. +- Compatibility: requires Dart ^3.12.0 and Flutter >=3.44.0; CI is validated with Flutter 3.47.2. These minimums are higher than the previously published 1.5.0 requirements. + ## Unreleased -- Added byte-safe worker-isolate execution and bounded streaming decompression; - aligned CocoaPods and SwiftPM native sources and pinned SwiftPM releases. -- Fixed intermittent macOS build failures caused by deleting synced zstd sources during compilation. -- Added Swift Package Manager support while retaining CocoaPods compatibility. +- Added bounded decompression (256 MiB default) with support for unknown-size + and concatenated frames across native, Web, CLI, and extension APIs. +- Moved native work to byte-safe isolates and WebAssembly work to a dedicated + Web Worker; empty input now always produces a valid zstd frame. +- Reworked CLI argument parsing, stdin/stdout support, collision protection, + library resolution, exported-symbol tests, and finite benchmark reporting. +- Pinned native provenance, dependency sources, Apple versions, Emscripten, + third-party Actions, and aligned SwiftPM/CocoaPods native source sets. +- Split release preparation from immutable tag-triggered OIDC publishing and + added a GitHub-hosted, read-only safety gate for untrusted pull requests and + a separate reviewed release-integration workflow. +- Fixed intermittent iOS and macOS build failures caused by deleting synced zstd sources during compilation. +- Updated the minimum supported SDK version to Flutter 3.44/Dart 3.12 and migrated Android builds to Built-in Kotlin on AGP 9+. +- Removed obsolete Android Jetifier settings, stale workflow paths, and 58 MiB of unreferenced repository media. +- Removed placeholder `WIP` tests, isolated the legacy Android example's analysis, + and migrated the CLI away from deprecated platform APIs. +- Added Swift Package Manager support for iOS and macOS while retaining CocoaPods compatibility. ## 1.5.0 - Dependencies Updated diff --git a/zstandard_macos/macos/zstandard_macos.podspec b/zstandard_macos/macos/zstandard_macos.podspec index 15855f74..0a091da9 100644 --- a/zstandard_macos/macos/zstandard_macos.podspec +++ b/zstandard_macos/macos/zstandard_macos.podspec @@ -5,7 +5,7 @@ Pod::Spec.new do |s| s.cocoapods_version = '>= 1.11.0' # for script_phase :before_headers s.name = 'zstandard_macos' - s.version = '1.5.0' + s.version = '1.5.3' s.summary = 'macOS implementation of the Zstandard Flutter plugin.' s.description = <<-DESC Native Zstandard compression and bounded decompression for Flutter on macOS. diff --git a/zstandard_macos/macos/zstandard_macos/Package.swift b/zstandard_macos/macos/zstandard_macos/Package.swift index dcaed68c..168fefb7 100644 --- a/zstandard_macos/macos/zstandard_macos/Package.swift +++ b/zstandard_macos/macos/zstandard_macos/Package.swift @@ -10,7 +10,7 @@ let nativePackageDependency: Package.Dependency = { // Match the immutable repository tag carrying this plugin release. return .package( url: "https://github.com/vypdev/zstandard.git", - exact: "1.5.0" + exact: "1.5.3" ) }() diff --git a/zstandard_macos/pubspec.yaml b/zstandard_macos/pubspec.yaml index 3280a786..66dcfc8a 100644 --- a/zstandard_macos/pubspec.yaml +++ b/zstandard_macos/pubspec.yaml @@ -1,6 +1,6 @@ name: zstandard_macos description: macOS implementation of the Zstandard compression plugin for Flutter applications. -version: 1.5.0 +version: 1.5.3 homepage: https://vyp.dev repository: https://github.com/vypdev/zstandard/tree/master/zstandard_macos @@ -12,8 +12,8 @@ environment: dependencies: flutter: sdk: flutter - zstandard_platform_interface: ^1.5.0 - zstandard_native: ^1.5.0 + zstandard_platform_interface: ^1.5.3 + zstandard_native: ^1.5.3 ffi: ^2.2.0 dev_dependencies: diff --git a/zstandard_native/CHANGELOG.md b/zstandard_native/CHANGELOG.md index 9f2ff0fc..25493a8a 100644 --- a/zstandard_native/CHANGELOG.md +++ b/zstandard_native/CHANGELOG.md @@ -1,9 +1,52 @@ +## 1.5.3 - Repair nested SDK setup before pub.dev publication + +- Fix nested publication SDK setup by disabling its optional analyzer problem matcher. Preserve fresh OIDC credentials for every package and all publication dry runs. +- Add regression coverage for an inherited composite action path with no matcher asset, including failure before any package upload. +- Retain the accepted publisher, checkout, Apple host cleanup and Android emulator-session fixes from production 1.5.2; regenerate all distributable outputs during deployment preparation. +- Document recovery from a tagged workflow that failed before its first package upload. +- Minimum SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; validation uses Flutter 3.47.2. + +## 1.5.2 - Release recover package publication safely + +- Fix package publication so Flutter resolves only the distributable package, without resolving example apps that mix local and hosted dependencies. +- Renew pub.dev OIDC credentials for each package after the previous package has finished indexing. +- Retain all release-preparation fixes and generated artifacts accepted in production by PR #169; regenerate every distributable artifact in the new deployment preparation phase. +- Document recovery from a partial immutable publication. +- Minimum supported SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; release validation uses Flutter 3.47.2. + +- Make release verification reproducible with job-local Flutter/Dart and native test dependencies, and isolate publication token references from ordinary CI. + +## 1.5.1 - Prepare patch release with complete deployment built artifacts + +- Fix iOS simulator startup by waiting for its log stream before launching the test application; preserve both SwiftPM and CocoaPods support and workspace/pub-cache native-source resolution. +- Isolate Flutter settings and Android Gradle state per CI job, select Java 17 explicitly, and retain both modern AGP 9 and legacy Android consumers. +- Add bounded native/Web/CLI decompression, including unknown-size and concatenated frames, with a 256 MiB default output limit and valid empty-input frames. +- Use byte-safe native isolate transfers and a dedicated WebAssembly worker; improve CLI argument parsing, stdin/stdout handling, collision protection and native-library resolution. +- Use the canonical reviewed zstd source across all native platforms and keep Apple dependency-manager source sets aligned. +- Generate matching WebAssembly/JavaScript assets and rebuild the complete macOS universal, Linux x64/arm64 and Windows x64/ARM64 CLI library set during deployment preparation. +- Validate one complete retained candidate before production promotion and publish the ten packages in dependency order using GitHub OIDC and the protected pub.dev environment. Copilot owns promotion, tag/release creation, recovery and reconciliation. +- Document artifact retention, exact-commit handoff, runner requirements and the distinction between validation builds and publication outputs. +- Compatibility: requires Dart ^3.12.0 and Flutter >=3.44.0; CI is validated with Flutter 3.47.2. These minimums are higher than the previously published 1.5.0 requirements. + ## Unreleased -- Added the shared byte-safe codec with bounded streaming decompression. -- Recorded the exact upstream zstd revision and applied upstream's allocation - null-check backport. -- Made the zstd update script deterministic and safe to rerun. +- Added bounded decompression (256 MiB default) with support for unknown-size + and concatenated frames across native, Web, CLI, and extension APIs. +- Moved native work to byte-safe isolates and WebAssembly work to a dedicated + Web Worker; empty input now always produces a valid zstd frame. +- Reworked CLI argument parsing, stdin/stdout support, collision protection, + library resolution, exported-symbol tests, and finite benchmark reporting. +- Pinned native provenance, dependency sources, Apple versions, Emscripten, + third-party Actions, and aligned SwiftPM/CocoaPods native source sets. +- Split release preparation from immutable tag-triggered OIDC publishing and + added a GitHub-hosted, read-only safety gate for untrusted pull requests and + a separate reviewed release-integration workflow. +- Fixed intermittent iOS and macOS build failures caused by deleting synced zstd sources during compilation. +- Updated the minimum supported SDK version to Flutter 3.44/Dart 3.12 and migrated Android builds to Built-in Kotlin on AGP 9+. +- Removed obsolete Android Jetifier settings, stale workflow paths, and 58 MiB of unreferenced repository media. +- Removed placeholder `WIP` tests, isolated the legacy Android example's analysis, + and migrated the CLI away from deprecated platform APIs. +- Added Swift Package Manager support for iOS and macOS while retaining CocoaPods compatibility. ## 1.5.0 - Dependencies Updated diff --git a/zstandard_native/pubspec.yaml b/zstandard_native/pubspec.yaml index 5511e18d..0a6ac562 100644 --- a/zstandard_native/pubspec.yaml +++ b/zstandard_native/pubspec.yaml @@ -1,6 +1,6 @@ name: zstandard_native description: Native zstd C sources and FFI bindings for the zstandard Flutter plugin. -version: 1.5.0 +version: 1.5.3 homepage: https://vyp.dev repository: https://github.com/vypdev/zstandard/tree/master/zstandard_native diff --git a/zstandard_platform_interface/CHANGELOG.md b/zstandard_platform_interface/CHANGELOG.md index e05b52d7..25493a8a 100644 --- a/zstandard_platform_interface/CHANGELOG.md +++ b/zstandard_platform_interface/CHANGELOG.md @@ -1,7 +1,52 @@ +## 1.5.3 - Repair nested SDK setup before pub.dev publication + +- Fix nested publication SDK setup by disabling its optional analyzer problem matcher. Preserve fresh OIDC credentials for every package and all publication dry runs. +- Add regression coverage for an inherited composite action path with no matcher asset, including failure before any package upload. +- Retain the accepted publisher, checkout, Apple host cleanup and Android emulator-session fixes from production 1.5.2; regenerate all distributable outputs during deployment preparation. +- Document recovery from a tagged workflow that failed before its first package upload. +- Minimum SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; validation uses Flutter 3.47.2. + +## 1.5.2 - Release recover package publication safely + +- Fix package publication so Flutter resolves only the distributable package, without resolving example apps that mix local and hosted dependencies. +- Renew pub.dev OIDC credentials for each package after the previous package has finished indexing. +- Retain all release-preparation fixes and generated artifacts accepted in production by PR #169; regenerate every distributable artifact in the new deployment preparation phase. +- Document recovery from a partial immutable publication. +- Minimum supported SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; release validation uses Flutter 3.47.2. + +- Make release verification reproducible with job-local Flutter/Dart and native test dependencies, and isolate publication token references from ordinary CI. + +## 1.5.1 - Prepare patch release with complete deployment built artifacts + +- Fix iOS simulator startup by waiting for its log stream before launching the test application; preserve both SwiftPM and CocoaPods support and workspace/pub-cache native-source resolution. +- Isolate Flutter settings and Android Gradle state per CI job, select Java 17 explicitly, and retain both modern AGP 9 and legacy Android consumers. +- Add bounded native/Web/CLI decompression, including unknown-size and concatenated frames, with a 256 MiB default output limit and valid empty-input frames. +- Use byte-safe native isolate transfers and a dedicated WebAssembly worker; improve CLI argument parsing, stdin/stdout handling, collision protection and native-library resolution. +- Use the canonical reviewed zstd source across all native platforms and keep Apple dependency-manager source sets aligned. +- Generate matching WebAssembly/JavaScript assets and rebuild the complete macOS universal, Linux x64/arm64 and Windows x64/ARM64 CLI library set during deployment preparation. +- Validate one complete retained candidate before production promotion and publish the ten packages in dependency order using GitHub OIDC and the protected pub.dev environment. Copilot owns promotion, tag/release creation, recovery and reconciliation. +- Document artifact retention, exact-commit handoff, runner requirements and the distinction between validation builds and publication outputs. +- Compatibility: requires Dart ^3.12.0 and Flutter >=3.44.0; CI is validated with Flutter 3.47.2. These minimums are higher than the previously published 1.5.0 requirements. + ## Unreleased -- Added the source-compatible `BoundedZstandardPlatform` capability and a - shared 256 MiB default decompression limit. +- Added bounded decompression (256 MiB default) with support for unknown-size + and concatenated frames across native, Web, CLI, and extension APIs. +- Moved native work to byte-safe isolates and WebAssembly work to a dedicated + Web Worker; empty input now always produces a valid zstd frame. +- Reworked CLI argument parsing, stdin/stdout support, collision protection, + library resolution, exported-symbol tests, and finite benchmark reporting. +- Pinned native provenance, dependency sources, Apple versions, Emscripten, + third-party Actions, and aligned SwiftPM/CocoaPods native source sets. +- Split release preparation from immutable tag-triggered OIDC publishing and + added a GitHub-hosted, read-only safety gate for untrusted pull requests and + a separate reviewed release-integration workflow. +- Fixed intermittent iOS and macOS build failures caused by deleting synced zstd sources during compilation. +- Updated the minimum supported SDK version to Flutter 3.44/Dart 3.12 and migrated Android builds to Built-in Kotlin on AGP 9+. +- Removed obsolete Android Jetifier settings, stale workflow paths, and 58 MiB of unreferenced repository media. +- Removed placeholder `WIP` tests, isolated the legacy Android example's analysis, + and migrated the CLI away from deprecated platform APIs. +- Added Swift Package Manager support for iOS and macOS while retaining CocoaPods compatibility. ## 1.5.0 - Dependencies Updated diff --git a/zstandard_platform_interface/pubspec.yaml b/zstandard_platform_interface/pubspec.yaml index 93bba10d..85f7da75 100644 --- a/zstandard_platform_interface/pubspec.yaml +++ b/zstandard_platform_interface/pubspec.yaml @@ -1,6 +1,6 @@ name: zstandard_platform_interface description: A common platform interface for the zstandard plugin. -version: 1.5.0 +version: 1.5.3 homepage: https://vyp.dev repository: https://github.com/vypdev/zstandard/tree/master/zstandard_platform_interface diff --git a/zstandard_web/CHANGELOG.md b/zstandard_web/CHANGELOG.md index cca7fd9c..25493a8a 100644 --- a/zstandard_web/CHANGELOG.md +++ b/zstandard_web/CHANGELOG.md @@ -1,16 +1,52 @@ +## 1.5.3 - Repair nested SDK setup before pub.dev publication + +- Fix nested publication SDK setup by disabling its optional analyzer problem matcher. Preserve fresh OIDC credentials for every package and all publication dry runs. +- Add regression coverage for an inherited composite action path with no matcher asset, including failure before any package upload. +- Retain the accepted publisher, checkout, Apple host cleanup and Android emulator-session fixes from production 1.5.2; regenerate all distributable outputs during deployment preparation. +- Document recovery from a tagged workflow that failed before its first package upload. +- Minimum SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; validation uses Flutter 3.47.2. + +## 1.5.2 - Release recover package publication safely + +- Fix package publication so Flutter resolves only the distributable package, without resolving example apps that mix local and hosted dependencies. +- Renew pub.dev OIDC credentials for each package after the previous package has finished indexing. +- Retain all release-preparation fixes and generated artifacts accepted in production by PR #169; regenerate every distributable artifact in the new deployment preparation phase. +- Document recovery from a partial immutable publication. +- Minimum supported SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; release validation uses Flutter 3.47.2. + +- Make release verification reproducible with job-local Flutter/Dart and native test dependencies, and isolate publication token references from ordinary CI. + +## 1.5.1 - Prepare patch release with complete deployment built artifacts + +- Fix iOS simulator startup by waiting for its log stream before launching the test application; preserve both SwiftPM and CocoaPods support and workspace/pub-cache native-source resolution. +- Isolate Flutter settings and Android Gradle state per CI job, select Java 17 explicitly, and retain both modern AGP 9 and legacy Android consumers. +- Add bounded native/Web/CLI decompression, including unknown-size and concatenated frames, with a 256 MiB default output limit and valid empty-input frames. +- Use byte-safe native isolate transfers and a dedicated WebAssembly worker; improve CLI argument parsing, stdin/stdout handling, collision protection and native-library resolution. +- Use the canonical reviewed zstd source across all native platforms and keep Apple dependency-manager source sets aligned. +- Generate matching WebAssembly/JavaScript assets and rebuild the complete macOS universal, Linux x64/arm64 and Windows x64/ARM64 CLI library set during deployment preparation. +- Validate one complete retained candidate before production promotion and publish the ten packages in dependency order using GitHub OIDC and the protected pub.dev environment. Copilot owns promotion, tag/release creation, recovery and reconciliation. +- Document artifact retention, exact-commit handoff, runner requirements and the distinction between validation builds and publication outputs. +- Compatibility: requires Dart ^3.12.0 and Flutter >=3.44.0; CI is validated with Flutter 3.47.2. These minimums are higher than the previously published 1.5.0 requirements. + ## Unreleased -- Moved zstd execution to a dedicated Web Worker with transferable buffers. -- Added consistent null semantics, bounded streaming decompression, - unknown-size and concatenated frame support, valid empty frames, and - release-browser integration coverage. -- Pin the Emscripten toolchain used to regenerate the checked-in WebAssembly - artifacts from the canonical `zstandard_native` C source. -- Normalize WebAssembly metadata with the matching `wasm-opt` tool and validate - the freshly regenerated module in the browser; raw WASM layout may still vary - across Linux x64 and macOS arm64. -- Verify WebAssembly regeneration is reproducible in CI before building and - testing the web example. +- Added bounded decompression (256 MiB default) with support for unknown-size + and concatenated frames across native, Web, CLI, and extension APIs. +- Moved native work to byte-safe isolates and WebAssembly work to a dedicated + Web Worker; empty input now always produces a valid zstd frame. +- Reworked CLI argument parsing, stdin/stdout support, collision protection, + library resolution, exported-symbol tests, and finite benchmark reporting. +- Pinned native provenance, dependency sources, Apple versions, Emscripten, + third-party Actions, and aligned SwiftPM/CocoaPods native source sets. +- Split release preparation from immutable tag-triggered OIDC publishing and + added a GitHub-hosted, read-only safety gate for untrusted pull requests and + a separate reviewed release-integration workflow. +- Fixed intermittent iOS and macOS build failures caused by deleting synced zstd sources during compilation. +- Updated the minimum supported SDK version to Flutter 3.44/Dart 3.12 and migrated Android builds to Built-in Kotlin on AGP 9+. +- Removed obsolete Android Jetifier settings, stale workflow paths, and 58 MiB of unreferenced repository media. +- Removed placeholder `WIP` tests, isolated the legacy Android example's analysis, + and migrated the CLI away from deprecated platform APIs. +- Added Swift Package Manager support for iOS and macOS while retaining CocoaPods compatibility. ## 1.5.0 - Dependencies Updated diff --git a/zstandard_web/pubspec.yaml b/zstandard_web/pubspec.yaml index c8849513..67e9cd0e 100644 --- a/zstandard_web/pubspec.yaml +++ b/zstandard_web/pubspec.yaml @@ -1,6 +1,6 @@ name: zstandard_web description: Web implementation of the Zstandard compression plugin for Flutter applications. -version: 1.5.0 +version: 1.5.3 homepage: https://vyp.dev repository: https://github.com/vypdev/zstandard/tree/master/zstandard_web @@ -22,7 +22,7 @@ dependencies: sdk: flutter flutter_web_plugins: sdk: flutter - zstandard_platform_interface: ^1.5.0 + zstandard_platform_interface: ^1.5.3 web: ^1.1.1 dev_dependencies: diff --git a/zstandard_windows/CHANGELOG.md b/zstandard_windows/CHANGELOG.md index d43dfa28..25493a8a 100644 --- a/zstandard_windows/CHANGELOG.md +++ b/zstandard_windows/CHANGELOG.md @@ -1,7 +1,52 @@ +## 1.5.3 - Repair nested SDK setup before pub.dev publication + +- Fix nested publication SDK setup by disabling its optional analyzer problem matcher. Preserve fresh OIDC credentials for every package and all publication dry runs. +- Add regression coverage for an inherited composite action path with no matcher asset, including failure before any package upload. +- Retain the accepted publisher, checkout, Apple host cleanup and Android emulator-session fixes from production 1.5.2; regenerate all distributable outputs during deployment preparation. +- Document recovery from a tagged workflow that failed before its first package upload. +- Minimum SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; validation uses Flutter 3.47.2. + +## 1.5.2 - Release recover package publication safely + +- Fix package publication so Flutter resolves only the distributable package, without resolving example apps that mix local and hosted dependencies. +- Renew pub.dev OIDC credentials for each package after the previous package has finished indexing. +- Retain all release-preparation fixes and generated artifacts accepted in production by PR #169; regenerate every distributable artifact in the new deployment preparation phase. +- Document recovery from a partial immutable publication. +- Minimum supported SDKs remain Dart ^3.12.0 and Flutter >=3.44.0; release validation uses Flutter 3.47.2. + +- Make release verification reproducible with job-local Flutter/Dart and native test dependencies, and isolate publication token references from ordinary CI. + +## 1.5.1 - Prepare patch release with complete deployment built artifacts + +- Fix iOS simulator startup by waiting for its log stream before launching the test application; preserve both SwiftPM and CocoaPods support and workspace/pub-cache native-source resolution. +- Isolate Flutter settings and Android Gradle state per CI job, select Java 17 explicitly, and retain both modern AGP 9 and legacy Android consumers. +- Add bounded native/Web/CLI decompression, including unknown-size and concatenated frames, with a 256 MiB default output limit and valid empty-input frames. +- Use byte-safe native isolate transfers and a dedicated WebAssembly worker; improve CLI argument parsing, stdin/stdout handling, collision protection and native-library resolution. +- Use the canonical reviewed zstd source across all native platforms and keep Apple dependency-manager source sets aligned. +- Generate matching WebAssembly/JavaScript assets and rebuild the complete macOS universal, Linux x64/arm64 and Windows x64/ARM64 CLI library set during deployment preparation. +- Validate one complete retained candidate before production promotion and publish the ten packages in dependency order using GitHub OIDC and the protected pub.dev environment. Copilot owns promotion, tag/release creation, recovery and reconciliation. +- Document artifact retention, exact-commit handoff, runner requirements and the distinction between validation builds and publication outputs. +- Compatibility: requires Dart ^3.12.0 and Flutter >=3.44.0; CI is validated with Flutter 3.47.2. These minimums are higher than the previously published 1.5.0 requirements. + ## Unreleased -- Added byte-safe worker-isolate execution and bounded streaming decompression - for unknown-size and concatenated frames. +- Added bounded decompression (256 MiB default) with support for unknown-size + and concatenated frames across native, Web, CLI, and extension APIs. +- Moved native work to byte-safe isolates and WebAssembly work to a dedicated + Web Worker; empty input now always produces a valid zstd frame. +- Reworked CLI argument parsing, stdin/stdout support, collision protection, + library resolution, exported-symbol tests, and finite benchmark reporting. +- Pinned native provenance, dependency sources, Apple versions, Emscripten, + third-party Actions, and aligned SwiftPM/CocoaPods native source sets. +- Split release preparation from immutable tag-triggered OIDC publishing and + added a GitHub-hosted, read-only safety gate for untrusted pull requests and + a separate reviewed release-integration workflow. +- Fixed intermittent iOS and macOS build failures caused by deleting synced zstd sources during compilation. +- Updated the minimum supported SDK version to Flutter 3.44/Dart 3.12 and migrated Android builds to Built-in Kotlin on AGP 9+. +- Removed obsolete Android Jetifier settings, stale workflow paths, and 58 MiB of unreferenced repository media. +- Removed placeholder `WIP` tests, isolated the legacy Android example's analysis, + and migrated the CLI away from deprecated platform APIs. +- Added Swift Package Manager support for iOS and macOS while retaining CocoaPods compatibility. ## 1.5.0 - Dependencies Updated diff --git a/zstandard_windows/pubspec.yaml b/zstandard_windows/pubspec.yaml index 09c33161..78855f9b 100644 --- a/zstandard_windows/pubspec.yaml +++ b/zstandard_windows/pubspec.yaml @@ -1,6 +1,6 @@ name: zstandard_windows description: Windows implementation of the Zstandard compression plugin for Flutter applications. -version: 1.5.0 +version: 1.5.3 homepage: https://vyp.dev repository: https://github.com/vypdev/zstandard/tree/master/zstandard_windows @@ -12,8 +12,8 @@ environment: dependencies: flutter: sdk: flutter - zstandard_platform_interface: ^1.5.0 - zstandard_native: ^1.5.0 + zstandard_platform_interface: ^1.5.3 + zstandard_native: ^1.5.3 ffi: ^2.2.0 dev_dependencies: