fix: Android fails to resolve native sources from the Pub cache on Windows (#177)

.github/workflows/android_windows_sources.yml

@@ -0,0 +1,122 @@
+name: Android native sources on Windows
+
+on:
+  pull_request:
+    paths:
+      - zstandard_android/**
+      - zstandard_native/**
+      - scripts/check_android_windows_sources.py
+      - scripts/check_android_native_sources.py
+      - scripts/download_android_windows_tools.py
+      - .github/workflows/android_windows_sources.yml
+  push:
+    branches: [develop]
+    paths:
+      - zstandard_android/**
+      - zstandard_native/**
+      - scripts/check_android_windows_sources.py
+      - scripts/check_android_native_sources.py
+      - scripts/download_android_windows_tools.py
+      - .github/workflows/android_windows_sources.yml
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  reproduce-source-resolution:
+    name: Android Pub-cache regression on Windows
+    if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
+    runs-on:
+      group: vypdev
+      labels: [self-hosted, Windows, X64]
+    timeout-minutes: 30
+    env:
+      PYTHONDONTWRITEBYTECODE: '1'
+      PYTHONUTF8: '1'
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - name: Prepare verified job-local Python without a Windows installer
+        shell: powershell
+        env:
+          REPRODUCTION_REPORT: ${{ runner.temp }}/zstandard-android-windows-${{ github.run_id }}-${{ github.run_attempt }}
+        run: |
+          $ErrorActionPreference = 'Stop'
+          $ProgressPreference = 'SilentlyContinue'
+          [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+          New-Item -ItemType Directory -Path $env:REPRODUCTION_REPORT -Force | Out-Null
+          $utf8 = New-Object System.Text.UTF8Encoding($false)
+          $version = '3.13.17'
+          $expected = '9e35319bffedab8b36ad44d8f377cb4f1733bcec257a0b6d04d94f01486ddb38'
+          $url = "https://www.python.org/ftp/python/$version/python-$version-embed-amd64.zip"
+          [IO.File]::WriteAllText((Join-Path $env:REPRODUCTION_REPORT 'python-bootstrap.txt'), "$url`nSHA256 $expected`n", $utf8)
+          $directory = Join-Path $env:RUNNER_TEMP "zstandard-python-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT"
+          New-Item -ItemType Directory -Path $directory | Out-Null
+          $archive = Join-Path $directory 'python.zip'
+          Invoke-WebRequest -UseBasicParsing -Uri $url -OutFile $archive
+          if ((Get-FileHash -Algorithm SHA256 $archive).Hash.ToLowerInvariant() -ne $expected) {
+            throw 'The portable Python archive does not match the official release checksum.'
+          }
+          $runtime = Join-Path $directory 'runtime'
+          Expand-Archive -LiteralPath $archive -DestinationPath $runtime
+          & (Join-Path $runtime 'python.exe') --version
+          if ($LASTEXITCODE -ne 0) { throw 'The job-local Python runtime could not execute.' }
+          [IO.File]::AppendAllText($env:GITHUB_PATH, "$runtime`n", $utf8)
+      - uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: '17'
+      - name: Isolate Gradle state
+        shell: powershell
+        run: |
+          $directory = Join-Path $env:RUNNER_TEMP "zstandard-gradle-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT"
+          New-Item -ItemType Directory -Path $directory | Out-Null
+          $utf8 = New-Object System.Text.UTF8Encoding($false)
+          [IO.File]::WriteAllText((Join-Path $directory 'gradle.properties'), "org.gradle.daemon=false`norg.gradle.workers.max=2`n", $utf8)
+          [IO.File]::AppendAllText($env:GITHUB_ENV, "GRADLE_USER_HOME=$directory`n", $utf8)
+      - uses: gradle/actions/setup-gradle@v4
+        with:
+          gradle-version: '8.14.2'
+          cache-disabled: true
+      - name: Prepare a private Windows Android SDK
+        shell: powershell
+        env:
+          REPRODUCTION_REPORT: ${{ runner.temp }}/zstandard-android-windows-${{ github.run_id }}-${{ github.run_attempt }}
+        run: |
+          $ErrorActionPreference = 'Stop'
+          $utf8 = New-Object System.Text.UTF8Encoding($false)
+          $sdk = Join-Path $env:RUNNER_TEMP ("zstandard-android-sdk-{0}-{1}" -f $env:GITHUB_RUN_ID, $env:GITHUB_RUN_ATTEMPT)
+          python scripts/download_android_windows_tools.py --sdk-root "$sdk" --report-directory "$env:REPRODUCTION_REPORT"
+          if ($LASTEXITCODE -ne 0) { throw 'The reviewed command-line tools could not be prepared.' }
+          $env:ANDROID_HOME = $sdk
+          $env:ANDROID_SDK_ROOT = $sdk
+          $env:ANDROID_USER_HOME = Join-Path $sdk 'user-state'
+          New-Item -ItemType Directory -Path $env:ANDROID_USER_HOME | Out-Null
+          python scripts/download_android_windows_tools.py --sdk-root "$sdk" --report-directory "$env:REPRODUCTION_REPORT" --install-packages
+          if ($LASTEXITCODE -ne 0) { throw 'The private Windows Android SDK could not be provisioned.' }
+          Write-Host ("[{0:o}] Private Android SDK provisioning finished" -f (Get-Date))
+          foreach ($name in @('ANDROID_HOME', 'ANDROID_SDK_ROOT', 'ANDROID_USER_HOME')) {
+            $value = [Environment]::GetEnvironmentVariable($name)
+            [IO.File]::AppendAllText($env:GITHUB_ENV, "$name=$value`n", $utf8)
+          }
+      - name: Reproduce the published failure and validate the candidate resolver
+        shell: powershell
+        env:
+          REPRODUCTION_REPORT: ${{ runner.temp }}/zstandard-android-windows-${{ github.run_id }}-${{ github.run_attempt }}
+        run: |
+          $ErrorActionPreference = 'Stop'
+          python scripts/check_android_windows_sources.py --baseline-version 1.5.3 --gradle (Get-Command gradle.bat).Source --report-directory "$env:REPRODUCTION_REPORT"
+          if ($LASTEXITCODE -ne 0) { throw 'Android source resolver regression failed.' }
+      - name: Retain Windows source resolution and toolchain evidence
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: android-windows-source-reproduction-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/zstandard-android-windows-${{ github.run_id }}-${{ github.run_attempt }}
+          if-no-files-found: error
+          retention-days: 14

docs/development/building.md

@@ -58,10 +58,11 @@ If you are developing or modifying a platform package’s native code:
 
 ### Android
 
-- The plugin builds the native library via `zstandard_android/zstd_build/CMakeLists.txt`, which compiles sources from `zstandard_native/src/zstd/` (resolved from the repo or pub cache).
+- The plugin builds the native library via `zstandard_android/zstd_build/CMakeLists.txt`. Gradle reads the consuming app or workspace's `.dart_tool/package_config.json`, resolves the exact `zstandard_native` file URI and passes its `src/zstd` path to CMake. The plugin and native package can both live in a versioned Pub cache outside the consuming app, including Windows paths with spaces.
 - The Android plugin uses Built-in Kotlin with AGP 9 and newer. The Android examples exercise this path with Flutter 3.47+.
 - Building the Android app (e.g. `flutter build apk` or running from Android Studio) triggers the native build via Gradle/CMake.
 - Ensure the NDK is installed and that `zstandard_native` is available (e.g. `flutter pub get` so the canonical `zstandard_native/src/zstd/` is resolved).
+- For an explicit source override, set the Gradle property `ZSTD_ROOT` (or its environment variable) to the directory containing `zstd.h`; direct CMake callers use `-DZSTD_ROOT=...`. Overrides take precedence. With no consumer package configuration, a direct repository build can use the sibling native package. A missing or invalid configured dependency fails rather than selecting a different cached package.
 
 ### iOS / macOS
 

docs/development/ci-cd.md

@@ -26,6 +26,12 @@ They provide Android emulator/instrumentation, Linux desktop, ChromeDriver,
 Windows desktop/native, and both SwiftPM and CocoaPods Apple integration. They
 also run publish dry-runs and coverage where applicable.
 
+`android_windows_sources.yml` separately verifies Android source discovery on
+a Windows host with both packages staged in a versioned cache outside the
+consumer and builds ARM64 native code. Android execution coverage remains in
+the Linux emulator matrix. See the [Windows Android runner
+contract](self-hosted-ci.md#android-builds-hosted-on-windows).
+
 Actions in workflows and composite actions must use version tags such as
 `@v1`, `@v2` or `@v3`. Commit SHA references are not permitted. The package
 preparation contract checks this policy when workflow or action files change.

docs/development/self-hosted-ci.md

@@ -94,6 +94,34 @@ Windows setup resolves CMake and CTest from the job's current `PATH`, a standalo
 
 The workflow builds `zstandard_windows/example` in Debug mode, runs the generated native test target (including a zstd compression/decompression round trip), runs the Windows package tests with the built DLL on `PATH`, and launches the example for Flutter integration tests. The same sequence is repeated with the current `zstandard_native` package staged outside the checkout and selected through `package_config.json`. A separate federated-example job builds `zstandard/example` and runs its integration suite.
 
+### Android builds hosted on Windows
+
+`android_windows_sources.yml` adds one targeted job for relevant pull requests
+and pushes to `develop`. It reproduces the original published Android source
+lookup failure, then stages both candidate packages outside the checkout and
+outside a separate consuming app. Production Gradle resolver checks cover two
+cached native versions, absolute and relative file URIs, Windows drive paths,
+spaces, Unicode, workspace ancestors, overrides and invalid dependencies.
+NDK 27.0.12077973 with CMake 3.22.1 builds the corrected ARM64/API 21 library.
+
+Linux retains the complete Flutter integration and JNI instrumentation matrix.
+The Windows job covers host-specific source discovery and compilation. It does
+not claim Android runtime coverage. A Windows emulator requires separate
+infrastructure validation: the current runner reported unavailable hardware
+virtualization, and API 30 ATD software-emulator trials did not reach a stable
+boot. Enabling Windows runtime checks requires a reproducible successful boot
+and test execution first.
+
+Portable Python and Android command-line tools use reviewed archive checksums.
+The job requires HTTPS access to Python, Google SDK repositories, Gradle
+distributions and pub.dev archives.
+Java 17 and Gradle 8.14.2 are selected per job. The Android SDK, Gradle state and
+test fixtures live in private job directories, including on cancellation. Only
+the NDK and CMake packages are provisioned; no system image is downloaded for
+the compilation check. The job does not change machine-wide SDK, virtualization
+or PATH settings. Missing tools, failed resolution and failed compilation are
+explicit errors; diagnostic artifacts are retained for 14 days.
+
 ## Web job requirements
 
 Flutter 3.47.2 does not support running `package:integration_test` directly with `flutter test -d chrome`. The workflow therefore uses the supported WebDriver path:

scripts/check_android_native_sources.py

@@ -0,0 +1,156 @@
+"""Exercise the production Gradle resolver with real, host-native file URIs."""
+
+import argparse
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+from urllib.parse import quote
+
+
+def verify(gradle, plugin, native, stage, report):
+    cases = []
+    cache = stage / 'Pub cache/hosted/pub.dev'
+    selected_package = cache / 'zstandard_native-9.8.7 é'
+    # Both versions coexist in the same fixture cache. The source bytes come
+    # from the candidate; these fixture version names are never published.
+    shutil.copytree(native / 'src', selected_package / 'src')
+    native = selected_package
+
+    def sources(directory):
+        root = directory / 'src/zstd'
+        for part in ['common', 'compress', 'decompress']:
+            (root / part).mkdir(parents=True, exist_ok=True)
+        (root / 'zstd.h').touch()
+        return root
+
+    def configuration(consumer, entries):
+        path = consumer / '.dart_tool/package_config.json'
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps({'configVersion': 2, 'packages': entries}), encoding='utf-8')
+        return path
+
+    def entry(uri):
+        return {'name': 'zstandard_native', 'rootUri': uri, 'packageUri': 'lib/'}
+
+    def case(name, consumer, expected=None, error=None, explicit=None, plugin_directory=plugin):
+        consumer.mkdir(parents=True, exist_ok=True)
+        cases.append({'name': name, 'consumer': str(consumer), 'plugin': str(plugin_directory),
+                      'expected': str(expected) if expected else None,
+                      'error': error, 'explicit': str(explicit) if explicit is not None else None})
+
+    selected = native / 'src/zstd'
+    decoy = sources(cache / 'zstandard_native-9.8.8')
+    repository = stage / 'repository'
+    sibling = sources(repository / 'zstandard_native')
+    repository_plugin = repository / 'zstandard_android/android'
+    repository_plugin.mkdir(parents=True)
+
+    absolute = stage / 'separate consumer é/absolute'
+    configuration(absolute, [entry(native.as_uri() + '/')])
+    case('absolute file URI, exact selected version, spaces and Unicode', absolute / 'android', selected)
+
+    relative = stage / 'separate consumer é/relative'
+    config = relative / '.dart_tool/package_config.json'
+    uri = quote(os.path.relpath(native, config.parent).replace(os.sep, '/'), safe='/') + '/'
+    configuration(relative, [entry(uri)])
+    case('relative URI is relative to package_config, not Gradle or plugin', relative / 'android', selected)
+
+    workspace = stage / 'workspace with spaces'
+    configuration(workspace, [entry(native.as_uri() + '/')])
+    case('workspace ancestor configuration', workspace / 'apps/mobile/android', selected)
+
+    case('explicit override precedes configured dependency', absolute / 'android', decoy, explicit=decoy)
+    explicit_relative = os.path.relpath(decoy, absolute / 'android')
+    case('relative explicit override uses consumer directory', absolute / 'android', decoy,
+         explicit=explicit_relative)
+    case('invalid explicit override fails without fallback', absolute / 'android',
+         error='explicit ZSTD_ROOT', explicit=stage / 'missing')
+    case('direct repository fallback without package configuration', repository / 'consumer/android',
+         sibling, plugin_directory=repository_plugin)
+
+    bad = [
+        ('missing native dependency', [], 'Expected exactly one'),
+        ('duplicate native dependencies', [entry(native.as_uri()), entry(decoy.parent.parent.as_uri())], 'Expected exactly one'),
+        ('empty URI', [entry('')], 'Expected exactly one'),
+        ('non-string URI', [entry(42)], 'Expected exactly one'),
+        ('non-file URI', [entry('https://example.invalid/native')], 'Cannot resolve'),
+        ('malformed URI', [entry('file:///%invalid')], 'Cannot resolve'),
+        ('missing selected sources', [entry((stage / 'missing package').as_uri())], 'zstd source not found'),
+    ]
+    for index, (name, entries, message) in enumerate(bad):
+        consumer = stage / f'invalid cases/{index}'
+        configuration(consumer, entries)
+        case(name + ' cannot select a cached or repository fallback', consumer / 'android',
+             error=message, plugin_directory=repository_plugin)
+
+    malformed = stage / 'malformed JSON'
+    config = configuration(malformed, [])
+    config.write_text('{bad JSON', encoding='utf-8')
+    case('malformed JSON reports pub get guidance', malformed / 'android', error='Cannot read')
+    invalid_shape = stage / 'invalid JSON shape'
+    config = configuration(invalid_shape, [])
+    config.write_text('{"packages": {}}', encoding='utf-8')
+    case('invalid package configuration shape', invalid_shape / 'android', error='Invalid package configuration')
+
+    project = stage / 'resolver verification'
+    project.mkdir()
+    (project / 'settings.gradle').write_text("rootProject.name = 'native-source-verification'\n", encoding='utf-8')
+    (project / 'cases.json').write_text(json.dumps(cases), encoding='utf-8')
+    (project / 'build.gradle').write_text('''import groovy.json.JsonSlurper
+import groovy.json.JsonOutput
+apply from: new File(new URI(RESOLVER_URI))
+tasks.register('verifySources') {
+    doLast {
+        def cases = new JsonSlurper().parse(file('cases.json'), 'UTF-8')
+        def results = []
+        cases.each { item ->
+            File resolved = null
+            Exception failure = null
+            try {
+                resolved = resolveZstandardNativeRoot(new File(item.consumer), new File(item.plugin), item.explicit)
+            } catch (Exception error) {
+                failure = error
+            }
+            if (item.error) {
+                assert failure != null : "${item.name}: unexpectedly succeeded"
+                assert failure.message.contains(item.error) : "${item.name}: ${failure.message}"
+            } else {
+                assert failure == null : "${item.name}: ${failure}"
+                assert resolved == new File(item.expected).canonicalFile : "${item.name}: selected ${resolved}"
+            }
+            println "PASS: ${item.name}"
+            results.add([name: item.name, passed: true, root: resolved?.path])
+        }
+        file('results.json').text = JsonOutput.prettyPrint(JsonOutput.toJson(results))
+    }
+}
+'''.replace('RESOLVER_URI', json.dumps((plugin / 'android/native_sources.gradle').as_uri())), encoding='utf-8')
+    command = [str(gradle), '--no-daemon', '--offline', '--console=plain', '-p', str(project), 'verifySources']
+    result = subprocess.run(command, capture_output=True, text=True, errors='replace', timeout=180)
+    output = result.stdout + result.stderr
+    (report / 'gradle-resolver.txt').write_text(output, encoding='utf-8')
+    print(output[-10000:], flush=True)
+    if result.returncode:
+        raise RuntimeError('Production Gradle resolver regression checks failed.')
+    results = json.loads((project / 'results.json').read_text(encoding='utf-8'))
+    (report / 'gradle-resolver.json').write_text(json.dumps(results, indent=2), encoding='utf-8')
+    resolved = Path(results[0]['root'])
+    (report / 'selected-native-root.txt').write_text(resolved.as_posix(), encoding='utf-8')
+    return resolved
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--gradle', required=True, type=Path)
+    parser.add_argument('--plugin', type=Path, default=Path(__file__).resolve().parents[1] / 'zstandard_android')
+    parser.add_argument('--native', type=Path, default=Path(__file__).resolve().parents[1] / 'zstandard_native')
+    parser.add_argument('--report-directory', required=True, type=Path)
+    arguments = parser.parse_args()
+    arguments.report_directory.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(prefix='zstandard resolver é ',
+                                     dir=os.environ.get('RUNNER_TEMP')) as temporary:
+        verify(arguments.gradle, arguments.plugin.resolve(), arguments.native.resolve(),
+               Path(temporary).resolve(), arguments.report_directory)