diff --git a/.github/workflows/android_windows_sources.yml b/.github/workflows/android_windows_sources.yml new file mode 100644 index 00000000..483c011e --- /dev/null +++ b/.github/workflows/android_windows_sources.yml @@ -0,0 +1,122 @@ +name: Android native sources on Windows + +on: + pull_request: + paths: + - zstandard_android/** + - zstandard_native/** + - scripts/check_android_windows_sources.py + - scripts/check_android_native_sources.py + - scripts/download_android_windows_tools.py + - .github/workflows/android_windows_sources.yml + push: + branches: [develop] + paths: + - zstandard_android/** + - zstandard_native/** + - scripts/check_android_windows_sources.py + - scripts/check_android_native_sources.py + - scripts/download_android_windows_tools.py + - .github/workflows/android_windows_sources.yml + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + reproduce-source-resolution: + name: Android Pub-cache regression on Windows + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: + group: vypdev + labels: [self-hosted, Windows, X64] + timeout-minutes: 30 + env: + PYTHONDONTWRITEBYTECODE: '1' + PYTHONUTF8: '1' + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Prepare verified job-local Python without a Windows installer + shell: powershell + env: + REPRODUCTION_REPORT: ${{ runner.temp }}/zstandard-android-windows-${{ github.run_id }}-${{ github.run_attempt }} + run: | + $ErrorActionPreference = 'Stop' + $ProgressPreference = 'SilentlyContinue' + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + New-Item -ItemType Directory -Path $env:REPRODUCTION_REPORT -Force | Out-Null + $utf8 = New-Object System.Text.UTF8Encoding($false) + $version = '3.13.17' + $expected = '9e35319bffedab8b36ad44d8f377cb4f1733bcec257a0b6d04d94f01486ddb38' + $url = "https://www.python.org/ftp/python/$version/python-$version-embed-amd64.zip" + [IO.File]::WriteAllText((Join-Path $env:REPRODUCTION_REPORT 'python-bootstrap.txt'), "$url`nSHA256 $expected`n", $utf8) + $directory = Join-Path $env:RUNNER_TEMP "zstandard-python-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT" + New-Item -ItemType Directory -Path $directory | Out-Null + $archive = Join-Path $directory 'python.zip' + Invoke-WebRequest -UseBasicParsing -Uri $url -OutFile $archive + if ((Get-FileHash -Algorithm SHA256 $archive).Hash.ToLowerInvariant() -ne $expected) { + throw 'The portable Python archive does not match the official release checksum.' + } + $runtime = Join-Path $directory 'runtime' + Expand-Archive -LiteralPath $archive -DestinationPath $runtime + & (Join-Path $runtime 'python.exe') --version + if ($LASTEXITCODE -ne 0) { throw 'The job-local Python runtime could not execute.' } + [IO.File]::AppendAllText($env:GITHUB_PATH, "$runtime`n", $utf8) + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: '17' + - name: Isolate Gradle state + shell: powershell + run: | + $directory = Join-Path $env:RUNNER_TEMP "zstandard-gradle-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT" + New-Item -ItemType Directory -Path $directory | Out-Null + $utf8 = New-Object System.Text.UTF8Encoding($false) + [IO.File]::WriteAllText((Join-Path $directory 'gradle.properties'), "org.gradle.daemon=false`norg.gradle.workers.max=2`n", $utf8) + [IO.File]::AppendAllText($env:GITHUB_ENV, "GRADLE_USER_HOME=$directory`n", $utf8) + - uses: gradle/actions/setup-gradle@v4 + with: + gradle-version: '8.14.2' + cache-disabled: true + - name: Prepare a private Windows Android SDK + shell: powershell + env: + REPRODUCTION_REPORT: ${{ runner.temp }}/zstandard-android-windows-${{ github.run_id }}-${{ github.run_attempt }} + run: | + $ErrorActionPreference = 'Stop' + $utf8 = New-Object System.Text.UTF8Encoding($false) + $sdk = Join-Path $env:RUNNER_TEMP ("zstandard-android-sdk-{0}-{1}" -f $env:GITHUB_RUN_ID, $env:GITHUB_RUN_ATTEMPT) + python scripts/download_android_windows_tools.py --sdk-root "$sdk" --report-directory "$env:REPRODUCTION_REPORT" + if ($LASTEXITCODE -ne 0) { throw 'The reviewed command-line tools could not be prepared.' } + $env:ANDROID_HOME = $sdk + $env:ANDROID_SDK_ROOT = $sdk + $env:ANDROID_USER_HOME = Join-Path $sdk 'user-state' + New-Item -ItemType Directory -Path $env:ANDROID_USER_HOME | Out-Null + python scripts/download_android_windows_tools.py --sdk-root "$sdk" --report-directory "$env:REPRODUCTION_REPORT" --install-packages + if ($LASTEXITCODE -ne 0) { throw 'The private Windows Android SDK could not be provisioned.' } + Write-Host ("[{0:o}] Private Android SDK provisioning finished" -f (Get-Date)) + foreach ($name in @('ANDROID_HOME', 'ANDROID_SDK_ROOT', 'ANDROID_USER_HOME')) { + $value = [Environment]::GetEnvironmentVariable($name) + [IO.File]::AppendAllText($env:GITHUB_ENV, "$name=$value`n", $utf8) + } + - name: Reproduce the published failure and validate the candidate resolver + shell: powershell + env: + REPRODUCTION_REPORT: ${{ runner.temp }}/zstandard-android-windows-${{ github.run_id }}-${{ github.run_attempt }} + run: | + $ErrorActionPreference = 'Stop' + python scripts/check_android_windows_sources.py --baseline-version 1.5.3 --gradle (Get-Command gradle.bat).Source --report-directory "$env:REPRODUCTION_REPORT" + if ($LASTEXITCODE -ne 0) { throw 'Android source resolver regression failed.' } + - name: Retain Windows source resolution and toolchain evidence + if: always() + uses: actions/upload-artifact@v5 + with: + name: android-windows-source-reproduction-${{ github.run_attempt }} + path: ${{ runner.temp }}/zstandard-android-windows-${{ github.run_id }}-${{ github.run_attempt }} + if-no-files-found: error + retention-days: 14 diff --git a/docs/development/building.md b/docs/development/building.md index 3f4aca0d..4a170385 100644 --- a/docs/development/building.md +++ b/docs/development/building.md @@ -58,10 +58,11 @@ If you are developing or modifying a platform package’s native code: ### Android -- The plugin builds the native library via `zstandard_android/zstd_build/CMakeLists.txt`, which compiles sources from `zstandard_native/src/zstd/` (resolved from the repo or pub cache). +- The plugin builds the native library via `zstandard_android/zstd_build/CMakeLists.txt`. Gradle reads the consuming app or workspace's `.dart_tool/package_config.json`, resolves the exact `zstandard_native` file URI and passes its `src/zstd` path to CMake. The plugin and native package can both live in a versioned Pub cache outside the consuming app, including Windows paths with spaces. - The Android plugin uses Built-in Kotlin with AGP 9 and newer. The Android examples exercise this path with Flutter 3.47+. - Building the Android app (e.g. `flutter build apk` or running from Android Studio) triggers the native build via Gradle/CMake. - Ensure the NDK is installed and that `zstandard_native` is available (e.g. `flutter pub get` so the canonical `zstandard_native/src/zstd/` is resolved). +- For an explicit source override, set the Gradle property `ZSTD_ROOT` (or its environment variable) to the directory containing `zstd.h`; direct CMake callers use `-DZSTD_ROOT=...`. Overrides take precedence. With no consumer package configuration, a direct repository build can use the sibling native package. A missing or invalid configured dependency fails rather than selecting a different cached package. ### iOS / macOS diff --git a/docs/development/ci-cd.md b/docs/development/ci-cd.md index be1b76da..1e335119 100644 --- a/docs/development/ci-cd.md +++ b/docs/development/ci-cd.md @@ -26,6 +26,12 @@ They provide Android emulator/instrumentation, Linux desktop, ChromeDriver, Windows desktop/native, and both SwiftPM and CocoaPods Apple integration. They also run publish dry-runs and coverage where applicable. +`android_windows_sources.yml` separately verifies Android source discovery on +a Windows host with both packages staged in a versioned cache outside the +consumer and builds ARM64 native code. Android execution coverage remains in +the Linux emulator matrix. See the [Windows Android runner +contract](self-hosted-ci.md#android-builds-hosted-on-windows). + Actions in workflows and composite actions must use version tags such as `@v1`, `@v2` or `@v3`. Commit SHA references are not permitted. The package preparation contract checks this policy when workflow or action files change. diff --git a/docs/development/self-hosted-ci.md b/docs/development/self-hosted-ci.md index fe12ecd0..aa86b0d1 100644 --- a/docs/development/self-hosted-ci.md +++ b/docs/development/self-hosted-ci.md @@ -94,6 +94,34 @@ Windows setup resolves CMake and CTest from the job's current `PATH`, a standalo The workflow builds `zstandard_windows/example` in Debug mode, runs the generated native test target (including a zstd compression/decompression round trip), runs the Windows package tests with the built DLL on `PATH`, and launches the example for Flutter integration tests. The same sequence is repeated with the current `zstandard_native` package staged outside the checkout and selected through `package_config.json`. A separate federated-example job builds `zstandard/example` and runs its integration suite. +### Android builds hosted on Windows + +`android_windows_sources.yml` adds one targeted job for relevant pull requests +and pushes to `develop`. It reproduces the original published Android source +lookup failure, then stages both candidate packages outside the checkout and +outside a separate consuming app. Production Gradle resolver checks cover two +cached native versions, absolute and relative file URIs, Windows drive paths, +spaces, Unicode, workspace ancestors, overrides and invalid dependencies. +NDK 27.0.12077973 with CMake 3.22.1 builds the corrected ARM64/API 21 library. + +Linux retains the complete Flutter integration and JNI instrumentation matrix. +The Windows job covers host-specific source discovery and compilation. It does +not claim Android runtime coverage. A Windows emulator requires separate +infrastructure validation: the current runner reported unavailable hardware +virtualization, and API 30 ATD software-emulator trials did not reach a stable +boot. Enabling Windows runtime checks requires a reproducible successful boot +and test execution first. + +Portable Python and Android command-line tools use reviewed archive checksums. +The job requires HTTPS access to Python, Google SDK repositories, Gradle +distributions and pub.dev archives. +Java 17 and Gradle 8.14.2 are selected per job. The Android SDK, Gradle state and +test fixtures live in private job directories, including on cancellation. Only +the NDK and CMake packages are provisioned; no system image is downloaded for +the compilation check. The job does not change machine-wide SDK, virtualization +or PATH settings. Missing tools, failed resolution and failed compilation are +explicit errors; diagnostic artifacts are retained for 14 days. + ## Web job requirements Flutter 3.47.2 does not support running `package:integration_test` directly with `flutter test -d chrome`. The workflow therefore uses the supported WebDriver path: diff --git a/scripts/check_android_native_sources.py b/scripts/check_android_native_sources.py new file mode 100644 index 00000000..84760f96 --- /dev/null +++ b/scripts/check_android_native_sources.py @@ -0,0 +1,156 @@ +"""Exercise the production Gradle resolver with real, host-native file URIs.""" + +import argparse +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +from urllib.parse import quote + + +def verify(gradle, plugin, native, stage, report): + cases = [] + cache = stage / 'Pub cache/hosted/pub.dev' + selected_package = cache / 'zstandard_native-9.8.7 é' + # Both versions coexist in the same fixture cache. The source bytes come + # from the candidate; these fixture version names are never published. + shutil.copytree(native / 'src', selected_package / 'src') + native = selected_package + + def sources(directory): + root = directory / 'src/zstd' + for part in ['common', 'compress', 'decompress']: + (root / part).mkdir(parents=True, exist_ok=True) + (root / 'zstd.h').touch() + return root + + def configuration(consumer, entries): + path = consumer / '.dart_tool/package_config.json' + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({'configVersion': 2, 'packages': entries}), encoding='utf-8') + return path + + def entry(uri): + return {'name': 'zstandard_native', 'rootUri': uri, 'packageUri': 'lib/'} + + def case(name, consumer, expected=None, error=None, explicit=None, plugin_directory=plugin): + consumer.mkdir(parents=True, exist_ok=True) + cases.append({'name': name, 'consumer': str(consumer), 'plugin': str(plugin_directory), + 'expected': str(expected) if expected else None, + 'error': error, 'explicit': str(explicit) if explicit is not None else None}) + + selected = native / 'src/zstd' + decoy = sources(cache / 'zstandard_native-9.8.8') + repository = stage / 'repository' + sibling = sources(repository / 'zstandard_native') + repository_plugin = repository / 'zstandard_android/android' + repository_plugin.mkdir(parents=True) + + absolute = stage / 'separate consumer é/absolute' + configuration(absolute, [entry(native.as_uri() + '/')]) + case('absolute file URI, exact selected version, spaces and Unicode', absolute / 'android', selected) + + relative = stage / 'separate consumer é/relative' + config = relative / '.dart_tool/package_config.json' + uri = quote(os.path.relpath(native, config.parent).replace(os.sep, '/'), safe='/') + '/' + configuration(relative, [entry(uri)]) + case('relative URI is relative to package_config, not Gradle or plugin', relative / 'android', selected) + + workspace = stage / 'workspace with spaces' + configuration(workspace, [entry(native.as_uri() + '/')]) + case('workspace ancestor configuration', workspace / 'apps/mobile/android', selected) + + case('explicit override precedes configured dependency', absolute / 'android', decoy, explicit=decoy) + explicit_relative = os.path.relpath(decoy, absolute / 'android') + case('relative explicit override uses consumer directory', absolute / 'android', decoy, + explicit=explicit_relative) + case('invalid explicit override fails without fallback', absolute / 'android', + error='explicit ZSTD_ROOT', explicit=stage / 'missing') + case('direct repository fallback without package configuration', repository / 'consumer/android', + sibling, plugin_directory=repository_plugin) + + bad = [ + ('missing native dependency', [], 'Expected exactly one'), + ('duplicate native dependencies', [entry(native.as_uri()), entry(decoy.parent.parent.as_uri())], 'Expected exactly one'), + ('empty URI', [entry('')], 'Expected exactly one'), + ('non-string URI', [entry(42)], 'Expected exactly one'), + ('non-file URI', [entry('https://example.invalid/native')], 'Cannot resolve'), + ('malformed URI', [entry('file:///%invalid')], 'Cannot resolve'), + ('missing selected sources', [entry((stage / 'missing package').as_uri())], 'zstd source not found'), + ] + for index, (name, entries, message) in enumerate(bad): + consumer = stage / f'invalid cases/{index}' + configuration(consumer, entries) + case(name + ' cannot select a cached or repository fallback', consumer / 'android', + error=message, plugin_directory=repository_plugin) + + malformed = stage / 'malformed JSON' + config = configuration(malformed, []) + config.write_text('{bad JSON', encoding='utf-8') + case('malformed JSON reports pub get guidance', malformed / 'android', error='Cannot read') + invalid_shape = stage / 'invalid JSON shape' + config = configuration(invalid_shape, []) + config.write_text('{"packages": {}}', encoding='utf-8') + case('invalid package configuration shape', invalid_shape / 'android', error='Invalid package configuration') + + project = stage / 'resolver verification' + project.mkdir() + (project / 'settings.gradle').write_text("rootProject.name = 'native-source-verification'\n", encoding='utf-8') + (project / 'cases.json').write_text(json.dumps(cases), encoding='utf-8') + (project / 'build.gradle').write_text('''import groovy.json.JsonSlurper +import groovy.json.JsonOutput +apply from: new File(new URI(RESOLVER_URI)) +tasks.register('verifySources') { + doLast { + def cases = new JsonSlurper().parse(file('cases.json'), 'UTF-8') + def results = [] + cases.each { item -> + File resolved = null + Exception failure = null + try { + resolved = resolveZstandardNativeRoot(new File(item.consumer), new File(item.plugin), item.explicit) + } catch (Exception error) { + failure = error + } + if (item.error) { + assert failure != null : "${item.name}: unexpectedly succeeded" + assert failure.message.contains(item.error) : "${item.name}: ${failure.message}" + } else { + assert failure == null : "${item.name}: ${failure}" + assert resolved == new File(item.expected).canonicalFile : "${item.name}: selected ${resolved}" + } + println "PASS: ${item.name}" + results.add([name: item.name, passed: true, root: resolved?.path]) + } + file('results.json').text = JsonOutput.prettyPrint(JsonOutput.toJson(results)) + } +} +'''.replace('RESOLVER_URI', json.dumps((plugin / 'android/native_sources.gradle').as_uri())), encoding='utf-8') + command = [str(gradle), '--no-daemon', '--offline', '--console=plain', '-p', str(project), 'verifySources'] + result = subprocess.run(command, capture_output=True, text=True, errors='replace', timeout=180) + output = result.stdout + result.stderr + (report / 'gradle-resolver.txt').write_text(output, encoding='utf-8') + print(output[-10000:], flush=True) + if result.returncode: + raise RuntimeError('Production Gradle resolver regression checks failed.') + results = json.loads((project / 'results.json').read_text(encoding='utf-8')) + (report / 'gradle-resolver.json').write_text(json.dumps(results, indent=2), encoding='utf-8') + resolved = Path(results[0]['root']) + (report / 'selected-native-root.txt').write_text(resolved.as_posix(), encoding='utf-8') + return resolved + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--gradle', required=True, type=Path) + parser.add_argument('--plugin', type=Path, default=Path(__file__).resolve().parents[1] / 'zstandard_android') + parser.add_argument('--native', type=Path, default=Path(__file__).resolve().parents[1] / 'zstandard_native') + parser.add_argument('--report-directory', required=True, type=Path) + arguments = parser.parse_args() + arguments.report_directory.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix='zstandard resolver é ', + dir=os.environ.get('RUNNER_TEMP')) as temporary: + verify(arguments.gradle, arguments.plugin.resolve(), arguments.native.resolve(), + Path(temporary).resolve(), arguments.report_directory) diff --git a/scripts/check_android_windows_sources.py b/scripts/check_android_windows_sources.py new file mode 100644 index 00000000..8e42b27f --- /dev/null +++ b/scripts/check_android_windows_sources.py @@ -0,0 +1,200 @@ +"""Reproduce Android source discovery with a real Windows-hosted NDK build.""" + +import argparse +import hashlib +import io +import json +import os +from pathlib import Path +import platform +import shutil +import struct +import subprocess +import sys +import tarfile +import tempfile +import urllib.request + +# The reviewed embedded Windows runtime deliberately omits the script directory. +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from check_android_native_sources import verify + + +def run(command, report, *, expected_success=True, timeout=300): + result = subprocess.run([str(part) for part in command], capture_output=True, + text=True, errors='replace', timeout=timeout) + output = result.stdout + result.stderr + report.write_text(output, encoding='utf-8') + print(output[-8000:], flush=True) + if expected_success and result.returncode: + raise RuntimeError(f'{command[0]} failed; see {report.name}') + return result.returncode, output + + +def sdk_tools(report): + roots = [Path(value) for name in ['ANDROID_SDK_ROOT', 'ANDROID_HOME'] + if (value := os.environ.get(name))] + if os.environ.get('LOCALAPPDATA'): + roots.append(Path(os.environ['LOCALAPPDATA']) / 'Android/Sdk') + roots += [Path.home() / 'Library/Android/sdk', Path.home() / 'Android/Sdk'] + sdk = next((root for root in roots if any((root / child).is_dir() + for child in ['cmdline-tools', 'platform-tools', 'ndk'])), None) + information = {'host': platform.system(), 'architecture': platform.machine(), + 'sdk': str(sdk) if sdk else None, + 'free_temporary_bytes': shutil.disk_usage(tempfile.gettempdir()).free} + executable = '.exe' if os.name == 'nt' else '' + ndks = sorted((sdk / 'ndk').iterdir(), reverse=True) if sdk and (sdk / 'ndk').is_dir() else [] + preferred = next((p for p in ndks if p.name == '27.0.12077973'), None) + ndk = preferred or next((p for p in ndks if + (p / 'build/cmake/android.toolchain.cmake').is_file()), None) + information['ndk'] = str(ndk) if ndk else None + candidates = [] + if sdk: + candidates.append(sdk / f'cmake/3.22.1/bin/cmake{executable}') + if (sdk / 'cmake').is_dir(): + candidates += [p / f'bin/cmake{executable}' + for p in sorted((sdk / 'cmake').iterdir(), reverse=True)] + if shutil.which('cmake'): + candidates.append(Path(shutil.which('cmake'))) + if os.name == 'nt': + for variable in ['ProgramFiles', 'ProgramFiles(x86)']: + if os.environ.get(variable): + candidates.append(Path(os.environ[variable]) / 'CMake/bin/cmake.exe') + if os.environ.get('ProgramFiles(x86)'): + vswhere = Path(os.environ['ProgramFiles(x86)']) / 'Microsoft Visual Studio/Installer/vswhere.exe' + if vswhere.is_file(): + installations = json.loads(subprocess.check_output( + [str(vswhere), '-products', '*', '-format', 'json'], text=True)) + candidates += [Path(item['installationPath']) / + 'Common7/IDE/CommonExtensions/Microsoft/CMake/CMake/bin/cmake.exe' + for item in installations] + cmake = next((p for p in candidates if p.is_file()), None) + information['cmake'] = str(cmake) if cmake else None + ninja_candidates = [cmake.parent / ('ninja' + executable)] if cmake else [] + if cmake and os.name == 'nt': + ninja_candidates.append(cmake.parent.parent.parent / 'Ninja/ninja.exe') + if shutil.which('ninja'): + ninja_candidates.append(Path(shutil.which('ninja'))) + ninja = next((p for p in ninja_candidates if p.is_file()), None) + information['ninja'] = str(ninja) if ninja else None + emulator = sdk / ('emulator/emulator' + executable) if sdk else None + information['emulator_available'] = bool(emulator and emulator.is_file()) + if information['emulator_available']: + result, _ = run([emulator, '-accel-check'], report / 'emulator-acceleration.txt', + expected_success=False, timeout=30) + information['emulator_acceleration_check_exit'] = result + (report / 'toolchain.json').write_text(json.dumps(information, indent=2), encoding='utf-8') + print(json.dumps(information, indent=2), flush=True) + if not all([sdk, ndk, cmake, ninja]): + raise RuntimeError('An Android SDK, NDK, CMake and Ninja are required; see toolchain.json.') + run([cmake, '--version'], report / 'cmake-version.txt') + return ndk, cmake, ninja + + +def published_package(package, version, destination): + def get(url): + with urllib.request.urlopen(url, timeout=60) as response: + return response.read() + + metadata = json.loads(get(f'https://pub.dev/api/packages/{package}/versions/{version}')) + assert metadata['version'] == version and metadata['pubspec']['name'] == package + archive = get(metadata['archive_url']) + digest = hashlib.sha256(archive).hexdigest() + assert digest == metadata['archive_sha256'] + destination.mkdir(parents=True) + with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: + for member in tar: + relative = Path(member.name) + target = destination / relative + if relative.is_absolute() or '..' in relative.parts or not target.resolve().is_relative_to(destination.resolve()): + raise ValueError('Unsafe package archive member') + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + elif member.isfile(): + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(tar.extractfile(member).read()) + else: + raise ValueError('Expected only regular package files and directories') + return {'package': package, 'version': version, 'archive_sha256': digest} + + +def reproduce(version, report, gradle=None): + ndk, cmake, ninja = sdk_tools(report) + with tempfile.TemporaryDirectory(prefix='zstandard android windows ', + dir=os.environ.get('RUNNER_TEMP')) as temporary: + stage = Path(temporary).resolve() + cache = stage / 'Pub cache/hosted/pub.dev' + plugin = cache / f'zstandard_android-{version}' + native = cache / f'zstandard_native-{version}' + packages = [published_package('zstandard_android', version, plugin), + published_package('zstandard_native', version, native)] + consumer = stage / 'separate consumer' + configuration = consumer / '.dart_tool/package_config.json' + configuration.parent.mkdir(parents=True) + configuration.write_text(json.dumps({'configVersion': 2, 'packages': [ + {'name': 'zstandard_android', 'rootUri': plugin.as_uri() + '/', 'packageUri': 'lib/'}, + {'name': 'zstandard_native', 'rootUri': native.as_uri() + '/', 'packageUri': 'lib/'} + ]}, indent=2), encoding='utf-8') + common = [cmake, '-G', 'Ninja', '-S', plugin / 'zstd_build', + f'-DCMAKE_MAKE_PROGRAM={ninja}', + f'-DCMAKE_TOOLCHAIN_FILE={ndk / "build/cmake/android.toolchain.cmake"}', + '-DANDROID_ABI=arm64-v8a', '-DANDROID_PLATFORM=android-21', + '-DCMAKE_BUILD_TYPE=RelWithDebInfo'] + result, output = run(common + ['-B', stage / 'baseline'], + report / 'baseline-configure.txt', expected_success=False) + if result == 0 or 'zstd source not found' not in output: + raise RuntimeError('The expected source-discovery failure was not reproduced.') + control = stage / 'explicit-root-control' + run(common + ['-B', control, f'-DZSTD_ROOT={(native / "src/zstd").as_posix()}'], + report / 'control-configure.txt') + run([cmake, '--build', control, '--parallel', '2'], report / 'control-build.txt') + library = control / 'libzstandard_android.so' + data = library.read_bytes() + assert data[:4] == b'\x7fELF' and struct.unpack_from(' 300: + raise TimeoutError('Android tools download exceeded five minutes.') + digest.update(block) + output.write(block) + downloaded += len(block) + if downloaded % (8 * 1024 * 1024) == 0: + receipt['downloaded_bytes'] = downloaded + note(f'Downloaded {downloaded // (1024 * 1024)} MiB in {time.monotonic() - started:.1f}s.') + receipt.update({'downloaded_bytes': downloaded, 'sha256': digest.hexdigest()}) + if digest.hexdigest() != SHA256: + raise ValueError('Android tools checksum mismatch.') + receipt['stage'] = 'extract' + note(f'Checksum verified after {time.monotonic() - started:.1f}s; extracting.') + extracted = sdk / 'archive-root' + with zipfile.ZipFile(archive) as package: + for index, member in enumerate(package.infolist(), 1): + destination = (extracted / member.filename).resolve() + if not destination.is_relative_to(extracted.resolve()): + raise ValueError('Unsafe Android tools archive member.') + package.extract(member, extracted) + if index % 32 == 0: + note(f'Extracted {index} files in {time.monotonic() - started:.1f}s.') + tools = sdk / 'cmdline-tools/latest' + tools.parent.mkdir() + shutil.move(str(extracted / 'cmdline-tools'), str(tools)) + if not (tools / 'bin/sdkmanager.bat').is_file(): + raise ValueError('The reviewed archive did not contain sdkmanager.bat.') + extracted.rmdir() + archive.unlink() + receipt.update({'stage': 'ready', 'duration_seconds': time.monotonic() - started}) + note('Reviewed Android command-line tools are ready.') + except Exception as error: + receipt.update({'stage': 'failed', 'error': str(error), 'duration_seconds': time.monotonic() - started}) + note('Android command-line tools preparation failed: ' + str(error)) + raise + + +def install_packages(sdk, report): + sdk = sdk.resolve() + if sdk.parent != Path(os.environ['RUNNER_TEMP']).resolve(): + raise ValueError('Package installation must remain inside the job-private SDK.') + tools = sdk / 'cmdline-tools/latest' + java = Path(os.environ['JAVA_HOME']) / 'bin/java.exe' + classpath = tools / 'lib/sdkmanager-classpath.jar' + if not java.is_file() or not classpath.is_file(): + raise ValueError('The selected JDK and reviewed SDK manager are required.') + # This is the Java invocation in the reviewed vendor sdkmanager.bat. Calling + # it directly gives Java explicit UTF-8 stdin without a PowerShell/batch pipe. + command = [str(java), '-Dfile.encoding=UTF-8', + f'-Dcom.android.sdkmanager.toolsdir={tools}', '-classpath', str(classpath), + 'com.android.sdklib.tool.sdkmanager.SdkManagerCli', f'--sdk_root={sdk}'] + requested = ['ndk;27.0.12077973', 'cmake;3.22.1'] + for phase, arguments, timeout in [('licenses', ['--licenses'], 180), + ('install', ['--install', *requested], 900)]: + print(f'Running SDK manager {phase} with explicit stdin (timeout {timeout}s).', flush=True) + receipt = {'stage': phase, 'requested_packages': requested, 'explicit_stdin': True} + (report / 'android-packages.json').write_text(json.dumps(receipt, indent=2), encoding='utf-8') + result = subprocess.run(command + arguments, input='y\n' * 256, + capture_output=True, text=True, encoding='utf-8', + errors='replace', timeout=timeout) + output = result.stdout + result.stderr + (report / f'android-{phase}.txt').write_text(output, encoding='utf-8') + print(output[-1800:], flush=True) + if result.returncode: + raise RuntimeError(f'Android SDK manager {phase} failed with exit {result.returncode}.') + required = ['ndk/27.0.12077973/build/cmake/android.toolchain.cmake', + 'cmake/3.22.1/bin/cmake.exe', 'cmake/3.22.1/bin/ninja.exe'] + missing = [relative for relative in required if not (sdk / relative).is_file()] + if missing: + raise RuntimeError('SDK manager did not install required files: ' + ', '.join(missing)) + receipt.update({'stage': 'ready', 'verified_files': required}) + (report / 'android-packages.json').write_text(json.dumps(receipt, indent=2), encoding='utf-8') + print('Private Android SDK packages and executables verified.', flush=True) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--sdk-root', type=Path, required=True) + parser.add_argument('--report-directory', type=Path, required=True) + parser.add_argument('--install-packages', action='store_true') + arguments = parser.parse_args() + if arguments.install_packages: + install_packages(arguments.sdk_root, arguments.report_directory) + else: + prepare(arguments.sdk_root, arguments.report_directory) diff --git a/zstandard_android/README.md b/zstandard_android/README.md index 47dab528..0bf6f2b5 100644 --- a/zstandard_android/README.md +++ b/zstandard_android/README.md @@ -52,6 +52,20 @@ send Dart-owned bytes to a worker isolate; native pointers never cross an isolate boundary. Decompression uses zstd's streaming API and enforces the configured output limit while producing bytes. +Gradle locates `zstandard_native` through the consuming app's +`.dart_tool/package_config.json`, including an enclosing Dart workspace. It +resolves the selected dependency's `rootUri` with the host's file-URI parser +and passes its `src/zstd` directory explicitly to CMake. This supports a +versioned Pub cache outside the app, Windows drive paths, spaces and Unicode, +without choosing another cached version. Run `flutter pub get` before building. + +An explicit Gradle property `-PZSTD_ROOT=...` or `ZSTD_ROOT` environment variable +can select a source directory containing `zstd.h`, `common`, `compress` and +`decompress`. Relative overrides use the consuming Android project directory. +Direct CMake builds use `-DZSTD_ROOT=...`. Repository-local fallback is available +only when no consumer package configuration or explicit override is present; +an invalid selected dependency fails with a diagnostic. + The plugin does not pin an Android Gradle Plugin version. The consuming app owns that choice. The repository contains two Android consumers for CI: diff --git a/zstandard_android/android/build.gradle b/zstandard_android/android/build.gradle index 240b1aba..d970b694 100644 --- a/zstandard_android/android/build.gradle +++ b/zstandard_android/android/build.gradle @@ -12,6 +12,11 @@ rootProject.allprojects { apply plugin: "com.android.library" +apply from: new File(project.projectDir, 'native_sources.gradle') +def explicitNativeRoot = project.findProperty('ZSTD_ROOT') ?: System.getenv('ZSTD_ROOT') +def nativeRoot = resolveZstandardNativeRoot(rootProject.projectDir, project.projectDir, explicitNativeRoot?.toString()) +def cmakeNativeRoot = nativeRoot.path.replace(File.separatorChar, '/' as char) + android { if (project.android.hasProperty("namespace")) { namespace = "dev.vyp.zstandard_android" @@ -56,6 +61,7 @@ android { cmake { cppFlags "-fmax-page-size=16384" arguments "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON" + arguments "-DZSTD_ROOT=${cmakeNativeRoot}" } } } diff --git a/zstandard_android/android/native_sources.gradle b/zstandard_android/android/native_sources.gradle new file mode 100644 index 00000000..89d87016 --- /dev/null +++ b/zstandard_android/android/native_sources.gradle @@ -0,0 +1,57 @@ +import groovy.json.JsonSlurper +import java.nio.file.Paths + +// Resolve from the consumer, not from this plugin's possibly hosted location. +ext.resolveZstandardNativeRoot = { File consumerDirectory, File pluginDirectory, String explicitRoot = null -> + def validate = { File candidate, String origin -> + File root = candidate.canonicalFile + if (!new File(root, 'zstd.h').isFile() || + !['common', 'compress', 'decompress'].every { new File(root, it).isDirectory() }) { + throw new GradleException("zstd source not found at ${root} (${origin}). Run flutter pub get in the consuming application or workspace.") + } + return root + } + + if (explicitRoot) { + File supplied = new File(explicitRoot) + if (!supplied.isAbsolute()) { + supplied = new File(consumerDirectory, explicitRoot) + } + return validate(supplied, 'explicit ZSTD_ROOT') + } + + for (File directory = consumerDirectory.canonicalFile; directory != null; directory = directory.parentFile) { + File configuration = new File(directory, '.dart_tool/package_config.json') + if (!configuration.isFile()) { + continue + } + def data + try { + data = new JsonSlurper().parse(configuration, 'UTF-8') + } catch (Exception error) { + throw new GradleException("Cannot read ${configuration}. Run flutter pub get in the consuming application or workspace.", error) + } + if (!(data instanceof Map) || !(data.packages instanceof List)) { + throw new GradleException("Invalid package configuration: ${configuration}.") + } + def selected = data.packages.findAll { it instanceof Map && it.name == 'zstandard_native' } + if (selected.size() != 1 || !(selected[0].rootUri instanceof String) || !selected[0].rootUri) { + throw new GradleException("Expected exactly one zstandard_native dependency in ${configuration}. Run flutter pub get.") + } + try { + URI uri = configuration.toURI().resolve(new URI(selected[0].rootUri)) + if (uri.scheme != 'file') { + throw new IllegalArgumentException('Expected a local file URI.') + } + File nativePackage = Paths.get(uri).toFile() + return validate(new File(nativePackage, 'src/zstd'), "selected by ${configuration}") + } catch (GradleException error) { + throw error + } catch (Exception error) { + throw new GradleException("Cannot resolve the zstandard_native file URI in ${configuration}.", error) + } + } + + // Direct repository builds can run before a consumer's pub configuration exists. + return validate(new File(pluginDirectory, '../../zstandard_native/src/zstd'), 'repository fallback') +} diff --git a/zstandard_android/zstd_build/CMakeLists.txt b/zstandard_android/zstd_build/CMakeLists.txt index 37505c19..b8d44dbd 100644 --- a/zstandard_android/zstd_build/CMakeLists.txt +++ b/zstandard_android/zstd_build/CMakeLists.txt @@ -5,47 +5,23 @@ cmake_minimum_required(VERSION 3.10) project(zstandard_android_library VERSION 0.0.1 LANGUAGES C) -# Resolve the parent containing the plugin (repo root or pub-cache hosted dir) -# for consistent paths. -get_filename_component(_PLUGIN_PARENT "${CMAKE_CURRENT_SOURCE_DIR}/../.." ABSOLUTE) -# 1. Prefer repo local zstandard_native/src/zstd. -set(_LOCAL_ZSTD "${_PLUGIN_PARENT}/zstandard_native/src/zstd") -if(EXISTS "${_LOCAL_ZSTD}/common") - set(ZSTD_ROOT "${_LOCAL_ZSTD}") - message(STATUS "Using local zstandard_native: ${ZSTD_ROOT}") -else() - # 2. Resolve the exact dependency selected by pub. Never choose the first - # matching cache directory, which can silently compile a stale version. - set(_SEARCH_PKG "${CMAKE_CURRENT_SOURCE_DIR}") - foreach(_PKG_IGNORE RANGE 10) - get_filename_component(_PARENT "${_SEARCH_PKG}/.." ABSOLUTE) - if(NOT _PARENT OR _PARENT STREQUAL _SEARCH_PKG) - break() - endif() - set(_PKG_CONFIG "${_SEARCH_PKG}/../.dart_tool/package_config.json") - if(EXISTS "${_PKG_CONFIG}") - file(READ "${_PKG_CONFIG}" PACKAGE_CONFIG_CONTENT) - string(REGEX MATCH "\"name\": \"zstandard_native\"[^}]*\"rootUri\": \"([^\"]+)\"" _ "${PACKAGE_CONFIG_CONTENT}") - set(NATIVE_ROOT_URI "${CMAKE_MATCH_1}") - if(NATIVE_ROOT_URI) - string(REGEX REPLACE "^file://" "" NATIVE_ROOT "${NATIVE_ROOT_URI}") - if(WIN32) - string(REGEX REPLACE "^/" "" NATIVE_ROOT "${NATIVE_ROOT}") - endif() - string(REPLACE "%20" " " NATIVE_ROOT "${NATIVE_ROOT}") - set(ZSTD_CANDIDATE "${NATIVE_ROOT}/src/zstd") - if(EXISTS "${ZSTD_CANDIDATE}/common") - set(ZSTD_ROOT "${ZSTD_CANDIDATE}") - message(STATUS "Using zstandard_native from package_config: ${ZSTD_ROOT}") - break() - endif() - endif() - endif() - set(_SEARCH_PKG "${_PARENT}") - endforeach() - if(NOT ZSTD_ROOT OR NOT EXISTS "${ZSTD_ROOT}/common") - message(FATAL_ERROR "zstd source not found. Set ZSTD_ROOT or run flutter pub get so package_config.json resolves zstandard_native.") +# Gradle supplies the exact dependency selected by the consuming application. +# Direct CMake callers can provide the same override explicitly. +if(DEFINED ZSTD_ROOT AND NOT "${ZSTD_ROOT}" STREQUAL "") + if(CMAKE_HOST_WIN32) + file(TO_CMAKE_PATH "${ZSTD_ROOT}" ZSTD_ROOT) endif() + get_filename_component(ZSTD_ROOT "${ZSTD_ROOT}" ABSOLUTE) + message(STATUS "Using explicit zstandard_native source: ${ZSTD_ROOT}") +else() + get_filename_component(_PLUGIN_PARENT "${CMAKE_CURRENT_SOURCE_DIR}/../.." ABSOLUTE) + set(ZSTD_ROOT "${_PLUGIN_PARENT}/zstandard_native/src/zstd") + message(STATUS "Using repository zstandard_native source: ${ZSTD_ROOT}") +endif() + +if(NOT EXISTS "${ZSTD_ROOT}/zstd.h" OR NOT IS_DIRECTORY "${ZSTD_ROOT}/common" + OR NOT IS_DIRECTORY "${ZSTD_ROOT}/compress" OR NOT IS_DIRECTORY "${ZSTD_ROOT}/decompress") + message(FATAL_ERROR "zstd source not found at ${ZSTD_ROOT}. Set ZSTD_ROOT or run flutter pub get in the consuming application or workspace.") endif() add_definitions(-DZSTD_ENABLE_ASM_X86_64_BMI2=0)