From c99fe597674d6440767281e77a1d59b18228f514 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 25 Sep 2026 00:58:40 +0200 Subject: [PATCH 1/9] Add Linux ARM64 bundle support Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../workflows/codeql-compilation-caches.yml | 37 ++++++-- README.md | 11 ++- codeql_bundle/cache.py | 46 +++++---- codeql_bundle/cache_cli.py | 29 ++++-- codeql_bundle/cli.py | 7 +- codeql_bundle/helpers/bundle.py | 61 +++++++++--- .../supported-codeql-bundles.schema.json | 2 + pyproject.toml | 2 +- tests/test_bundle_platform.py | 23 +++++ tests/test_cache.py | 95 ++++++++++++++++++- tests/test_cache_cli.py | 1 + 11 files changed, 256 insertions(+), 58 deletions(-) create mode 100644 tests/test_bundle_platform.py diff --git a/.github/workflows/codeql-compilation-caches.yml b/.github/workflows/codeql-compilation-caches.yml index 101a950..539a240 100644 --- a/.github/workflows/codeql-compilation-caches.yml +++ b/.github/workflows/codeql-compilation-caches.yml @@ -45,6 +45,7 @@ jobs: matrix: ${{ steps.plan.outputs.matrix || '[{"language":"skip","target":"skip"}]' }} release: ${{ steps.version.outputs.release }} skip: ${{ steps.existing.outputs.skip }} + verify_matrix: ${{ steps.plan.outputs.verify_matrix || '[{"platform":"skip","os":"ubuntu-latest"}]' }} steps: - uses: actions/checkout@v4 @@ -115,7 +116,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.CODEQL_BUNDLE_CACHE_DIR }}/sources/${{ steps.version.outputs.release }} - key: codeql-source-${{ steps.version.outputs.release }}-${{ runner.os }} + key: codeql-source-${{ steps.version.outputs.release }}-${{ runner.os }}-${{ runner.arch }} - if: steps.existing.outputs.skip != 'true' id: plan @@ -130,6 +131,22 @@ jobs: --output release-plan.json echo "cache_release=$(jq -r '.cache_release' release-plan.json)" >> "$GITHUB_OUTPUT" echo "matrix=$(jq -c '.targets' release-plan.json)" >> "$GITHUB_OUTPUT" + verify_matrix="$(jq -c ' + [.source_assets[].platform | + select(. != "all") | + { + platform: ., + os: ( + if . == "linux64" then "ubuntu-latest" + elif . == "linux-arm64" then "ubuntu-24.04-arm" + elif . == "osx64" then "macos-latest" + elif . == "win64" then "windows-latest" + else error("Unsupported source platform: " + .) + end + ) + } + ]' release-plan.json)" + echo "verify_matrix=$verify_matrix" >> "$GITHUB_OUTPUT" - if: steps.existing.outputs.skip != 'true' uses: actions/upload-artifact@v4 @@ -162,7 +179,7 @@ jobs: - uses: actions/cache@v4 with: path: ${{ env.CODEQL_BUNDLE_CACHE_DIR }}/sources/${{ needs.plan.outputs.release }} - key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }} + key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }}-${{ runner.arch }} - name: Build and verify cache run: | @@ -189,10 +206,7 @@ jobs: strategy: fail-fast: false matrix: - os: - - ubuntu-latest - - macos-latest - - windows-latest + include: ${{ fromJSON(needs.plan.outputs.verify_matrix) }} runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -216,7 +230,7 @@ jobs: - uses: actions/cache@v4 with: path: ${{ env.CODEQL_BUNDLE_CACHE_DIR }}/sources/${{ needs.plan.outputs.release }} - key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }} + key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }}-${{ runner.arch }} - name: Verify all caches run: codeql-bundle-cache verify-all --plan release-plan.json --assets-dir dist @@ -299,13 +313,16 @@ jobs: - name: Create candidate catalog entry if: steps.recheck.outputs.skip != 'true' + shell: bash run: | + validated_platform_args=() + while IFS= read -r platform; do + validated_platform_args+=(--validated-platform "$platform") + done < <(jq -r '.source_assets[].platform | select(. != "all")' release-plan.json) codeql-bundle-cache catalog-entry \ --plan release-plan.json \ --assets-dir dist \ - --validated-platform linux64 \ - --validated-platform osx64 \ - --validated-platform win64 \ + "${validated_platform_args[@]}" \ --output catalog-entry.json codeql-bundle-cache verify-entry \ --entry catalog-entry.json \ diff --git a/README.md b/README.md index cdf2c8e..7f9489d 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ For more details on CodeQL customization packs see the section [CodeQL customiza The CodeQL bundle application can be installed using `pip` with the command: ```bash -python3.11 -m pip install https://github.com/advanced-security/codeql-bundle/releases/download/v0.5.0/codeql_bundle-0.5.0-py3-none-any.whl +python3.11 -m pip install https://github.com/advanced-security/codeql-bundle/releases/download/v0.6.0/codeql_bundle-0.6.0-py3-none-any.whl ``` ## Usage @@ -38,12 +38,19 @@ codeql-bundle --bundle codeql-bundle-v2.26.1 --output codeql-custom-bundle.tar.g ``` If the source bundle is the platform agnostic bundle then you can create platform specific bundles to reduce the size of the used bundle(s). -The following example creates platform specific bundles for all the currently supported platforms. +The following example creates bundles for the platforms included in that source archive. ```bash codeql-bundle --bundle --output --workspace --log INFO -p linux64 -p osx64 -p win64 ``` +Linux ARM64 binaries are not included in the platform-agnostic upstream bundle. +Create a Linux ARM64 custom bundle on a Linux ARM64 host: + +```bash +codeql-bundle --bundle codeql-bundle-v2.27.0 --output --workspace --log INFO -p linux-arm64 +``` + ### Compilation caches The repository maintains diff --git a/codeql_bundle/cache.py b/codeql_bundle/cache.py index 2aeeee7..65c3d6f 100644 --- a/codeql_bundle/cache.py +++ b/codeql_bundle/cache.py @@ -36,6 +36,8 @@ CACHE_FORMAT_VERSION = 1 DOWNLOAD_CHUNK_SIZE = 1024 * 1024 CATALOG_REFRESH_SECONDS = 60 * 60 +BUNDLE_PLATFORMS = ("linux64", "linux-arm64", "osx64", "win64") +SOURCE_PLATFORMS = ("all", *BUNDLE_PLATFORMS) RELEASE_PATTERN = re.compile(r"^codeql-bundle-v\d+\.\d+\.\d+$") CACHE_RELEASE_PATTERN = re.compile( r"^codeql-compilation-cache-v\d+\.\d+\.\d+(?:-[A-Za-z0-9._-]+)?$" @@ -323,7 +325,12 @@ def default_cache_dir() -> Path: def current_bundle_platform() -> str: system = platform.system() if system == "Linux": - return "linux64" + machine = platform.machine().lower() + if machine in {"x86_64", "amd64"}: + return "linux64" + if machine in {"aarch64", "arm64"}: + return "linux-arm64" + raise CacheException(f"Unsupported Linux architecture: {machine}") if system == "Darwin": return "osx64" if system == "Windows": @@ -721,22 +728,19 @@ def _resolve_release( bundle = self.catalog.find_release(release) if bundle: asset = bundle.source_asset_for_platform(platform_name) - if asset is None and platform_name != "all": + if asset is None and platform_name not in {"all", "linux-arm64"}: asset = bundle.source_asset_for_platform("all") - if asset is None: - raise CatalogException( - f"Bundle {release} has no source asset for {platform_name}." + if asset is not None: + destination = cache_path( + self.cache_dir, "sources", release, asset.name ) - destination = cache_path( - self.cache_dir, "sources", release, asset.name - ) - digest = download_file( - asset.url, - destination, - expected_sha256=asset.sha256, - expected_size=asset.size, - ) - return ResolvedBundleSource(destination, bundle, digest) + digest = download_file( + asset.url, + destination, + expected_sha256=asset.sha256, + expected_size=asset.size, + ) + return ResolvedBundleSource(destination, bundle, digest) asset_name = source_asset_name(platform_name) release_value = self.release_client.release(release) @@ -1009,7 +1013,7 @@ def write_json(path: Path, value: dict[str, Any]) -> None: def source_asset_name(platform_name: str) -> str: if platform_name == "all": return "codeql-bundle.tar.gz" - if platform_name not in {"linux64", "osx64", "win64"}: + if platform_name not in BUNDLE_PLATFORMS: raise CacheException(f"Unsupported bundle platform: {platform_name}") return f"codeql-bundle-{platform_name}.tar.gz" @@ -1017,6 +1021,16 @@ def source_asset_name(platform_name: str) -> str: def source_platform_for_request(requested_platforms: Iterable[str]) -> str: requested = tuple(requested_platforms) current = current_bundle_platform() + if "linux-arm64" in requested and current != "linux-arm64": + raise CacheException( + "Linux ARM64 bundles must be built on a Linux ARM64 host." + ) + if current == "linux-arm64": + if not requested or requested == (current,): + return current + raise CacheException( + "Linux ARM64 hosts can only build Linux ARM64 bundles." + ) if not requested: return "all" if requested == (current,): diff --git a/codeql_bundle/cache_cli.py b/codeql_bundle/cache_cli.py index 5b92983..77decbc 100644 --- a/codeql_bundle/cache_cli.py +++ b/codeql_bundle/cache_cli.py @@ -14,6 +14,7 @@ from semantic_version import Version from codeql_bundle.cache import ( + BUNDLE_PLATFORMS, CACHE_FORMAT_VERSION, CODEQL_ACTION_REPOSITORY, BundleCatalog, @@ -25,6 +26,7 @@ GitHubReleaseClient, ReleaseAsset, SourceAsset, + SOURCE_PLATFORMS, SupportedBundle, cache_path, compute_pack_fingerprint, @@ -50,7 +52,7 @@ logger = logging.getLogger(__name__) -SOURCE_PLATFORMS = ("all", "linux64", "osx64", "win64") +REQUIRED_SOURCE_PLATFORMS = set(SOURCE_PLATFORMS) - {"linux-arm64"} MAX_RELEASE_ASSET_SIZE = 2 * 1024 * 1024 * 1024 DEFAULT_COMPILATION_CACHE_SIZE_MB = 1536 @@ -142,7 +144,7 @@ def prune_cache(cache_dir: Path, max_age_days: float, dry_run: bool) -> None: "--platform", "platforms", multiple=True, - type=click.Choice(["linux64", "osx64", "win64"]), + type=click.Choice(BUNDLE_PLATFORMS), ) @click.option( "--cache-dir", @@ -212,11 +214,19 @@ def plan_release( client = GitHubReleaseClient() release_value = client.release(release) source_assets = _release_source_assets(release_value) + platform_name = current_bundle_platform() source_asset = next( - asset - for asset in source_assets - if asset.platform == current_bundle_platform() + ( + asset + for asset in source_assets + if asset.platform == platform_name + ), + None, ) + if source_asset is None: + raise click.ClickException( + f"Upstream release {release} has no source bundle for {platform_name}." + ) source_path = cache_path(cache_dir, "sources", release, source_asset.name) download_file( source_asset.url, @@ -493,7 +503,7 @@ def _verify_cache_with_bundle( "validated_platforms", multiple=True, required=True, - type=click.Choice(["linux64", "osx64", "win64"]), + type=click.Choice(BUNDLE_PLATFORMS), ) @click.option( "--output", @@ -585,6 +595,8 @@ def _release_source_assets(release: dict[str, Any]) -> tuple[SourceAsset, ...]: name = source_asset_name(platform_name) asset = assets.get(name) if asset is None: + if platform_name == "linux-arm64": + continue raise click.ClickException( f"Upstream release {release['tag_name']} has no {name}." ) @@ -800,8 +812,9 @@ def _read_plan(path: Path) -> dict[str, Any]: validate_remote_url(source.url) source_platforms.add(source.platform) if ( - source_platforms != set(SOURCE_PLATFORMS) - or len(value["source_assets"]) != len(SOURCE_PLATFORMS) + not REQUIRED_SOURCE_PLATFORMS.issubset(source_platforms) + or not source_platforms.issubset(SOURCE_PLATFORMS) + or len(value["source_assets"]) != len(source_platforms) ): raise ValueError("incomplete source platform inventory") diff --git a/codeql_bundle/cli.py b/codeql_bundle/cli.py index d8d8c1b..1d64d08 100644 --- a/codeql_bundle/cli.py +++ b/codeql_bundle/cli.py @@ -12,11 +12,13 @@ from codeql_bundle.helpers.codeql import CodeQLException from codeql_bundle.helpers.bundle import CustomBundle, BundleException, BundlePlatform from codeql_bundle.cache import ( + BUNDLE_PLATFORMS, BundleCatalog, BundleSourceResolver, CacheException, CatalogLoader, CompilationCacheManager, + current_bundle_platform, default_cache_dir, ) from typing import List, Optional @@ -65,7 +67,7 @@ "-p", "--platform", multiple=True, - type=click.Choice(["linux64", "osx64", "win64"], case_sensitive=False), + type=click.Choice(BUNDLE_PLATFORMS, case_sensitive=False), help="Target platform for the bundle", ) @click.option( @@ -139,6 +141,9 @@ def main( workspace = workspace.parent try: + if not platform and current_bundle_platform() == "linux-arm64": + platform = ["linux-arm64"] + use_compilation_cache = not no_compilation_cache and not no_precompile catalog = ( CatalogLoader(cache_dir).load(cache_manifest) diff --git a/codeql_bundle/helpers/bundle.py b/codeql_bundle/helpers/bundle.py index 24f2975..44283f6 100644 --- a/codeql_bundle/helpers/bundle.py +++ b/codeql_bundle/helpers/bundle.py @@ -20,6 +20,7 @@ from codeql_bundle.cache import ( CompilationCacheManager, compute_pack_fingerprint, + current_bundle_platform, safe_extract_tar, ) @@ -199,11 +200,14 @@ class BundlePlatform(Enum): LINUX = 1 WINDOWS = 2 OSX = 3 + LINUX_ARM64 = 4 @staticmethod def from_string(platform: str) -> "BundlePlatform": if platform.lower() == "linux" or platform.lower() == "linux64": return BundlePlatform.LINUX + elif platform.lower() == "linux-arm64": + return BundlePlatform.LINUX_ARM64 elif platform.lower() == "windows" or platform.lower() == "win64": return BundlePlatform.WINDOWS elif platform.lower() == "osx" or platform.lower() == "osx64": @@ -214,6 +218,8 @@ def from_string(platform: str) -> "BundlePlatform": def __str__(self): if self == BundlePlatform.LINUX: return "linux64" + elif self == BundlePlatform.LINUX_ARM64: + return "linux-arm64" elif self == BundlePlatform.WINDOWS: return "win64" elif self == BundlePlatform.OSX: @@ -248,6 +254,12 @@ def supports_linux() -> set[BundlePlatform]: else: return set() + def supports_linux_arm64() -> set[BundlePlatform]: + if (self.bundle_path / "cpp" / "tools" / "linux-arm64").exists(): + return {BundlePlatform.LINUX_ARM64} + else: + return set() + def supports_macos() -> set[BundlePlatform]: if (self.bundle_path / "cpp" / "tools" / "osx64").exists(): return {BundlePlatform.OSX} @@ -261,20 +273,15 @@ def supports_windows() -> set[BundlePlatform]: return set() self.platforms: set[BundlePlatform] = ( - supports_linux() | supports_macos() | supports_windows() + supports_linux() + | supports_linux_arm64() + | supports_macos() + | supports_windows() ) - current_system = platform.system() - if not current_system in ["Linux", "Darwin", "Windows"]: - raise BundleException(f"Unsupported system: {current_system}") - if current_system == "Linux" and BundlePlatform.LINUX not in self.platforms: - raise BundleException("Bundle doesn't support Linux!") - elif current_system == "Darwin" and BundlePlatform.OSX not in self.platforms: - raise BundleException("Bundle doesn't support OSX!") - elif ( - current_system == "Windows" and BundlePlatform.WINDOWS not in self.platforms - ): - raise BundleException("Bundle doesn't support Windows!") + current_platform = BundlePlatform.from_string(current_bundle_platform()) + if current_platform not in self.platforms: + raise BundleException(f"Bundle doesn't support {current_platform}!") self.codeql = CodeQL(self.bundle_codeql_exe) @@ -919,22 +926,36 @@ def get_nonplatform_tool_paths( """Get a list of paths to tools that are not for the specified platform relative to the root of a bundle.""" specialize_path: Optional[Callable[[Path], List[Path]]] = None linux64_subpaths = [Path("linux64"), Path("linux")] + linux_arm64_subpaths = [Path("linux-arm64")] osx64_subpaths = [Path("osx64"), Path("macos")] win64_subpaths = [Path("win64"), Path("windows")] if platform == BundlePlatform.LINUX: specialize_path = lambda p: [ p / subpath - for subpath in osx64_subpaths + win64_subpaths + for subpath in linux_arm64_subpaths + + osx64_subpaths + + win64_subpaths + ] + elif platform == BundlePlatform.LINUX_ARM64: + specialize_path = lambda p: [ + p / subpath + for subpath in linux64_subpaths + + osx64_subpaths + + win64_subpaths ] elif platform == BundlePlatform.WINDOWS: specialize_path = lambda p: [ p / subpath - for subpath in osx64_subpaths + linux64_subpaths + for subpath in osx64_subpaths + + linux64_subpaths + + linux_arm64_subpaths ] elif platform == BundlePlatform.OSX: specialize_path = lambda p: [ p / subpath - for subpath in linux64_subpaths + win64_subpaths + for subpath in linux64_subpaths + + linux_arm64_subpaths + + win64_subpaths ] else: raise BundleException(f"Unsupported platform {platform}.") @@ -960,10 +981,20 @@ def filter(tarinfo: tarfile.TarInfo) -> Optional[tarfile.TarInfo]: if platform == BundlePlatform.LINUX: exclusion_paths.append(Path("swift/qltest/osx64")) exclusion_paths.append(Path("swift/resource-dir/osx64")) + exclusion_paths.append(Path("swift/qltest/linux-arm64")) + exclusion_paths.append(Path("swift/resource-dir/linux-arm64")) + + if platform == BundlePlatform.LINUX_ARM64: + exclusion_paths.append(Path("swift/qltest/osx64")) + exclusion_paths.append(Path("swift/resource-dir/osx64")) + exclusion_paths.append(Path("swift/qltest/linux64")) + exclusion_paths.append(Path("swift/resource-dir/linux64")) if platform == BundlePlatform.OSX: exclusion_paths.append(Path("swift/qltest/linux64")) exclusion_paths.append(Path("swift/resource-dir/linux64")) + exclusion_paths.append(Path("swift/qltest/linux-arm64")) + exclusion_paths.append(Path("swift/resource-dir/linux-arm64")) tarfile_path_root = Path(tarfile_path.parts[0]) exclusion_paths = [ diff --git a/codeql_bundle/supported-codeql-bundles.schema.json b/codeql_bundle/supported-codeql-bundles.schema.json index d69afd4..af9458c 100644 --- a/codeql_bundle/supported-codeql-bundles.schema.json +++ b/codeql_bundle/supported-codeql-bundles.schema.json @@ -84,6 +84,7 @@ "enum": [ "all", "linux64", + "linux-arm64", "osx64", "win64" ] @@ -177,6 +178,7 @@ "items": { "enum": [ "linux64", + "linux-arm64", "osx64", "win64" ] diff --git a/pyproject.toml b/pyproject.toml index 664a87d..2022d0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "codeql-bundle" -version = "0.5.0" +version = "0.6.0" description = "Tool to create custom CodeQL bundles" authors = ["Remco Vermeulen "] readme = "README.md" diff --git a/tests/test_bundle_platform.py b/tests/test_bundle_platform.py new file mode 100644 index 0000000..dbd8608 --- /dev/null +++ b/tests/test_bundle_platform.py @@ -0,0 +1,23 @@ +import unittest + +from codeql_bundle.helpers.bundle import BundleException, BundlePlatform + + +class BundlePlatformTests(unittest.TestCase): + def test_supported_platform_names_round_trip(self) -> None: + for platform_name in ("linux64", "linux-arm64", "osx64", "win64"): + self.assertEqual( + platform_name, + str(BundlePlatform.from_string(platform_name)), + ) + + def test_linux_alias_remains_linux_x64(self) -> None: + self.assertEqual(BundlePlatform.LINUX, BundlePlatform.from_string("linux")) + + def test_invalid_platform_is_rejected(self) -> None: + with self.assertRaises(BundleException): + BundlePlatform.from_string("linux-riscv64") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cache.py b/tests/test_cache.py index 748e757..ea0ce73 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -233,12 +233,46 @@ def test_release_source_is_runnable_on_current_platform(self) -> None: for platform in ("linux64", "osx64", "win64") if platform != current ) - self.assertEqual("all", source_platform_for_request(())) + expected_default = "linux-arm64" if current == "linux-arm64" else "all" + self.assertEqual(expected_default, source_platform_for_request(())) self.assertEqual(current, source_platform_for_request((current,))) - self.assertEqual("all", source_platform_for_request((other,))) - self.assertEqual( - "all", source_platform_for_request(("linux64", "win64")) - ) + if current == "linux-arm64": + with self.assertRaises(CacheException): + source_platform_for_request((other,)) + with self.assertRaises(CacheException): + source_platform_for_request(("linux-arm64", "win64")) + else: + self.assertEqual("all", source_platform_for_request((other,))) + self.assertEqual( + "all", source_platform_for_request(("linux64", "win64")) + ) + with self.assertRaises(CacheException): + source_platform_for_request(("linux-arm64",)) + + def test_current_bundle_platform_distinguishes_linux_architecture(self) -> None: + with patch("codeql_bundle.cache.platform.system", return_value="Linux"): + with patch( + "codeql_bundle.cache.platform.machine", return_value="x86_64" + ): + self.assertEqual("linux64", current_bundle_platform()) + with patch( + "codeql_bundle.cache.platform.machine", return_value="aarch64" + ): + self.assertEqual("linux-arm64", current_bundle_platform()) + self.assertEqual( + "linux-arm64", source_platform_for_request(()) + ) + self.assertEqual( + "linux-arm64", + source_platform_for_request(("linux-arm64",)), + ) + with self.assertRaises(CacheException): + source_platform_for_request(("linux64",)) + with patch( + "codeql_bundle.cache.platform.machine", return_value="riscv64" + ): + with self.assertRaises(CacheException): + current_bundle_platform() def test_local_archive_matches_catalog_digest(self) -> None: with TemporaryDirectory() as directory: @@ -352,6 +386,57 @@ def release(self, tag: str) -> dict[str, object]: self.assertEqual(sha256_file(archive), resolved.digest) + def test_linux_arm64_does_not_fall_back_to_all_platform_source(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + served = root / "served" + served.mkdir() + arm_archive = served / "codeql-bundle-linux-arm64.tar.gz" + arm_archive.write_bytes(b"arm64") + all_source = SourceAsset( + name="codeql-bundle.tar.gz", + url="https://example.test/codeql-bundle.tar.gz", + sha256="1" * 64, + size=100, + platform="all", + ) + cache = ReleaseAsset( + name="cache.tar.gz", + url="https://example.test/cache.tar.gz", + sha256="2" * 64, + size=10, + ) + with serve(served) as base_url: + release = { + "tag_name": "codeql-bundle-v1.2.3", + "assets": [ + { + "browser_download_url": f"{base_url}/{arm_archive.name}", + "digest": f"sha256:{sha256_file(arm_archive)}", + "name": arm_archive.name, + "size": arm_archive.stat().st_size, + } + ], + } + + class ReleaseClient(GitHubReleaseClient): + def release(self, tag: str) -> dict[str, object]: + return release + + with patch( + "codeql_bundle.cache.current_bundle_platform", + return_value="linux-arm64", + ): + resolved = BundleSourceResolver( + BundleCatalog([bundle_with_assets(all_source, cache)]), + root / "downloads", + release_client=ReleaseClient(), + ).resolve("codeql-bundle-v1.2.3", ["linux-arm64"]) + + self.assertEqual(arm_archive.name, resolved.path.name) + self.assertEqual(arm_archive.read_bytes(), resolved.path.read_bytes()) + self.assertIsNone(resolved.supported_bundle) + def test_safe_extract_rejects_parent_path(self) -> None: with TemporaryDirectory() as directory: root = Path(directory) diff --git a/tests/test_cache_cli.py b/tests/test_cache_cli.py index 9bbe664..9eb2b69 100644 --- a/tests/test_cache_cli.py +++ b/tests/test_cache_cli.py @@ -87,6 +87,7 @@ def test_catalog_entry_can_be_added_to_catalog(self) -> None: [ ("all", "codeql-bundle.tar.gz"), ("linux64", "codeql-bundle-linux64.tar.gz"), + ("linux-arm64", "codeql-bundle-linux-arm64.tar.gz"), ("osx64", "codeql-bundle-osx64.tar.gz"), ("win64", "codeql-bundle-win64.tar.gz"), ], From 5febdd1a54f25f7e729f289bb7e282d5a582375e Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 25 Sep 2026 01:11:25 +0200 Subject: [PATCH 2/9] Simplify Linux ARM64 support Keep ARM64 compilation-cache publication out of scope and reuse the existing Linux filtering path for the platform-specific source archive. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../workflows/codeql-compilation-caches.yml | 37 ++---- codeql_bundle/cache.py | 46 ++++--- codeql_bundle/cache_cli.py | 29 ++--- codeql_bundle/cli.py | 10 +- codeql_bundle/helpers/bundle.py | 41 ++----- .../supported-codeql-bundles.schema.json | 2 - tests/test_bundle_platform.py | 23 ---- tests/test_cache.py | 113 ++++-------------- tests/test_cache_cli.py | 1 - 9 files changed, 81 insertions(+), 221 deletions(-) delete mode 100644 tests/test_bundle_platform.py diff --git a/.github/workflows/codeql-compilation-caches.yml b/.github/workflows/codeql-compilation-caches.yml index 539a240..101a950 100644 --- a/.github/workflows/codeql-compilation-caches.yml +++ b/.github/workflows/codeql-compilation-caches.yml @@ -45,7 +45,6 @@ jobs: matrix: ${{ steps.plan.outputs.matrix || '[{"language":"skip","target":"skip"}]' }} release: ${{ steps.version.outputs.release }} skip: ${{ steps.existing.outputs.skip }} - verify_matrix: ${{ steps.plan.outputs.verify_matrix || '[{"platform":"skip","os":"ubuntu-latest"}]' }} steps: - uses: actions/checkout@v4 @@ -116,7 +115,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.CODEQL_BUNDLE_CACHE_DIR }}/sources/${{ steps.version.outputs.release }} - key: codeql-source-${{ steps.version.outputs.release }}-${{ runner.os }}-${{ runner.arch }} + key: codeql-source-${{ steps.version.outputs.release }}-${{ runner.os }} - if: steps.existing.outputs.skip != 'true' id: plan @@ -131,22 +130,6 @@ jobs: --output release-plan.json echo "cache_release=$(jq -r '.cache_release' release-plan.json)" >> "$GITHUB_OUTPUT" echo "matrix=$(jq -c '.targets' release-plan.json)" >> "$GITHUB_OUTPUT" - verify_matrix="$(jq -c ' - [.source_assets[].platform | - select(. != "all") | - { - platform: ., - os: ( - if . == "linux64" then "ubuntu-latest" - elif . == "linux-arm64" then "ubuntu-24.04-arm" - elif . == "osx64" then "macos-latest" - elif . == "win64" then "windows-latest" - else error("Unsupported source platform: " + .) - end - ) - } - ]' release-plan.json)" - echo "verify_matrix=$verify_matrix" >> "$GITHUB_OUTPUT" - if: steps.existing.outputs.skip != 'true' uses: actions/upload-artifact@v4 @@ -179,7 +162,7 @@ jobs: - uses: actions/cache@v4 with: path: ${{ env.CODEQL_BUNDLE_CACHE_DIR }}/sources/${{ needs.plan.outputs.release }} - key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }}-${{ runner.arch }} + key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }} - name: Build and verify cache run: | @@ -206,7 +189,10 @@ jobs: strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.plan.outputs.verify_matrix) }} + os: + - ubuntu-latest + - macos-latest + - windows-latest runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -230,7 +216,7 @@ jobs: - uses: actions/cache@v4 with: path: ${{ env.CODEQL_BUNDLE_CACHE_DIR }}/sources/${{ needs.plan.outputs.release }} - key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }}-${{ runner.arch }} + key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }} - name: Verify all caches run: codeql-bundle-cache verify-all --plan release-plan.json --assets-dir dist @@ -313,16 +299,13 @@ jobs: - name: Create candidate catalog entry if: steps.recheck.outputs.skip != 'true' - shell: bash run: | - validated_platform_args=() - while IFS= read -r platform; do - validated_platform_args+=(--validated-platform "$platform") - done < <(jq -r '.source_assets[].platform | select(. != "all")' release-plan.json) codeql-bundle-cache catalog-entry \ --plan release-plan.json \ --assets-dir dist \ - "${validated_platform_args[@]}" \ + --validated-platform linux64 \ + --validated-platform osx64 \ + --validated-platform win64 \ --output catalog-entry.json codeql-bundle-cache verify-entry \ --entry catalog-entry.json \ diff --git a/codeql_bundle/cache.py b/codeql_bundle/cache.py index 65c3d6f..ee001bc 100644 --- a/codeql_bundle/cache.py +++ b/codeql_bundle/cache.py @@ -36,8 +36,6 @@ CACHE_FORMAT_VERSION = 1 DOWNLOAD_CHUNK_SIZE = 1024 * 1024 CATALOG_REFRESH_SECONDS = 60 * 60 -BUNDLE_PLATFORMS = ("linux64", "linux-arm64", "osx64", "win64") -SOURCE_PLATFORMS = ("all", *BUNDLE_PLATFORMS) RELEASE_PATTERN = re.compile(r"^codeql-bundle-v\d+\.\d+\.\d+$") CACHE_RELEASE_PATTERN = re.compile( r"^codeql-compilation-cache-v\d+\.\d+\.\d+(?:-[A-Za-z0-9._-]+)?$" @@ -725,22 +723,30 @@ def _resolve_release( ) -> ResolvedBundleSource: platform_name = source_platform_for_request(requested_platforms) validate_release(release) - bundle = self.catalog.find_release(release) + # ponytail: ARM64 has no compilation-cache catalog; fetch its release asset. + bundle = ( + None + if platform_name == "linux-arm64" + else self.catalog.find_release(release) + ) if bundle: asset = bundle.source_asset_for_platform(platform_name) - if asset is None and platform_name not in {"all", "linux-arm64"}: + if asset is None and platform_name != "all": asset = bundle.source_asset_for_platform("all") - if asset is not None: - destination = cache_path( - self.cache_dir, "sources", release, asset.name - ) - digest = download_file( - asset.url, - destination, - expected_sha256=asset.sha256, - expected_size=asset.size, + if asset is None: + raise CatalogException( + f"Bundle {release} has no source asset for {platform_name}." ) - return ResolvedBundleSource(destination, bundle, digest) + destination = cache_path( + self.cache_dir, "sources", release, asset.name + ) + digest = download_file( + asset.url, + destination, + expected_sha256=asset.sha256, + expected_size=asset.size, + ) + return ResolvedBundleSource(destination, bundle, digest) asset_name = source_asset_name(platform_name) release_value = self.release_client.release(release) @@ -1013,7 +1019,7 @@ def write_json(path: Path, value: dict[str, Any]) -> None: def source_asset_name(platform_name: str) -> str: if platform_name == "all": return "codeql-bundle.tar.gz" - if platform_name not in BUNDLE_PLATFORMS: + if platform_name not in {"linux64", "linux-arm64", "osx64", "win64"}: raise CacheException(f"Unsupported bundle platform: {platform_name}") return f"codeql-bundle-{platform_name}.tar.gz" @@ -1021,16 +1027,8 @@ def source_asset_name(platform_name: str) -> str: def source_platform_for_request(requested_platforms: Iterable[str]) -> str: requested = tuple(requested_platforms) current = current_bundle_platform() - if "linux-arm64" in requested and current != "linux-arm64": - raise CacheException( - "Linux ARM64 bundles must be built on a Linux ARM64 host." - ) if current == "linux-arm64": - if not requested or requested == (current,): - return current - raise CacheException( - "Linux ARM64 hosts can only build Linux ARM64 bundles." - ) + return current if not requested: return "all" if requested == (current,): diff --git a/codeql_bundle/cache_cli.py b/codeql_bundle/cache_cli.py index 77decbc..5b92983 100644 --- a/codeql_bundle/cache_cli.py +++ b/codeql_bundle/cache_cli.py @@ -14,7 +14,6 @@ from semantic_version import Version from codeql_bundle.cache import ( - BUNDLE_PLATFORMS, CACHE_FORMAT_VERSION, CODEQL_ACTION_REPOSITORY, BundleCatalog, @@ -26,7 +25,6 @@ GitHubReleaseClient, ReleaseAsset, SourceAsset, - SOURCE_PLATFORMS, SupportedBundle, cache_path, compute_pack_fingerprint, @@ -52,7 +50,7 @@ logger = logging.getLogger(__name__) -REQUIRED_SOURCE_PLATFORMS = set(SOURCE_PLATFORMS) - {"linux-arm64"} +SOURCE_PLATFORMS = ("all", "linux64", "osx64", "win64") MAX_RELEASE_ASSET_SIZE = 2 * 1024 * 1024 * 1024 DEFAULT_COMPILATION_CACHE_SIZE_MB = 1536 @@ -144,7 +142,7 @@ def prune_cache(cache_dir: Path, max_age_days: float, dry_run: bool) -> None: "--platform", "platforms", multiple=True, - type=click.Choice(BUNDLE_PLATFORMS), + type=click.Choice(["linux64", "osx64", "win64"]), ) @click.option( "--cache-dir", @@ -214,19 +212,11 @@ def plan_release( client = GitHubReleaseClient() release_value = client.release(release) source_assets = _release_source_assets(release_value) - platform_name = current_bundle_platform() source_asset = next( - ( - asset - for asset in source_assets - if asset.platform == platform_name - ), - None, + asset + for asset in source_assets + if asset.platform == current_bundle_platform() ) - if source_asset is None: - raise click.ClickException( - f"Upstream release {release} has no source bundle for {platform_name}." - ) source_path = cache_path(cache_dir, "sources", release, source_asset.name) download_file( source_asset.url, @@ -503,7 +493,7 @@ def _verify_cache_with_bundle( "validated_platforms", multiple=True, required=True, - type=click.Choice(BUNDLE_PLATFORMS), + type=click.Choice(["linux64", "osx64", "win64"]), ) @click.option( "--output", @@ -595,8 +585,6 @@ def _release_source_assets(release: dict[str, Any]) -> tuple[SourceAsset, ...]: name = source_asset_name(platform_name) asset = assets.get(name) if asset is None: - if platform_name == "linux-arm64": - continue raise click.ClickException( f"Upstream release {release['tag_name']} has no {name}." ) @@ -812,9 +800,8 @@ def _read_plan(path: Path) -> dict[str, Any]: validate_remote_url(source.url) source_platforms.add(source.platform) if ( - not REQUIRED_SOURCE_PLATFORMS.issubset(source_platforms) - or not source_platforms.issubset(SOURCE_PLATFORMS) - or len(value["source_assets"]) != len(source_platforms) + source_platforms != set(SOURCE_PLATFORMS) + or len(value["source_assets"]) != len(SOURCE_PLATFORMS) ): raise ValueError("incomplete source platform inventory") diff --git a/codeql_bundle/cli.py b/codeql_bundle/cli.py index 1d64d08..542121d 100644 --- a/codeql_bundle/cli.py +++ b/codeql_bundle/cli.py @@ -12,13 +12,11 @@ from codeql_bundle.helpers.codeql import CodeQLException from codeql_bundle.helpers.bundle import CustomBundle, BundleException, BundlePlatform from codeql_bundle.cache import ( - BUNDLE_PLATFORMS, BundleCatalog, BundleSourceResolver, CacheException, CatalogLoader, CompilationCacheManager, - current_bundle_platform, default_cache_dir, ) from typing import List, Optional @@ -67,7 +65,10 @@ "-p", "--platform", multiple=True, - type=click.Choice(BUNDLE_PLATFORMS, case_sensitive=False), + type=click.Choice( + ["linux64", "linux-arm64", "osx64", "win64"], + case_sensitive=False, + ), help="Target platform for the bundle", ) @click.option( @@ -141,9 +142,6 @@ def main( workspace = workspace.parent try: - if not platform and current_bundle_platform() == "linux-arm64": - platform = ["linux-arm64"] - use_compilation_cache = not no_compilation_cache and not no_precompile catalog = ( CatalogLoader(cache_dir).load(cache_manifest) diff --git a/codeql_bundle/helpers/bundle.py b/codeql_bundle/helpers/bundle.py index 44283f6..bdce99d 100644 --- a/codeql_bundle/helpers/bundle.py +++ b/codeql_bundle/helpers/bundle.py @@ -926,36 +926,26 @@ def get_nonplatform_tool_paths( """Get a list of paths to tools that are not for the specified platform relative to the root of a bundle.""" specialize_path: Optional[Callable[[Path], List[Path]]] = None linux64_subpaths = [Path("linux64"), Path("linux")] - linux_arm64_subpaths = [Path("linux-arm64")] osx64_subpaths = [Path("osx64"), Path("macos")] win64_subpaths = [Path("win64"), Path("windows")] - if platform == BundlePlatform.LINUX: + # ponytail: ARM64 is only shipped in its own source bundle. + if platform in { + BundlePlatform.LINUX, + BundlePlatform.LINUX_ARM64, + }: specialize_path = lambda p: [ p / subpath - for subpath in linux_arm64_subpaths - + osx64_subpaths - + win64_subpaths - ] - elif platform == BundlePlatform.LINUX_ARM64: - specialize_path = lambda p: [ - p / subpath - for subpath in linux64_subpaths - + osx64_subpaths - + win64_subpaths + for subpath in osx64_subpaths + win64_subpaths ] elif platform == BundlePlatform.WINDOWS: specialize_path = lambda p: [ p / subpath - for subpath in osx64_subpaths - + linux64_subpaths - + linux_arm64_subpaths + for subpath in osx64_subpaths + linux64_subpaths ] elif platform == BundlePlatform.OSX: specialize_path = lambda p: [ p / subpath - for subpath in linux64_subpaths - + linux_arm64_subpaths - + win64_subpaths + for subpath in linux64_subpaths + win64_subpaths ] else: raise BundleException(f"Unsupported platform {platform}.") @@ -978,23 +968,16 @@ def filter(tarinfo: tarfile.TarInfo) -> Optional[tarfile.TarInfo]: exclusion_paths.append(Path("swift/qltest")) exclusion_paths.append(Path("swift/resource-dir")) - if platform == BundlePlatform.LINUX: - exclusion_paths.append(Path("swift/qltest/osx64")) - exclusion_paths.append(Path("swift/resource-dir/osx64")) - exclusion_paths.append(Path("swift/qltest/linux-arm64")) - exclusion_paths.append(Path("swift/resource-dir/linux-arm64")) - - if platform == BundlePlatform.LINUX_ARM64: + if platform in { + BundlePlatform.LINUX, + BundlePlatform.LINUX_ARM64, + }: exclusion_paths.append(Path("swift/qltest/osx64")) exclusion_paths.append(Path("swift/resource-dir/osx64")) - exclusion_paths.append(Path("swift/qltest/linux64")) - exclusion_paths.append(Path("swift/resource-dir/linux64")) if platform == BundlePlatform.OSX: exclusion_paths.append(Path("swift/qltest/linux64")) exclusion_paths.append(Path("swift/resource-dir/linux64")) - exclusion_paths.append(Path("swift/qltest/linux-arm64")) - exclusion_paths.append(Path("swift/resource-dir/linux-arm64")) tarfile_path_root = Path(tarfile_path.parts[0]) exclusion_paths = [ diff --git a/codeql_bundle/supported-codeql-bundles.schema.json b/codeql_bundle/supported-codeql-bundles.schema.json index af9458c..d69afd4 100644 --- a/codeql_bundle/supported-codeql-bundles.schema.json +++ b/codeql_bundle/supported-codeql-bundles.schema.json @@ -84,7 +84,6 @@ "enum": [ "all", "linux64", - "linux-arm64", "osx64", "win64" ] @@ -178,7 +177,6 @@ "items": { "enum": [ "linux64", - "linux-arm64", "osx64", "win64" ] diff --git a/tests/test_bundle_platform.py b/tests/test_bundle_platform.py deleted file mode 100644 index dbd8608..0000000 --- a/tests/test_bundle_platform.py +++ /dev/null @@ -1,23 +0,0 @@ -import unittest - -from codeql_bundle.helpers.bundle import BundleException, BundlePlatform - - -class BundlePlatformTests(unittest.TestCase): - def test_supported_platform_names_round_trip(self) -> None: - for platform_name in ("linux64", "linux-arm64", "osx64", "win64"): - self.assertEqual( - platform_name, - str(BundlePlatform.from_string(platform_name)), - ) - - def test_linux_alias_remains_linux_x64(self) -> None: - self.assertEqual(BundlePlatform.LINUX, BundlePlatform.from_string("linux")) - - def test_invalid_platform_is_rejected(self) -> None: - with self.assertRaises(BundleException): - BundlePlatform.from_string("linux-riscv64") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_cache.py b/tests/test_cache.py index ea0ce73..89d837d 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -30,8 +30,10 @@ current_bundle_platform, safe_extract_tar, sha256_file, + source_asset_name, source_platform_for_request, ) +from codeql_bundle.helpers.bundle import BundlePlatform from codeql_bundle.helpers.codeql import CodeQLPack, CodeQLPackConfig @@ -233,46 +235,32 @@ def test_release_source_is_runnable_on_current_platform(self) -> None: for platform in ("linux64", "osx64", "win64") if platform != current ) - expected_default = "linux-arm64" if current == "linux-arm64" else "all" - self.assertEqual(expected_default, source_platform_for_request(())) - self.assertEqual(current, source_platform_for_request((current,))) if current == "linux-arm64": - with self.assertRaises(CacheException): - source_platform_for_request((other,)) - with self.assertRaises(CacheException): - source_platform_for_request(("linux-arm64", "win64")) - else: - self.assertEqual("all", source_platform_for_request((other,))) + self.assertEqual(current, source_platform_for_request(())) + return + self.assertEqual("all", source_platform_for_request(())) + self.assertEqual(current, source_platform_for_request((current,))) + self.assertEqual("all", source_platform_for_request((other,))) + self.assertEqual( + "all", source_platform_for_request(("linux64", "win64")) + ) + + def test_linux_arm64_platform(self) -> None: + with patch( + "codeql_bundle.cache.platform.system", return_value="Linux" + ), patch( + "codeql_bundle.cache.platform.machine", return_value="aarch64" + ): + self.assertEqual("linux-arm64", current_bundle_platform()) + self.assertEqual("linux-arm64", source_platform_for_request(())) self.assertEqual( - "all", source_platform_for_request(("linux64", "win64")) + "codeql-bundle-linux-arm64.tar.gz", + source_asset_name("linux-arm64"), + ) + self.assertEqual( + "linux-arm64", + str(BundlePlatform.from_string("linux-arm64")), ) - with self.assertRaises(CacheException): - source_platform_for_request(("linux-arm64",)) - - def test_current_bundle_platform_distinguishes_linux_architecture(self) -> None: - with patch("codeql_bundle.cache.platform.system", return_value="Linux"): - with patch( - "codeql_bundle.cache.platform.machine", return_value="x86_64" - ): - self.assertEqual("linux64", current_bundle_platform()) - with patch( - "codeql_bundle.cache.platform.machine", return_value="aarch64" - ): - self.assertEqual("linux-arm64", current_bundle_platform()) - self.assertEqual( - "linux-arm64", source_platform_for_request(()) - ) - self.assertEqual( - "linux-arm64", - source_platform_for_request(("linux-arm64",)), - ) - with self.assertRaises(CacheException): - source_platform_for_request(("linux64",)) - with patch( - "codeql_bundle.cache.platform.machine", return_value="riscv64" - ): - with self.assertRaises(CacheException): - current_bundle_platform() def test_local_archive_matches_catalog_digest(self) -> None: with TemporaryDirectory() as directory: @@ -386,57 +374,6 @@ def release(self, tag: str) -> dict[str, object]: self.assertEqual(sha256_file(archive), resolved.digest) - def test_linux_arm64_does_not_fall_back_to_all_platform_source(self) -> None: - with TemporaryDirectory() as directory: - root = Path(directory) - served = root / "served" - served.mkdir() - arm_archive = served / "codeql-bundle-linux-arm64.tar.gz" - arm_archive.write_bytes(b"arm64") - all_source = SourceAsset( - name="codeql-bundle.tar.gz", - url="https://example.test/codeql-bundle.tar.gz", - sha256="1" * 64, - size=100, - platform="all", - ) - cache = ReleaseAsset( - name="cache.tar.gz", - url="https://example.test/cache.tar.gz", - sha256="2" * 64, - size=10, - ) - with serve(served) as base_url: - release = { - "tag_name": "codeql-bundle-v1.2.3", - "assets": [ - { - "browser_download_url": f"{base_url}/{arm_archive.name}", - "digest": f"sha256:{sha256_file(arm_archive)}", - "name": arm_archive.name, - "size": arm_archive.stat().st_size, - } - ], - } - - class ReleaseClient(GitHubReleaseClient): - def release(self, tag: str) -> dict[str, object]: - return release - - with patch( - "codeql_bundle.cache.current_bundle_platform", - return_value="linux-arm64", - ): - resolved = BundleSourceResolver( - BundleCatalog([bundle_with_assets(all_source, cache)]), - root / "downloads", - release_client=ReleaseClient(), - ).resolve("codeql-bundle-v1.2.3", ["linux-arm64"]) - - self.assertEqual(arm_archive.name, resolved.path.name) - self.assertEqual(arm_archive.read_bytes(), resolved.path.read_bytes()) - self.assertIsNone(resolved.supported_bundle) - def test_safe_extract_rejects_parent_path(self) -> None: with TemporaryDirectory() as directory: root = Path(directory) diff --git a/tests/test_cache_cli.py b/tests/test_cache_cli.py index 9eb2b69..9bbe664 100644 --- a/tests/test_cache_cli.py +++ b/tests/test_cache_cli.py @@ -87,7 +87,6 @@ def test_catalog_entry_can_be_added_to_catalog(self) -> None: [ ("all", "codeql-bundle.tar.gz"), ("linux64", "codeql-bundle-linux64.tar.gz"), - ("linux-arm64", "codeql-bundle-linux-arm64.tar.gz"), ("osx64", "codeql-bundle-osx64.tar.gz"), ("win64", "codeql-bundle-win64.tar.gz"), ], From 4c09ec3381a71a67f9e71a198ce6ce7143dd02f4 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 25 Sep 2026 12:17:17 +0200 Subject: [PATCH 3/9] Use platform-specific CodeQL bundles Stop depending on the deprecated all-platform release asset and select the host-specific bundle for release tags. Also use the native Linux ARM64 Java tools when installing certificates. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 29 ++++++++++------- codeql_bundle/cache.py | 22 ++++++------- codeql_bundle/cache_cli.py | 6 ++-- codeql_bundle/cli.py | 3 +- codeql_bundle/helpers/bundle.py | 47 +++++++++++++------------- tests/test_bundle.py | 58 +++++++++++++++++++++++++++++++++ tests/test_cache.py | 27 +++++++-------- tests/test_cache_cli.py | 1 - 8 files changed, 128 insertions(+), 65 deletions(-) create mode 100644 tests/test_bundle.py diff --git a/README.md b/README.md index 7f9489d..2d19705 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ python3.11 -m pip install https://github.com/advanced-security/codeql-bundle/rel ## Usage The source bundle can be an existing local archive or directory, a -`github/codeql-action` release tag, or an HTTP(S) URL. Release tags and URLs are -downloaded into a persistent local cache. +`github/codeql-action` release tag, or an HTTP(S) URL. Release tags select the +current platform's bundle, and downloads are stored in a persistent local cache. The CodeQL bundle application requires a [CodeQL workspace](https://codeql.github.com/docs/codeql-cli/about-codeql-workspaces/) to locate the packs you want to include in a custom bundle. You can see the packs available in your workspace by running `codeql pack ls -- ` where `` is the root directory of your CodeQL workspace. @@ -37,20 +37,25 @@ with the command: codeql-bundle --bundle codeql-bundle-v2.26.1 --output codeql-custom-bundle.tar.gz --workspace --log INFO ``` -If the source bundle is the platform agnostic bundle then you can create platform specific bundles to reduce the size of the used bundle(s). -The following example creates bundles for the platforms included in that source archive. +The upstream all-platform bundle is +[deprecated](https://github.blog/changelog/2026-09-22-deprecation-notice-all-platform-codeql-bundle/) +and will be removed in mid-March 2027. Release tags therefore use a +platform-specific source and can only build for the current platform. Run +`codeql-bundle` on each target platform. ```bash -codeql-bundle --bundle --output --workspace --log INFO -p linux64 -p osx64 -p win64 +codeql-bundle --bundle codeql-bundle-v2.27.0 --output --workspace --log INFO -p linux64 ``` -Linux ARM64 binaries are not included in the platform-agnostic upstream bundle. -Create a Linux ARM64 custom bundle on a Linux ARM64 host: +On a Linux ARM64 host, use the native target: ```bash codeql-bundle --bundle codeql-bundle-v2.27.0 --output --workspace --log INFO -p linux-arm64 ``` +Existing local all-platform archives remain supported for creating multiple +platform-specific bundles. + ### Compilation caches The repository maintains @@ -113,11 +118,11 @@ codeql-bundle-cache verify-all \ --assets-dir dist ``` -`plan-release` validates upstream release metadata and records every source -asset. `build` compiles the real standard query packs into a per-language cache -bounded to 1536 MiB and requires a second real compilation to report a cache -hit. `verify-all` repeats that check using the current platform's upstream -bundle. +`plan-release` validates upstream release metadata and records the supported +platform-specific source assets. `build` compiles the real standard query packs +into a per-language cache bounded to 1536 MiB and requires a second real +compilation to report a cache hit. `verify-all` repeats that check using the +current platform's upstream bundle. After release assets have been published, `catalog-entry`, `verify-entry`, and `update-catalog` create, download-test, and insert the candidate catalog entry. diff --git a/codeql_bundle/cache.py b/codeql_bundle/cache.py index ee001bc..caca2df 100644 --- a/codeql_bundle/cache.py +++ b/codeql_bundle/cache.py @@ -36,6 +36,7 @@ CACHE_FORMAT_VERSION = 1 DOWNLOAD_CHUNK_SIZE = 1024 * 1024 CATALOG_REFRESH_SECONDS = 60 * 60 +BUNDLE_PLATFORMS = ("linux64", "linux-arm64", "osx64", "win64") RELEASE_PATTERN = re.compile(r"^codeql-bundle-v\d+\.\d+\.\d+$") CACHE_RELEASE_PATTERN = re.compile( r"^codeql-compilation-cache-v\d+\.\d+\.\d+(?:-[A-Za-z0-9._-]+)?$" @@ -731,8 +732,6 @@ def _resolve_release( ) if bundle: asset = bundle.source_asset_for_platform(platform_name) - if asset is None and platform_name != "all": - asset = bundle.source_asset_for_platform("all") if asset is None: raise CatalogException( f"Bundle {release} has no source asset for {platform_name}." @@ -1019,21 +1018,22 @@ def write_json(path: Path, value: dict[str, Any]) -> None: def source_asset_name(platform_name: str) -> str: if platform_name == "all": return "codeql-bundle.tar.gz" - if platform_name not in {"linux64", "linux-arm64", "osx64", "win64"}: + if platform_name not in BUNDLE_PLATFORMS: raise CacheException(f"Unsupported bundle platform: {platform_name}") return f"codeql-bundle-{platform_name}.tar.gz" def source_platform_for_request(requested_platforms: Iterable[str]) -> str: - requested = tuple(requested_platforms) + requested = set(requested_platforms) current = current_bundle_platform() - if current == "linux-arm64": - return current - if not requested: - return "all" - if requested == (current,): - return current - return "all" + if requested and requested != {current}: + raise CacheException( + "Release tags can only build for the current platform " + f"({current}); requested {', '.join(sorted(requested))}. " + "Run codeql-bundle on each target platform or provide a local " + "bundle containing every requested platform." + ) + return current def github_asset_digest(asset: dict[str, Any]) -> Optional[str]: diff --git a/codeql_bundle/cache_cli.py b/codeql_bundle/cache_cli.py index 5b92983..c5d1395 100644 --- a/codeql_bundle/cache_cli.py +++ b/codeql_bundle/cache_cli.py @@ -14,6 +14,7 @@ from semantic_version import Version from codeql_bundle.cache import ( + BUNDLE_PLATFORMS, CACHE_FORMAT_VERSION, CODEQL_ACTION_REPOSITORY, BundleCatalog, @@ -50,7 +51,8 @@ logger = logging.getLogger(__name__) -SOURCE_PLATFORMS = ("all", "linux64", "osx64", "win64") +# Linux ARM64 compilation-cache publication remains out of scope. +SOURCE_PLATFORMS = ("linux64", "osx64", "win64") MAX_RELEASE_ASSET_SIZE = 2 * 1024 * 1024 * 1024 DEFAULT_COMPILATION_CACHE_SIZE_MB = 1536 @@ -142,7 +144,7 @@ def prune_cache(cache_dir: Path, max_age_days: float, dry_run: bool) -> None: "--platform", "platforms", multiple=True, - type=click.Choice(["linux64", "osx64", "win64"]), + type=click.Choice(BUNDLE_PLATFORMS), ) @click.option( "--cache-dir", diff --git a/codeql_bundle/cli.py b/codeql_bundle/cli.py index 542121d..e418741 100644 --- a/codeql_bundle/cli.py +++ b/codeql_bundle/cli.py @@ -12,6 +12,7 @@ from codeql_bundle.helpers.codeql import CodeQLException from codeql_bundle.helpers.bundle import CustomBundle, BundleException, BundlePlatform from codeql_bundle.cache import ( + BUNDLE_PLATFORMS, BundleCatalog, BundleSourceResolver, CacheException, @@ -66,7 +67,7 @@ "--platform", multiple=True, type=click.Choice( - ["linux64", "linux-arm64", "osx64", "win64"], + BUNDLE_PLATFORMS, case_sensitive=False, ), help="Target platform for the bundle", diff --git a/codeql_bundle/helpers/bundle.py b/codeql_bundle/helpers/bundle.py index bdce99d..28c565b 100644 --- a/codeql_bundle/helpers/bundle.py +++ b/codeql_bundle/helpers/bundle.py @@ -792,28 +792,32 @@ def is_unsafe_path(basedir: Path, path: Path) -> bool: config = load_config(config_path) - if platform.system() == "Windows": - keytool = "tools/win64/java/bin/keytool.exe" - elif platform.system() == "Linux": - keytool = "tools/linux64/java/bin/keytool" - elif platform.system() == "Darwin": - keytool = "tools/osx64/java/bin/keytool" - else: - raise BundleException(f"Unsupported platform {platform.system()}") - - keytool = self.bundle_path / keytool - if not keytool.exists(): - raise BundleException(f"Keytool {keytool} does not exist.") - - keystores: list[str] = [ - "tools/win64/java/lib/security/cacerts", - "tools/linux64/java/lib/security/cacerts", - "tools/osx64/java/lib/security/cacerts", - "tools/osx64/java-aarch64/lib/security/cacerts", - ] - # Add the certificates to the Java keystores if "CodeQLBundleAdditionalCertificates" in config: + keytool_paths = { + "linux64": "tools/linux64/java/bin/keytool", + "linux-arm64": "tools/linux-arm64/java/bin/keytool", + "osx64": "tools/osx64/java/bin/keytool", + "win64": "tools/win64/java/bin/keytool.exe", + } + keytool = self.bundle_path / keytool_paths[current_bundle_platform()] + if not keytool.exists(): + raise BundleException(f"Keytool {keytool} does not exist.") + + keystores = [ + self.bundle_path / path + for path in [ + "tools/win64/java/lib/security/cacerts", + "tools/linux64/java/lib/security/cacerts", + "tools/linux-arm64/java/lib/security/cacerts", + "tools/osx64/java/lib/security/cacerts", + "tools/osx64/java-aarch64/lib/security/cacerts", + ] + if (self.bundle_path / path).exists() + ] + if not keystores: + raise BundleException("The bundle contains no Java keystores.") + for cert in config["CodeQLBundleAdditionalCertificates"]: src = workspace_path / Path(cert["Source"]) src = src.resolve() @@ -825,9 +829,6 @@ def is_unsafe_path(basedir: Path, path: Path) -> bool: raise BundleException(f"Certificate file {src} does not exist.") for keystore in keystores: - keystore = self.bundle_path / keystore - if not keystore.exists(): - raise BundleException(f"Keystore {keystore} does not exist.") logging.info(f"Adding certificate {src} to keystore {keystore}") subprocess.run( [ diff --git a/tests/test_bundle.py b/tests/test_bundle.py new file mode 100644 index 0000000..72810ec --- /dev/null +++ b/tests/test_bundle.py @@ -0,0 +1,58 @@ +from pathlib import Path +from tempfile import TemporaryDirectory +import json +import unittest +from unittest.mock import patch + +from codeql_bundle.helpers.bundle import CustomBundle + + +class AdditionalDataTests(unittest.TestCase): + def test_linux_arm64_certificate_uses_native_java_tools(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + workspace = root / "workspace" + workspace.mkdir() + certificate = workspace / "certificate.pem" + certificate.write_text("certificate") + config = workspace / "additional-data.json" + config.write_text( + json.dumps( + { + "CodeQLBundleAdditionalCertificates": [ + {"Source": certificate.name} + ] + } + ) + ) + + bundle = object.__new__(CustomBundle) + bundle.tmp_dir = None + bundle.bundle_path = root / "bundle" + keytool = ( + bundle.bundle_path + / "tools/linux-arm64/java/bin/keytool" + ) + keystore = ( + bundle.bundle_path + / "tools/linux-arm64/java/lib/security/cacerts" + ) + keytool.parent.mkdir(parents=True) + keytool.touch() + keystore.parent.mkdir(parents=True) + keystore.touch() + + with patch( + "codeql_bundle.helpers.bundle.current_bundle_platform", + return_value="linux-arm64", + ), patch("codeql_bundle.helpers.bundle.subprocess.run") as run: + bundle.add_files_and_certs(config, workspace) + + run.assert_called_once() + command = run.call_args.args[0] + self.assertEqual(str(keytool), command[0]) + self.assertEqual(str(keystore), command[command.index("-keystore") + 1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cache.py b/tests/test_cache.py index 89d837d..53270fb 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -13,6 +13,7 @@ from semantic_version import Version from codeql_bundle.cache import ( + BUNDLE_PLATFORMS, CACHE_FORMAT_VERSION, BundleCatalog, BundleSourceResolver, @@ -231,19 +232,18 @@ def test_empty_catalog_does_not_hash_local_archive(self) -> None: def test_release_source_is_runnable_on_current_platform(self) -> None: current = current_bundle_platform() other = next( - platform - for platform in ("linux64", "osx64", "win64") - if platform != current + platform for platform in BUNDLE_PLATFORMS if platform != current ) - if current == "linux-arm64": - self.assertEqual(current, source_platform_for_request(())) - return - self.assertEqual("all", source_platform_for_request(())) + self.assertEqual(current, source_platform_for_request(())) self.assertEqual(current, source_platform_for_request((current,))) - self.assertEqual("all", source_platform_for_request((other,))) - self.assertEqual( - "all", source_platform_for_request(("linux64", "win64")) - ) + with self.assertRaisesRegex( + CacheException, "only build for the current platform" + ): + source_platform_for_request((other,)) + with self.assertRaisesRegex( + CacheException, "only build for the current platform" + ): + source_platform_for_request((current, other)) def test_linux_arm64_platform(self) -> None: with patch( @@ -367,10 +367,7 @@ def release(self, tag: str) -> dict[str, object]: BundleCatalog([]), root / "downloads", release_client=ReleaseClient(), - ).resolve( - "codeql-bundle-v1.2.3", - [current_bundle_platform()], - ) + ).resolve("codeql-bundle-v1.2.3") self.assertEqual(sha256_file(archive), resolved.digest) diff --git a/tests/test_cache_cli.py b/tests/test_cache_cli.py index 9bbe664..bfde90e 100644 --- a/tests/test_cache_cli.py +++ b/tests/test_cache_cli.py @@ -85,7 +85,6 @@ def test_catalog_entry_can_be_added_to_catalog(self) -> None: } for index, (platform, name) in enumerate( [ - ("all", "codeql-bundle.tar.gz"), ("linux64", "codeql-bundle-linux64.tar.gz"), ("osx64", "codeql-bundle-osx64.tar.gz"), ("win64", "codeql-bundle-win64.tar.gz"), From 2e5daa8a2a66719046c0306ee8d90e070645cf05 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 25 Sep 2026 12:35:10 +0200 Subject: [PATCH 4/9] Publish compilation caches for Linux ARM64 Discover native ARM64 assets when available, verify the shared caches on an ARM64 runner, and include the platform in generated catalog entries while preserving older release backfills. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../workflows/codeql-compilation-caches.yml | 34 +++-- README.md | 7 +- codeql_bundle/cache.py | 7 +- codeql_bundle/cache_cli.py | 30 +++-- codeql_bundle/cli.py | 5 +- codeql_bundle/helpers/bundle.py | 73 ++++------- .../supported-codeql-bundles.schema.json | 2 + tests/test_cache.py | 32 +++++ tests/test_cache_cli.py | 117 ++++++++++++------ 9 files changed, 188 insertions(+), 119 deletions(-) diff --git a/.github/workflows/codeql-compilation-caches.yml b/.github/workflows/codeql-compilation-caches.yml index 101a950..5cc1f9a 100644 --- a/.github/workflows/codeql-compilation-caches.yml +++ b/.github/workflows/codeql-compilation-caches.yml @@ -45,6 +45,7 @@ jobs: matrix: ${{ steps.plan.outputs.matrix || '[{"language":"skip","target":"skip"}]' }} release: ${{ steps.version.outputs.release }} skip: ${{ steps.existing.outputs.skip }} + verify_matrix: ${{ steps.plan.outputs.verify_matrix || '[{"platform":"skip","os":"ubuntu-latest"}]' }} steps: - uses: actions/checkout@v4 @@ -115,7 +116,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.CODEQL_BUNDLE_CACHE_DIR }}/sources/${{ steps.version.outputs.release }} - key: codeql-source-${{ steps.version.outputs.release }}-${{ runner.os }} + key: codeql-source-${{ steps.version.outputs.release }}-${{ runner.os }}-${{ runner.arch }} - if: steps.existing.outputs.skip != 'true' id: plan @@ -130,6 +131,19 @@ jobs: --output release-plan.json echo "cache_release=$(jq -r '.cache_release' release-plan.json)" >> "$GITHUB_OUTPUT" echo "matrix=$(jq -c '.targets' release-plan.json)" >> "$GITHUB_OUTPUT" + verify_matrix="$(jq -c ' + [.source_assets[].platform as $platform | + { + platform: $platform, + os: ({ + "linux64": "ubuntu-latest", + "linux-arm64": "ubuntu-24.04-arm", + "osx64": "macos-latest", + "win64": "windows-latest" + }[$platform]) + } + ]' release-plan.json)" + echo "verify_matrix=$verify_matrix" >> "$GITHUB_OUTPUT" - if: steps.existing.outputs.skip != 'true' uses: actions/upload-artifact@v4 @@ -162,7 +176,7 @@ jobs: - uses: actions/cache@v4 with: path: ${{ env.CODEQL_BUNDLE_CACHE_DIR }}/sources/${{ needs.plan.outputs.release }} - key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }} + key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }}-${{ runner.arch }} - name: Build and verify cache run: | @@ -189,10 +203,7 @@ jobs: strategy: fail-fast: false matrix: - os: - - ubuntu-latest - - macos-latest - - windows-latest + include: ${{ fromJSON(needs.plan.outputs.verify_matrix) }} runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -216,7 +227,7 @@ jobs: - uses: actions/cache@v4 with: path: ${{ env.CODEQL_BUNDLE_CACHE_DIR }}/sources/${{ needs.plan.outputs.release }} - key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }} + key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }}-${{ runner.arch }} - name: Verify all caches run: codeql-bundle-cache verify-all --plan release-plan.json --assets-dir dist @@ -299,13 +310,16 @@ jobs: - name: Create candidate catalog entry if: steps.recheck.outputs.skip != 'true' + shell: bash run: | + validated_platform_args=() + while IFS= read -r platform; do + validated_platform_args+=(--validated-platform "$platform") + done < <(jq -r '.source_assets[].platform' release-plan.json) codeql-bundle-cache catalog-entry \ --plan release-plan.json \ --assets-dir dist \ - --validated-platform linux64 \ - --validated-platform osx64 \ - --validated-platform win64 \ + "${validated_platform_args[@]}" \ --output catalog-entry.json codeql-bundle-cache verify-entry \ --entry catalog-entry.json \ diff --git a/README.md b/README.md index 2d19705..6b4b648 100644 --- a/README.md +++ b/README.md @@ -134,9 +134,10 @@ The [`Build CodeQL compilation caches`](.github/workflows/codeql-compilation-caches.yml) workflow polls for the latest stable upstream release and runs immediately when the cache implementation lands on `main`. It uses the local commands above, -parallelizes cache construction, validates every cache on Linux, macOS, and -Windows, publishes a dedicated release, performs a consumer-path customization -test, and opens a pull request for manual review of the catalog update. +parallelizes cache construction, validates every cache on each platform +published by the upstream release, publishes a dedicated release, performs a +consumer-path customization test, and opens a pull request for manual review of +the catalog update. Use `workflow_dispatch` with `bundle_version` to backfill a specific release. The latest stable release is independent of backfill work; releases from diff --git a/codeql_bundle/cache.py b/codeql_bundle/cache.py index caca2df..ebf875b 100644 --- a/codeql_bundle/cache.py +++ b/codeql_bundle/cache.py @@ -724,12 +724,7 @@ def _resolve_release( ) -> ResolvedBundleSource: platform_name = source_platform_for_request(requested_platforms) validate_release(release) - # ponytail: ARM64 has no compilation-cache catalog; fetch its release asset. - bundle = ( - None - if platform_name == "linux-arm64" - else self.catalog.find_release(release) - ) + bundle = self.catalog.find_release(release) if bundle: asset = bundle.source_asset_for_platform(platform_name) if asset is None: diff --git a/codeql_bundle/cache_cli.py b/codeql_bundle/cache_cli.py index c5d1395..0fdda4d 100644 --- a/codeql_bundle/cache_cli.py +++ b/codeql_bundle/cache_cli.py @@ -51,8 +51,7 @@ logger = logging.getLogger(__name__) -# Linux ARM64 compilation-cache publication remains out of scope. -SOURCE_PLATFORMS = ("linux64", "osx64", "win64") +REQUIRED_SOURCE_PLATFORMS = frozenset(("linux64", "osx64", "win64")) MAX_RELEASE_ASSET_SIZE = 2 * 1024 * 1024 * 1024 DEFAULT_COMPILATION_CACHE_SIZE_MB = 1536 @@ -214,11 +213,19 @@ def plan_release( client = GitHubReleaseClient() release_value = client.release(release) source_assets = _release_source_assets(release_value) + platform_name = current_bundle_platform() source_asset = next( - asset - for asset in source_assets - if asset.platform == current_bundle_platform() + ( + asset + for asset in source_assets + if asset.platform == platform_name + ), + None, ) + if source_asset is None: + raise click.ClickException( + f"Release {release} has no source bundle for {platform_name}." + ) source_path = cache_path(cache_dir, "sources", release, source_asset.name) download_file( source_asset.url, @@ -495,7 +502,7 @@ def _verify_cache_with_bundle( "validated_platforms", multiple=True, required=True, - type=click.Choice(["linux64", "osx64", "win64"]), + type=click.Choice(BUNDLE_PLATFORMS), ) @click.option( "--output", @@ -583,10 +590,12 @@ def update_catalog( def _release_source_assets(release: dict[str, Any]) -> tuple[SourceAsset, ...]: assets = {asset["name"]: asset for asset in release.get("assets", [])} result = [] - for platform_name in SOURCE_PLATFORMS: + for platform_name in BUNDLE_PLATFORMS: name = source_asset_name(platform_name) asset = assets.get(name) if asset is None: + if platform_name not in REQUIRED_SOURCE_PLATFORMS: + continue raise click.ClickException( f"Upstream release {release['tag_name']} has no {name}." ) @@ -802,10 +811,11 @@ def _read_plan(path: Path) -> dict[str, Any]: validate_remote_url(source.url) source_platforms.add(source.platform) if ( - source_platforms != set(SOURCE_PLATFORMS) - or len(value["source_assets"]) != len(SOURCE_PLATFORMS) + not REQUIRED_SOURCE_PLATFORMS.issubset(source_platforms) + or not source_platforms.issubset(BUNDLE_PLATFORMS) + or len(value["source_assets"]) != len(source_platforms) ): - raise ValueError("incomplete source platform inventory") + raise ValueError("invalid source platform inventory") target_names = set() languages = set() diff --git a/codeql_bundle/cli.py b/codeql_bundle/cli.py index e418741..5c03a76 100644 --- a/codeql_bundle/cli.py +++ b/codeql_bundle/cli.py @@ -66,10 +66,7 @@ "-p", "--platform", multiple=True, - type=click.Choice( - BUNDLE_PLATFORMS, - case_sensitive=False, - ), + type=click.Choice(BUNDLE_PLATFORMS, case_sensitive=False), help="Target platform for the bundle", ) @click.option( diff --git a/codeql_bundle/helpers/bundle.py b/codeql_bundle/helpers/bundle.py index 28c565b..4869bd8 100644 --- a/codeql_bundle/helpers/bundle.py +++ b/codeql_bundle/helpers/bundle.py @@ -248,36 +248,17 @@ def __init__(self, bundle_path: Path) -> None: else: raise BundleException("Invalid CodeQL bundle path") - def supports_linux() -> set[BundlePlatform]: - if (self.bundle_path / "cpp" / "tools" / "linux64").exists(): - return {BundlePlatform.LINUX} - else: - return set() - - def supports_linux_arm64() -> set[BundlePlatform]: - if (self.bundle_path / "cpp" / "tools" / "linux-arm64").exists(): - return {BundlePlatform.LINUX_ARM64} - else: - return set() - - def supports_macos() -> set[BundlePlatform]: - if (self.bundle_path / "cpp" / "tools" / "osx64").exists(): - return {BundlePlatform.OSX} - else: - return set() - - def supports_windows() -> set[BundlePlatform]: - if (self.bundle_path / "cpp" / "tools" / "win64").exists(): - return {BundlePlatform.WINDOWS} - else: - return set() - - self.platforms: set[BundlePlatform] = ( - supports_linux() - | supports_linux_arm64() - | supports_macos() - | supports_windows() - ) + platform_tools = { + BundlePlatform.LINUX: "linux64", + BundlePlatform.LINUX_ARM64: "linux-arm64", + BundlePlatform.OSX: "osx64", + BundlePlatform.WINDOWS: "win64", + } + self.platforms = { + bundle_platform + for bundle_platform, tools in platform_tools.items() + if (self.bundle_path / "cpp" / "tools" / tools).exists() + } current_platform = BundlePlatform.from_string(current_bundle_platform()) if current_platform not in self.platforms: @@ -794,27 +775,23 @@ def is_unsafe_path(basedir: Path, path: Path) -> bool: # Add the certificates to the Java keystores if "CodeQLBundleAdditionalCertificates" in config: - keytool_paths = { - "linux64": "tools/linux64/java/bin/keytool", - "linux-arm64": "tools/linux-arm64/java/bin/keytool", - "osx64": "tools/osx64/java/bin/keytool", - "win64": "tools/win64/java/bin/keytool.exe", - } - keytool = self.bundle_path / keytool_paths[current_bundle_platform()] + platform_name = current_bundle_platform() + keytool = ( + self.bundle_path + / "tools" + / platform_name + / "java" + / "bin" + / ("keytool.exe" if platform_name == "win64" else "keytool") + ) if not keytool.exists(): raise BundleException(f"Keytool {keytool} does not exist.") - keystores = [ - self.bundle_path / path - for path in [ - "tools/win64/java/lib/security/cacerts", - "tools/linux64/java/lib/security/cacerts", - "tools/linux-arm64/java/lib/security/cacerts", - "tools/osx64/java/lib/security/cacerts", - "tools/osx64/java-aarch64/lib/security/cacerts", - ] - if (self.bundle_path / path).exists() - ] + keystores = list( + (self.bundle_path / "tools").glob( + "*/java*/lib/security/cacerts" + ) + ) if not keystores: raise BundleException("The bundle contains no Java keystores.") diff --git a/codeql_bundle/supported-codeql-bundles.schema.json b/codeql_bundle/supported-codeql-bundles.schema.json index d69afd4..af9458c 100644 --- a/codeql_bundle/supported-codeql-bundles.schema.json +++ b/codeql_bundle/supported-codeql-bundles.schema.json @@ -84,6 +84,7 @@ "enum": [ "all", "linux64", + "linux-arm64", "osx64", "win64" ] @@ -177,6 +178,7 @@ "items": { "enum": [ "linux64", + "linux-arm64", "osx64", "win64" ] diff --git a/tests/test_cache.py b/tests/test_cache.py index 53270fb..e37a0df 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -262,6 +262,38 @@ def test_linux_arm64_platform(self) -> None: str(BundlePlatform.from_string("linux-arm64")), ) + def test_linux_arm64_release_uses_catalog_asset(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + served = root / "served" + served.mkdir() + archive = served / "codeql-bundle-linux-arm64.tar.gz" + archive.write_bytes(b"bundle") + with serve(served) as base_url, patch( + "codeql_bundle.cache.current_bundle_platform", + return_value="linux-arm64", + ): + source = SourceAsset( + name=archive.name, + url=f"{base_url}/{archive.name}", + sha256=sha256_file(archive), + size=archive.stat().st_size, + platform="linux-arm64", + ) + cache = ReleaseAsset( + name="cache.tar.gz", + url=f"{base_url}/cache.tar.gz", + sha256="2" * 64, + size=10, + ) + bundle = bundle_with_assets(source, cache) + resolved = BundleSourceResolver( + BundleCatalog([bundle]), root / "downloads" + ).resolve(bundle.release) + + self.assertEqual(bundle, resolved.supported_bundle) + self.assertEqual(1, CountingHandler.requests) + def test_local_archive_matches_catalog_digest(self) -> None: with TemporaryDirectory() as directory: root = Path(directory) diff --git a/tests/test_cache_cli.py b/tests/test_cache_cli.py index bfde90e..52c39bc 100644 --- a/tests/test_cache_cli.py +++ b/tests/test_cache_cli.py @@ -6,8 +6,36 @@ from click.testing import CliRunner -from codeql_bundle.cache import CatalogLoader -from codeql_bundle.cache_cli import main +from codeql_bundle.cache import BUNDLE_PLATFORMS, CatalogLoader, source_asset_name +from codeql_bundle.cache_cli import _read_plan, _release_source_assets, main + + +def _release_plan(platforms: tuple[str, ...]) -> dict[str, object]: + return { + "cache_format": 1, + "cache_release": "codeql-compilation-cache-v1.2.3", + "cli_version": "1.2.3", + "pack_fingerprint": "0" * 64, + "release": "codeql-bundle-v1.2.3", + "source_assets": [ + { + "name": source_asset_name(platform), + "platform": platform, + "sha256": f"{index:x}" * 64, + "size": 100, + "url": f"https://example.test/{source_asset_name(platform)}", + } + for index, platform in enumerate(platforms, start=1) + ], + "source_repository": "github/codeql-action", + "targets": [ + { + "language": "cpp", + "query_packs": ["codeql/cpp-queries@1.2.3"], + "target": "codeql/cpp-all", + } + ], + } class CacheCliTests(unittest.TestCase): @@ -61,7 +89,50 @@ def test_build_rejects_invalid_release_plan(self) -> None: self.assertNotEqual(0, result.exit_code) self.assertIn("Invalid release plan", result.output) - def test_catalog_entry_can_be_added_to_catalog(self) -> None: + def test_release_source_assets_include_arm64_when_available(self) -> None: + for platforms in ( + ("linux64", "osx64", "win64"), + BUNDLE_PLATFORMS, + ): + with self.subTest(platforms=platforms): + release = { + "tag_name": "codeql-bundle-v1.2.3", + "assets": [ + { + "browser_download_url": f"https://example.test/{source_asset_name(platform)}", + "digest": f"sha256:{index:064x}", + "name": source_asset_name(platform), + "size": 100, + } + for index, platform in enumerate(platforms, start=1) + ], + } + + assets = _release_source_assets(release) + + self.assertEqual( + platforms, tuple(asset.platform for asset in assets) + ) + + def test_release_plans_support_legacy_and_arm64_sources(self) -> None: + runner = CliRunner() + with runner.isolated_filesystem(): + plan_path = Path("plan.json") + for platforms in ( + ("linux64", "osx64", "win64"), + BUNDLE_PLATFORMS, + ): + with self.subTest(platforms=platforms): + plan_path.write_text(json.dumps(_release_plan(platforms))) + self.assertEqual( + platforms, + tuple( + asset["platform"] + for asset in _read_plan(plan_path)["source_assets"] + ), + ) + + def test_arm64_catalog_entry_can_be_added_to_catalog(self) -> None: runner = CliRunner() with runner.isolated_filesystem(): root = Path.cwd() @@ -69,45 +140,13 @@ def test_catalog_entry_can_be_added_to_catalog(self) -> None: assets.mkdir() cache_asset = assets / "codeql-compilation-cache-cpp.tar.gz" cache_asset.write_bytes(b"cache") - plan = { - "cache_format": 1, - "cache_release": "codeql-compilation-cache-v1.2.3", - "cli_version": "1.2.3", - "pack_fingerprint": "0" * 64, - "release": "codeql-bundle-v1.2.3", - "source_assets": [ - { - "name": name, - "platform": platform, - "sha256": str(index) * 64, - "size": 100, - "url": f"https://example.test/{name}", - } - for index, (platform, name) in enumerate( - [ - ("linux64", "codeql-bundle-linux64.tar.gz"), - ("osx64", "codeql-bundle-osx64.tar.gz"), - ("win64", "codeql-bundle-win64.tar.gz"), - ], - start=1, - ) - ], - "source_repository": "github/codeql-action", - "targets": [ - { - "language": "cpp", - "query_packs": ["codeql/cpp-queries@1.2.3"], - "target": "codeql/cpp-all", - } - ], - } plan_path = root / "plan.json" - plan_path.write_text(json.dumps(plan)) catalog_path = root / "catalog.json" catalog_path.write_text( json.dumps({"schema_version": 1, "bundles": []}) ) entry_path = root / "entry.json" + plan_path.write_text(json.dumps(_release_plan(BUNDLE_PLATFORMS))) result = runner.invoke( main, @@ -118,7 +157,7 @@ def test_catalog_entry_can_be_added_to_catalog(self) -> None: "--assets-dir", str(assets), "--validated-platform", - "linux64", + "linux-arm64", "--output", str(entry_path), ], @@ -138,7 +177,9 @@ def test_catalog_entry_can_be_added_to_catalog(self) -> None: self.assertEqual(0, result.exit_code, result.output) catalog = CatalogLoader().load(str(catalog_path)) - self.assertIsNotNone(catalog.find_release("codeql-bundle-v1.2.3")) + bundle = catalog.find_release("codeql-bundle-v1.2.3") + self.assertIsNotNone(bundle) + self.assertEqual(("linux-arm64",), bundle.validated_platforms) if __name__ == "__main__": From 838696b5f15ba00216bf3a9759a22fc620d2e176 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 25 Sep 2026 13:00:40 +0200 Subject: [PATCH 5/9] Address ARM64 review findings Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- codeql_bundle/cache_cli.py | 12 +++++++++ codeql_bundle/helpers/bundle.py | 45 +++++++++++++-------------------- tests/test_bundle.py | 44 +++++++++++++++++++++++++++++++- tests/test_cache_cli.py | 34 +++++++++++++++++++++++++ 4 files changed, 107 insertions(+), 28 deletions(-) diff --git a/codeql_bundle/cache_cli.py b/codeql_bundle/cache_cli.py index 0fdda4d..03c42be 100644 --- a/codeql_bundle/cache_cli.py +++ b/codeql_bundle/cache_cli.py @@ -518,6 +518,18 @@ def catalog_entry( ) -> None: """Create a catalog entry for verified release assets.""" plan = _read_plan(plan_path) + source_platforms = { + source_asset["platform"] for source_asset in plan["source_assets"] + } + missing_source_platforms = sorted( + set(validated_platforms) - source_platforms + ) + if missing_source_platforms: + raise click.ClickException( + "Release plan has no source bundle for validated platform(s): " + f"{', '.join(missing_source_platforms)}." + ) + compilation_caches = {} for target in plan["targets"]: asset_name = _cache_asset_name(target["language"]) diff --git a/codeql_bundle/helpers/bundle.py b/codeql_bundle/helpers/bundle.py index 4869bd8..b3b9ae3 100644 --- a/codeql_bundle/helpers/bundle.py +++ b/codeql_bundle/helpers/bundle.py @@ -902,36 +902,27 @@ def get_nonplatform_tool_paths( platform: BundlePlatform, ) -> List[Path]: """Get a list of paths to tools that are not for the specified platform relative to the root of a bundle.""" - specialize_path: Optional[Callable[[Path], List[Path]]] = None - linux64_subpaths = [Path("linux64"), Path("linux")] - osx64_subpaths = [Path("osx64"), Path("macos")] - win64_subpaths = [Path("win64"), Path("windows")] - # ponytail: ARM64 is only shipped in its own source bundle. - if platform in { - BundlePlatform.LINUX, - BundlePlatform.LINUX_ARM64, - }: - specialize_path = lambda p: [ - p / subpath - for subpath in osx64_subpaths + win64_subpaths - ] - elif platform == BundlePlatform.WINDOWS: - specialize_path = lambda p: [ - p / subpath - for subpath in osx64_subpaths + linux64_subpaths - ] - elif platform == BundlePlatform.OSX: - specialize_path = lambda p: [ - p / subpath - for subpath in linux64_subpaths + win64_subpaths - ] - else: + platform_subpaths = { + BundlePlatform.LINUX: [ + Path("linux64"), + Path("linux"), + ], + BundlePlatform.LINUX_ARM64: [Path("linux-arm64")], + BundlePlatform.OSX: [Path("osx64"), Path("macos")], + BundlePlatform.WINDOWS: [ + Path("win64"), + Path("windows"), + ], + } + if platform not in platform_subpaths: raise BundleException(f"Unsupported platform {platform}.") return [ - candidate - for candidates in map(specialize_path, relative_tools_paths) - for candidate in candidates + tools_path / subpath + for tools_path in relative_tools_paths + for candidate_platform, subpaths in platform_subpaths.items() + if candidate_platform != platform + for subpath in subpaths ] def filter(tarinfo: tarfile.TarInfo) -> Optional[tarfile.TarInfo]: diff --git a/tests/test_bundle.py b/tests/test_bundle.py index 72810ec..b68e743 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -1,10 +1,11 @@ from pathlib import Path from tempfile import TemporaryDirectory import json +import tarfile import unittest from unittest.mock import patch -from codeql_bundle.helpers.bundle import CustomBundle +from codeql_bundle.helpers.bundle import BundlePlatform, CustomBundle class AdditionalDataTests(unittest.TestCase): @@ -54,5 +55,46 @@ def test_linux_arm64_certificate_uses_native_java_tools(self) -> None: self.assertEqual(str(keystore), command[command.index("-keystore") + 1]) +class PlatformBundleTests(unittest.TestCase): + def test_platform_archives_exclude_other_platform_tools(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + bundle = object.__new__(CustomBundle) + bundle.tmp_dir = None + bundle.bundle_path = root / "bundle" + bundle.languages = [] + platform_tools = { + BundlePlatform.LINUX: "linux64", + BundlePlatform.LINUX_ARM64: "linux-arm64", + BundlePlatform.OSX: "osx64", + BundlePlatform.WINDOWS: "win64", + } + bundle.platforms = set(platform_tools) + for tool_path in platform_tools.values(): + tool = bundle.bundle_path / "tools" / tool_path / "tool" + tool.parent.mkdir(parents=True) + tool.write_text(tool_path) + + output = root / "output" + output.mkdir() + bundle.bundle(output, set(platform_tools)) + + for platform, tool_path in platform_tools.items(): + with self.subTest(platform=platform), tarfile.open( + output / f"codeql-bundle-{platform}.tar.gz" + ) as archive: + archived_paths = set(archive.getnames()) + self.assertIn( + f"codeql/tools/{tool_path}/tool", archived_paths + ) + for other_tool_path in ( + set(platform_tools.values()) - {tool_path} + ): + self.assertNotIn( + f"codeql/tools/{other_tool_path}/tool", + archived_paths, + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_cache_cli.py b/tests/test_cache_cli.py index 52c39bc..ba6b374 100644 --- a/tests/test_cache_cli.py +++ b/tests/test_cache_cli.py @@ -181,6 +181,40 @@ def test_arm64_catalog_entry_can_be_added_to_catalog(self) -> None: self.assertIsNotNone(bundle) self.assertEqual(("linux-arm64",), bundle.validated_platforms) + def test_catalog_entry_rejects_platform_missing_from_plan(self) -> None: + runner = CliRunner() + with runner.isolated_filesystem(): + assets = Path("assets") + assets.mkdir() + plan_path = Path("plan.json") + plan_path.write_text( + json.dumps( + _release_plan(("linux64", "osx64", "win64")) + ) + ) + + result = runner.invoke( + main, + [ + "catalog-entry", + "--plan", + str(plan_path), + "--assets-dir", + str(assets), + "--validated-platform", + "linux-arm64", + "--output", + "entry.json", + ], + ) + + self.assertNotEqual(0, result.exit_code) + self.assertIn( + "Release plan has no source bundle for validated platform(s): " + "linux-arm64.", + result.output, + ) + if __name__ == "__main__": unittest.main() From 96d16263b40aec0c03d5f2481e1a02596e81869e Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 25 Sep 2026 13:09:32 +0200 Subject: [PATCH 6/9] Simplify Linux ARM64 support Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../workflows/codeql-compilation-caches.yml | 23 +-- README.md | 36 ++--- codeql_bundle/cache.py | 6 +- codeql_bundle/cache_cli.py | 19 +-- codeql_bundle/helpers/bundle.py | 74 +++------ tests/test_bundle.py | 95 ++++------- tests/test_cache.py | 55 +------ tests/test_cache_cli.py | 151 +++++++----------- 8 files changed, 141 insertions(+), 318 deletions(-) diff --git a/.github/workflows/codeql-compilation-caches.yml b/.github/workflows/codeql-compilation-caches.yml index 5cc1f9a..f28dfb0 100644 --- a/.github/workflows/codeql-compilation-caches.yml +++ b/.github/workflows/codeql-compilation-caches.yml @@ -45,7 +45,7 @@ jobs: matrix: ${{ steps.plan.outputs.matrix || '[{"language":"skip","target":"skip"}]' }} release: ${{ steps.version.outputs.release }} skip: ${{ steps.existing.outputs.skip }} - verify_matrix: ${{ steps.plan.outputs.verify_matrix || '[{"platform":"skip","os":"ubuntu-latest"}]' }} + verify_os: ${{ steps.plan.outputs.verify_os || '["ubuntu-latest"]' }} steps: - uses: actions/checkout@v4 @@ -131,19 +131,12 @@ jobs: --output release-plan.json echo "cache_release=$(jq -r '.cache_release' release-plan.json)" >> "$GITHUB_OUTPUT" echo "matrix=$(jq -c '.targets' release-plan.json)" >> "$GITHUB_OUTPUT" - verify_matrix="$(jq -c ' - [.source_assets[].platform as $platform | - { - platform: $platform, - os: ({ - "linux64": "ubuntu-latest", - "linux-arm64": "ubuntu-24.04-arm", - "osx64": "macos-latest", - "win64": "windows-latest" - }[$platform]) - } - ]' release-plan.json)" - echo "verify_matrix=$verify_matrix" >> "$GITHUB_OUTPUT" + echo "verify_os=$(jq -c '[.source_assets[].platform | { + "linux64": "ubuntu-latest", + "linux-arm64": "ubuntu-24.04-arm", + "osx64": "macos-latest", + "win64": "windows-latest" + }[.]]' release-plan.json)" >> "$GITHUB_OUTPUT" - if: steps.existing.outputs.skip != 'true' uses: actions/upload-artifact@v4 @@ -203,7 +196,7 @@ jobs: strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.plan.outputs.verify_matrix) }} + os: ${{ fromJSON(needs.plan.outputs.verify_os) }} runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index 6b4b648..a71dec7 100644 --- a/README.md +++ b/README.md @@ -37,25 +37,16 @@ with the command: codeql-bundle --bundle codeql-bundle-v2.26.1 --output codeql-custom-bundle.tar.gz --workspace --log INFO ``` -The upstream all-platform bundle is -[deprecated](https://github.blog/changelog/2026-09-22-deprecation-notice-all-platform-codeql-bundle/) -and will be removed in mid-March 2027. Release tags therefore use a -platform-specific source and can only build for the current platform. Run -`codeql-bundle` on each target platform. +Because the upstream all-platform bundle is +[deprecated](https://github.blog/changelog/2026-09-22-deprecation-notice-all-platform-codeql-bundle/), +release tags only build for the current target (`linux64`, `linux-arm64`, +`osx64`, or `win64`). Run once per target; local all-platform archives still +support multiple targets. ```bash codeql-bundle --bundle codeql-bundle-v2.27.0 --output --workspace --log INFO -p linux64 ``` -On a Linux ARM64 host, use the native target: - -```bash -codeql-bundle --bundle codeql-bundle-v2.27.0 --output --workspace --log INFO -p linux-arm64 -``` - -Existing local all-platform archives remain supported for creating multiple -platform-specific bundles. - ### Compilation caches The repository maintains @@ -118,11 +109,11 @@ codeql-bundle-cache verify-all \ --assets-dir dist ``` -`plan-release` validates upstream release metadata and records the supported -platform-specific source assets. `build` compiles the real standard query packs -into a per-language cache bounded to 1536 MiB and requires a second real -compilation to report a cache hit. `verify-all` repeats that check using the -current platform's upstream bundle. +`plan-release` validates upstream release metadata and records every source +asset. `build` compiles the real standard query packs into a per-language cache +bounded to 1536 MiB and requires a second real compilation to report a cache +hit. `verify-all` repeats that check using the current platform's upstream +bundle. After release assets have been published, `catalog-entry`, `verify-entry`, and `update-catalog` create, download-test, and insert the candidate catalog entry. @@ -134,10 +125,9 @@ The [`Build CodeQL compilation caches`](.github/workflows/codeql-compilation-caches.yml) workflow polls for the latest stable upstream release and runs immediately when the cache implementation lands on `main`. It uses the local commands above, -parallelizes cache construction, validates every cache on each platform -published by the upstream release, publishes a dedicated release, performs a -consumer-path customization test, and opens a pull request for manual review of -the catalog update. +parallelizes cache construction, validates every cache on Linux, macOS, and +Windows, publishes a dedicated release, performs a consumer-path customization +test, and opens a pull request for manual review of the catalog update. Use `workflow_dispatch` with `bundle_version` to backfill a specific release. The latest stable release is independent of backfill work; releases from diff --git a/codeql_bundle/cache.py b/codeql_bundle/cache.py index ebf875b..1244197 100644 --- a/codeql_bundle/cache.py +++ b/codeql_bundle/cache.py @@ -1023,10 +1023,8 @@ def source_platform_for_request(requested_platforms: Iterable[str]) -> str: current = current_bundle_platform() if requested and requested != {current}: raise CacheException( - "Release tags can only build for the current platform " - f"({current}); requested {', '.join(sorted(requested))}. " - "Run codeql-bundle on each target platform or provide a local " - "bundle containing every requested platform." + f"Release tags can only build for the current platform ({current}); " + f"requested {', '.join(sorted(requested))}." ) return current diff --git a/codeql_bundle/cache_cli.py b/codeql_bundle/cache_cli.py index 03c42be..b4f5b0b 100644 --- a/codeql_bundle/cache_cli.py +++ b/codeql_bundle/cache_cli.py @@ -215,11 +215,7 @@ def plan_release( source_assets = _release_source_assets(release_value) platform_name = current_bundle_platform() source_asset = next( - ( - asset - for asset in source_assets - if asset.platform == platform_name - ), + (asset for asset in source_assets if asset.platform == platform_name), None, ) if source_asset is None: @@ -518,16 +514,13 @@ def catalog_entry( ) -> None: """Create a catalog entry for verified release assets.""" plan = _read_plan(plan_path) - source_platforms = { - source_asset["platform"] for source_asset in plan["source_assets"] - } - missing_source_platforms = sorted( - set(validated_platforms) - source_platforms - ) - if missing_source_platforms: + if missing := sorted( + set(validated_platforms) + - {asset["platform"] for asset in plan["source_assets"]} + ): raise click.ClickException( "Release plan has no source bundle for validated platform(s): " - f"{', '.join(missing_source_platforms)}." + f"{', '.join(missing)}." ) compilation_caches = {} diff --git a/codeql_bundle/helpers/bundle.py b/codeql_bundle/helpers/bundle.py index b3b9ae3..001e30c 100644 --- a/codeql_bundle/helpers/bundle.py +++ b/codeql_bundle/helpers/bundle.py @@ -12,7 +12,7 @@ import os import subprocess from jsonschema import validate, ValidationError -from enum import Enum, verify, UNIQUE +from enum import Enum, StrEnum, verify, UNIQUE from dataclasses import dataclass from graphlib import TopologicalSorter import platform @@ -195,37 +195,19 @@ def get_compilation_cache_targets( return targets -@verify(UNIQUE) -class BundlePlatform(Enum): - LINUX = 1 - WINDOWS = 2 - OSX = 3 - LINUX_ARM64 = 4 - - @staticmethod - def from_string(platform: str) -> "BundlePlatform": - if platform.lower() == "linux" or platform.lower() == "linux64": - return BundlePlatform.LINUX - elif platform.lower() == "linux-arm64": - return BundlePlatform.LINUX_ARM64 - elif platform.lower() == "windows" or platform.lower() == "win64": - return BundlePlatform.WINDOWS - elif platform.lower() == "osx" or platform.lower() == "osx64": - return BundlePlatform.OSX - else: - raise BundleException(f"Invalid platform {platform}") - - def __str__(self): - if self == BundlePlatform.LINUX: - return "linux64" - elif self == BundlePlatform.LINUX_ARM64: - return "linux-arm64" - elif self == BundlePlatform.WINDOWS: - return "win64" - elif self == BundlePlatform.OSX: - return "osx64" - else: - raise BundleException(f"Invalid platform {self}") +class BundlePlatform(StrEnum): + LINUX = "linux64" + LINUX_ARM64 = "linux-arm64" + WINDOWS = "win64" + OSX = "osx64" + + @classmethod + def from_string(cls, platform: str) -> "BundlePlatform": + aliases = {"linux": "linux64", "windows": "win64", "osx": "osx64"} + try: + return cls(aliases.get(platform.lower(), platform.lower())) + except ValueError: + raise BundleException(f"Invalid platform {platform}") from None class Bundle: @@ -248,16 +230,10 @@ def __init__(self, bundle_path: Path) -> None: else: raise BundleException("Invalid CodeQL bundle path") - platform_tools = { - BundlePlatform.LINUX: "linux64", - BundlePlatform.LINUX_ARM64: "linux-arm64", - BundlePlatform.OSX: "osx64", - BundlePlatform.WINDOWS: "win64", - } self.platforms = { - bundle_platform - for bundle_platform, tools in platform_tools.items() - if (self.bundle_path / "cpp" / "tools" / tools).exists() + platform + for platform in BundlePlatform + if (self.bundle_path / "cpp" / "tools" / platform).exists() } current_platform = BundlePlatform.from_string(current_bundle_platform()) @@ -903,19 +879,11 @@ def get_nonplatform_tool_paths( ) -> List[Path]: """Get a list of paths to tools that are not for the specified platform relative to the root of a bundle.""" platform_subpaths = { - BundlePlatform.LINUX: [ - Path("linux64"), - Path("linux"), - ], - BundlePlatform.LINUX_ARM64: [Path("linux-arm64")], - BundlePlatform.OSX: [Path("osx64"), Path("macos")], - BundlePlatform.WINDOWS: [ - Path("win64"), - Path("windows"), - ], + BundlePlatform.LINUX: ("linux64", "linux"), + BundlePlatform.LINUX_ARM64: ("linux-arm64",), + BundlePlatform.OSX: ("osx64", "macos"), + BundlePlatform.WINDOWS: ("win64", "windows"), } - if platform not in platform_subpaths: - raise BundleException(f"Unsupported platform {platform}.") return [ tools_path / subpath diff --git a/tests/test_bundle.py b/tests/test_bundle.py index b68e743..6dd6b19 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -1,6 +1,5 @@ from pathlib import Path from tempfile import TemporaryDirectory -import json import tarfile import unittest from unittest.mock import patch @@ -8,92 +7,58 @@ from codeql_bundle.helpers.bundle import BundlePlatform, CustomBundle -class AdditionalDataTests(unittest.TestCase): - def test_linux_arm64_certificate_uses_native_java_tools(self) -> None: +class BundleTests(unittest.TestCase): + def test_linux_arm64_bundle_uses_only_native_tools(self) -> None: with TemporaryDirectory() as directory: root = Path(directory) - workspace = root / "workspace" - workspace.mkdir() - certificate = workspace / "certificate.pem" + certificate = root / "certificate.pem" certificate.write_text("certificate") - config = workspace / "additional-data.json" + config = root / "additional-data.json" config.write_text( - json.dumps( - { - "CodeQLBundleAdditionalCertificates": [ - {"Source": certificate.name} - ] - } - ) + '{"CodeQLBundleAdditionalCertificates":' + '[{"Source":"certificate.pem"}]}' ) bundle = object.__new__(CustomBundle) bundle.tmp_dir = None bundle.bundle_path = root / "bundle" - keytool = ( - bundle.bundle_path - / "tools/linux-arm64/java/bin/keytool" - ) + bundle.languages = [] + bundle.platforms = {BundlePlatform.LINUX_ARM64} + keytool = bundle.bundle_path / "tools/linux-arm64/java/bin/keytool" keystore = ( bundle.bundle_path / "tools/linux-arm64/java/lib/security/cacerts" ) - keytool.parent.mkdir(parents=True) - keytool.touch() - keystore.parent.mkdir(parents=True) - keystore.touch() + for path in (keytool, keystore): + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() with patch( "codeql_bundle.helpers.bundle.current_bundle_platform", return_value="linux-arm64", - ), patch("codeql_bundle.helpers.bundle.subprocess.run") as run: - bundle.add_files_and_certs(config, workspace) + ), patch( + "codeql_bundle.helpers.bundle.subprocess.run" + ) as run, patch( + "codeql_bundle.helpers.bundle.tarfile.open" + ) as open_archive: + bundle.add_files_and_certs(config, root) + bundle.bundle(root, {BundlePlatform.LINUX_ARM64}) run.assert_called_once() command = run.call_args.args[0] self.assertEqual(str(keytool), command[0]) self.assertEqual(str(keystore), command[command.index("-keystore") + 1]) - - -class PlatformBundleTests(unittest.TestCase): - def test_platform_archives_exclude_other_platform_tools(self) -> None: - with TemporaryDirectory() as directory: - root = Path(directory) - bundle = object.__new__(CustomBundle) - bundle.tmp_dir = None - bundle.bundle_path = root / "bundle" - bundle.languages = [] - platform_tools = { - BundlePlatform.LINUX: "linux64", - BundlePlatform.LINUX_ARM64: "linux-arm64", - BundlePlatform.OSX: "osx64", - BundlePlatform.WINDOWS: "win64", - } - bundle.platforms = set(platform_tools) - for tool_path in platform_tools.values(): - tool = bundle.bundle_path / "tools" / tool_path / "tool" - tool.parent.mkdir(parents=True) - tool.write_text(tool_path) - - output = root / "output" - output.mkdir() - bundle.bundle(output, set(platform_tools)) - - for platform, tool_path in platform_tools.items(): - with self.subTest(platform=platform), tarfile.open( - output / f"codeql-bundle-{platform}.tar.gz" - ) as archive: - archived_paths = set(archive.getnames()) - self.assertIn( - f"codeql/tools/{tool_path}/tool", archived_paths - ) - for other_tool_path in ( - set(platform_tools.values()) - {tool_path} - ): - self.assertNotIn( - f"codeql/tools/{other_tool_path}/tool", - archived_paths, - ) + archive_filter = ( + open_archive.return_value.__enter__.return_value.add.call_args.kwargs[ + "filter" + ] + ) + self.assertIsNone( + archive_filter(tarfile.TarInfo("codeql/tools/linux64/tool")) + ) + self.assertIsNotNone( + archive_filter(tarfile.TarInfo("codeql/tools/linux-arm64/tool")) + ) if __name__ == "__main__": diff --git a/tests/test_cache.py b/tests/test_cache.py index e37a0df..a63b0ba 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -13,7 +13,6 @@ from semantic_version import Version from codeql_bundle.cache import ( - BUNDLE_PLATFORMS, CACHE_FORMAT_VERSION, BundleCatalog, BundleSourceResolver, @@ -34,7 +33,6 @@ source_asset_name, source_platform_for_request, ) -from codeql_bundle.helpers.bundle import BundlePlatform from codeql_bundle.helpers.codeql import CodeQLPack, CodeQLPackConfig @@ -231,19 +229,13 @@ def test_empty_catalog_does_not_hash_local_archive(self) -> None: def test_release_source_is_runnable_on_current_platform(self) -> None: current = current_bundle_platform() - other = next( - platform for platform in BUNDLE_PLATFORMS if platform != current - ) + other = "win64" if current != "win64" else "linux64" self.assertEqual(current, source_platform_for_request(())) self.assertEqual(current, source_platform_for_request((current,))) with self.assertRaisesRegex( CacheException, "only build for the current platform" ): source_platform_for_request((other,)) - with self.assertRaisesRegex( - CacheException, "only build for the current platform" - ): - source_platform_for_request((current, other)) def test_linux_arm64_platform(self) -> None: with patch( @@ -252,47 +244,10 @@ def test_linux_arm64_platform(self) -> None: "codeql_bundle.cache.platform.machine", return_value="aarch64" ): self.assertEqual("linux-arm64", current_bundle_platform()) - self.assertEqual("linux-arm64", source_platform_for_request(())) - self.assertEqual( - "codeql-bundle-linux-arm64.tar.gz", - source_asset_name("linux-arm64"), - ) - self.assertEqual( - "linux-arm64", - str(BundlePlatform.from_string("linux-arm64")), - ) - - def test_linux_arm64_release_uses_catalog_asset(self) -> None: - with TemporaryDirectory() as directory: - root = Path(directory) - served = root / "served" - served.mkdir() - archive = served / "codeql-bundle-linux-arm64.tar.gz" - archive.write_bytes(b"bundle") - with serve(served) as base_url, patch( - "codeql_bundle.cache.current_bundle_platform", - return_value="linux-arm64", - ): - source = SourceAsset( - name=archive.name, - url=f"{base_url}/{archive.name}", - sha256=sha256_file(archive), - size=archive.stat().st_size, - platform="linux-arm64", - ) - cache = ReleaseAsset( - name="cache.tar.gz", - url=f"{base_url}/cache.tar.gz", - sha256="2" * 64, - size=10, - ) - bundle = bundle_with_assets(source, cache) - resolved = BundleSourceResolver( - BundleCatalog([bundle]), root / "downloads" - ).resolve(bundle.release) - - self.assertEqual(bundle, resolved.supported_bundle) - self.assertEqual(1, CountingHandler.requests) + self.assertEqual( + "codeql-bundle-linux-arm64.tar.gz", + source_asset_name("linux-arm64"), + ) def test_local_archive_matches_catalog_digest(self) -> None: with TemporaryDirectory() as directory: diff --git a/tests/test_cache_cli.py b/tests/test_cache_cli.py index ba6b374..2b2b23d 100644 --- a/tests/test_cache_cli.py +++ b/tests/test_cache_cli.py @@ -7,35 +7,7 @@ from click.testing import CliRunner from codeql_bundle.cache import BUNDLE_PLATFORMS, CatalogLoader, source_asset_name -from codeql_bundle.cache_cli import _read_plan, _release_source_assets, main - - -def _release_plan(platforms: tuple[str, ...]) -> dict[str, object]: - return { - "cache_format": 1, - "cache_release": "codeql-compilation-cache-v1.2.3", - "cli_version": "1.2.3", - "pack_fingerprint": "0" * 64, - "release": "codeql-bundle-v1.2.3", - "source_assets": [ - { - "name": source_asset_name(platform), - "platform": platform, - "sha256": f"{index:x}" * 64, - "size": 100, - "url": f"https://example.test/{source_asset_name(platform)}", - } - for index, platform in enumerate(platforms, start=1) - ], - "source_repository": "github/codeql-action", - "targets": [ - { - "language": "cpp", - "query_packs": ["codeql/cpp-queries@1.2.3"], - "target": "codeql/cpp-all", - } - ], - } +from codeql_bundle.cache_cli import _release_source_assets, main class CacheCliTests(unittest.TestCase): @@ -114,24 +86,6 @@ def test_release_source_assets_include_arm64_when_available(self) -> None: platforms, tuple(asset.platform for asset in assets) ) - def test_release_plans_support_legacy_and_arm64_sources(self) -> None: - runner = CliRunner() - with runner.isolated_filesystem(): - plan_path = Path("plan.json") - for platforms in ( - ("linux64", "osx64", "win64"), - BUNDLE_PLATFORMS, - ): - with self.subTest(platforms=platforms): - plan_path.write_text(json.dumps(_release_plan(platforms))) - self.assertEqual( - platforms, - tuple( - asset["platform"] - for asset in _read_plan(plan_path)["source_assets"] - ), - ) - def test_arm64_catalog_entry_can_be_added_to_catalog(self) -> None: runner = CliRunner() with runner.isolated_filesystem(): @@ -141,27 +95,53 @@ def test_arm64_catalog_entry_can_be_added_to_catalog(self) -> None: cache_asset = assets / "codeql-compilation-cache-cpp.tar.gz" cache_asset.write_bytes(b"cache") plan_path = root / "plan.json" + plan = { + "cache_format": 1, + "cache_release": "codeql-compilation-cache-v1.2.3", + "cli_version": "1.2.3", + "pack_fingerprint": "0" * 64, + "release": "codeql-bundle-v1.2.3", + "source_assets": [ + { + "name": source_asset_name(platform), + "platform": platform, + "sha256": str(index) * 64, + "size": 100, + "url": f"https://example.test/{source_asset_name(platform)}", + } + for index, platform in enumerate( + BUNDLE_PLATFORMS, + start=1, + ) + ], + "source_repository": "github/codeql-action", + "targets": [ + { + "language": "cpp", + "query_packs": ["codeql/cpp-queries@1.2.3"], + "target": "codeql/cpp-all", + } + ], + } + plan_path.write_text(json.dumps(plan)) catalog_path = root / "catalog.json" catalog_path.write_text( json.dumps({"schema_version": 1, "bundles": []}) ) entry_path = root / "entry.json" - plan_path.write_text(json.dumps(_release_plan(BUNDLE_PLATFORMS))) - - result = runner.invoke( - main, - [ - "catalog-entry", - "--plan", - str(plan_path), - "--assets-dir", - str(assets), - "--validated-platform", - "linux-arm64", - "--output", - str(entry_path), - ], - ) + catalog_entry_args = [ + "catalog-entry", + "--plan", + str(plan_path), + "--assets-dir", + str(assets), + "--validated-platform", + "linux-arm64", + "--output", + str(entry_path), + ] + + result = runner.invoke(main, catalog_entry_args) self.assertEqual(0, result.exit_code, result.output) result = runner.invoke( @@ -181,40 +161,21 @@ def test_arm64_catalog_entry_can_be_added_to_catalog(self) -> None: self.assertIsNotNone(bundle) self.assertEqual(("linux-arm64",), bundle.validated_platforms) - def test_catalog_entry_rejects_platform_missing_from_plan(self) -> None: - runner = CliRunner() - with runner.isolated_filesystem(): - assets = Path("assets") - assets.mkdir() - plan_path = Path("plan.json") - plan_path.write_text( - json.dumps( - _release_plan(("linux64", "osx64", "win64")) - ) + plan["source_assets"] = [ + asset + for asset in plan["source_assets"] + if asset["platform"] != "linux-arm64" + ] + plan_path.write_text(json.dumps(plan)) + result = runner.invoke(main, catalog_entry_args) + + self.assertNotEqual(0, result.exit_code) + self.assertIn( + "Release plan has no source bundle for validated platform(s): " + "linux-arm64.", + result.output, ) - result = runner.invoke( - main, - [ - "catalog-entry", - "--plan", - str(plan_path), - "--assets-dir", - str(assets), - "--validated-platform", - "linux-arm64", - "--output", - "entry.json", - ], - ) - - self.assertNotEqual(0, result.exit_code) - self.assertIn( - "Release plan has no source bundle for validated platform(s): " - "linux-arm64.", - result.output, - ) - if __name__ == "__main__": unittest.main() From 59fa45280b25cfd013d5edebc58e1842c73014b0 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 25 Sep 2026 13:29:08 +0200 Subject: [PATCH 7/9] Exclude x64 Swift tools from ARM64 bundles Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- codeql_bundle/helpers/bundle.py | 5 ++++- tests/test_bundle.py | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/codeql_bundle/helpers/bundle.py b/codeql_bundle/helpers/bundle.py index 001e30c..c61a919 100644 --- a/codeql_bundle/helpers/bundle.py +++ b/codeql_bundle/helpers/bundle.py @@ -912,7 +912,10 @@ def filter(tarinfo: tarfile.TarInfo) -> Optional[tarfile.TarInfo]: exclusion_paths.append(Path("swift/qltest/osx64")) exclusion_paths.append(Path("swift/resource-dir/osx64")) - if platform == BundlePlatform.OSX: + if platform in { + BundlePlatform.LINUX_ARM64, + BundlePlatform.OSX, + }: exclusion_paths.append(Path("swift/qltest/linux64")) exclusion_paths.append(Path("swift/resource-dir/linux64")) diff --git a/tests/test_bundle.py b/tests/test_bundle.py index 6dd6b19..44c538f 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -56,6 +56,14 @@ def test_linux_arm64_bundle_uses_only_native_tools(self) -> None: self.assertIsNone( archive_filter(tarfile.TarInfo("codeql/tools/linux64/tool")) ) + self.assertIsNone( + archive_filter(tarfile.TarInfo("codeql/swift/qltest/linux64/tool")) + ) + self.assertIsNone( + archive_filter( + tarfile.TarInfo("codeql/swift/resource-dir/linux64/tool") + ) + ) self.assertIsNotNone( archive_filter(tarfile.TarInfo("codeql/tools/linux-arm64/tool")) ) From f6f98584255e9c97a546c8a85cc1fefd6925655b Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 25 Sep 2026 13:46:19 +0200 Subject: [PATCH 8/9] Restore legacy all-platform bundle fallback Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- codeql_bundle/cache.py | 8 +++++++- tests/test_cache.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/codeql_bundle/cache.py b/codeql_bundle/cache.py index 1244197..470e277 100644 --- a/codeql_bundle/cache.py +++ b/codeql_bundle/cache.py @@ -173,7 +173,7 @@ def to_dict(self) -> dict[str, Any]: } def source_asset_for_platform(self, platform_name: str) -> Optional[SourceAsset]: - return next( + asset = next( ( asset for asset in self.source_assets @@ -181,6 +181,12 @@ def source_asset_for_platform(self, platform_name: str) -> Optional[SourceAsset] ), None, ) + if asset is not None: + return asset + return next( + (asset for asset in self.source_assets if asset.platform == "all"), + None, + ) class BundleCatalog: diff --git a/tests/test_cache.py b/tests/test_cache.py index a63b0ba..030ae80 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -274,6 +274,36 @@ def test_local_archive_matches_catalog_digest(self) -> None: self.assertEqual(bundle, resolved.supported_bundle) + def test_cataloged_release_uses_legacy_all_platform_asset(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + served = root / "served" + served.mkdir() + archive = served / "codeql-bundle.tar.gz" + archive.write_bytes(b"bundle") + with serve(served) as base_url: + source = SourceAsset( + name=archive.name, + url=f"{base_url}/{archive.name}", + sha256=sha256_file(archive), + size=archive.stat().st_size, + platform="all", + ) + cache = ReleaseAsset( + name="cache.tar.gz", + url=f"{base_url}/cache.tar.gz", + sha256="2" * 64, + size=10, + ) + bundle = bundle_with_assets(source, cache) + resolved = BundleSourceResolver( + BundleCatalog([bundle]), root / "downloads" + ).resolve(bundle.release) + + self.assertEqual(bundle, resolved.supported_bundle) + self.assertEqual(source.sha256, resolved.digest) + self.assertEqual(archive.read_bytes(), resolved.path.read_bytes()) + def test_url_download_is_reused(self) -> None: with TemporaryDirectory() as directory: root = Path(directory) From 268282f1363e310ec137781bc7ebd89f349e6f99 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 25 Sep 2026 14:05:40 +0200 Subject: [PATCH 9/9] Exclude ARM64 Swift tools from non-ARM bundles Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- codeql_bundle/helpers/bundle.py | 9 +++++++ tests/test_bundle.py | 43 +++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/codeql_bundle/helpers/bundle.py b/codeql_bundle/helpers/bundle.py index c61a919..7bb252d 100644 --- a/codeql_bundle/helpers/bundle.py +++ b/codeql_bundle/helpers/bundle.py @@ -919,6 +919,15 @@ def filter(tarinfo: tarfile.TarInfo) -> Optional[tarfile.TarInfo]: exclusion_paths.append(Path("swift/qltest/linux64")) exclusion_paths.append(Path("swift/resource-dir/linux64")) + if platform in { + BundlePlatform.LINUX, + BundlePlatform.OSX, + }: + exclusion_paths.append(Path("swift/qltest/linux-arm64")) + exclusion_paths.append( + Path("swift/resource-dir/linux-arm64") + ) + tarfile_path_root = Path(tarfile_path.parts[0]) exclusion_paths = [ tarfile_path_root / path for path in exclusion_paths diff --git a/tests/test_bundle.py b/tests/test_bundle.py index 44c538f..2d7db84 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -68,6 +68,49 @@ def test_linux_arm64_bundle_uses_only_native_tools(self) -> None: archive_filter(tarfile.TarInfo("codeql/tools/linux-arm64/tool")) ) + def test_non_arm_bundles_exclude_linux_arm64_swift_tools(self) -> None: + for platform, native_swift_platform in ( + (BundlePlatform.LINUX, "linux64"), + (BundlePlatform.OSX, "osx64"), + ): + with self.subTest(platform=platform), TemporaryDirectory() as directory: + root = Path(directory) + bundle = object.__new__(CustomBundle) + bundle.tmp_dir = TemporaryDirectory() + bundle.bundle_path = root / "bundle" + bundle.languages = set() + bundle.platforms = {platform} + + with patch( + "codeql_bundle.helpers.bundle.tarfile.open" + ) as open_archive: + bundle.bundle(root, {platform}) + + archive_filter = ( + open_archive.return_value.__enter__.return_value.add.call_args.kwargs[ + "filter" + ] + ) + self.assertIsNone( + archive_filter( + tarfile.TarInfo("codeql/swift/qltest/linux-arm64/tool") + ) + ) + self.assertIsNone( + archive_filter( + tarfile.TarInfo( + "codeql/swift/resource-dir/linux-arm64/tool" + ) + ) + ) + self.assertIsNotNone( + archive_filter( + tarfile.TarInfo( + f"codeql/swift/qltest/{native_swift_platform}/tool" + ) + ) + ) + if __name__ == "__main__": unittest.main()