diff --git a/.github/workflows/codeql-compilation-caches.yml b/.github/workflows/codeql-compilation-caches.yml index 101a950..f28dfb0 100644 --- a/.github/workflows/codeql-compilation-caches.yml +++ b/.github/workflows/codeql-compilation-caches.yml @@ -45,6 +45,7 @@ jobs: matrix: ${{ steps.plan.outputs.matrix || '[{"language":"skip","target":"skip"}]' }} release: ${{ steps.version.outputs.release }} skip: ${{ steps.existing.outputs.skip }} + verify_os: ${{ steps.plan.outputs.verify_os || '["ubuntu-latest"]' }} steps: - uses: actions/checkout@v4 @@ -115,7 +116,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.CODEQL_BUNDLE_CACHE_DIR }}/sources/${{ steps.version.outputs.release }} - key: codeql-source-${{ steps.version.outputs.release }}-${{ runner.os }} + key: codeql-source-${{ steps.version.outputs.release }}-${{ runner.os }}-${{ runner.arch }} - if: steps.existing.outputs.skip != 'true' id: plan @@ -130,6 +131,12 @@ jobs: --output release-plan.json echo "cache_release=$(jq -r '.cache_release' release-plan.json)" >> "$GITHUB_OUTPUT" echo "matrix=$(jq -c '.targets' release-plan.json)" >> "$GITHUB_OUTPUT" + echo "verify_os=$(jq -c '[.source_assets[].platform | { + "linux64": "ubuntu-latest", + "linux-arm64": "ubuntu-24.04-arm", + "osx64": "macos-latest", + "win64": "windows-latest" + }[.]]' release-plan.json)" >> "$GITHUB_OUTPUT" - if: steps.existing.outputs.skip != 'true' uses: actions/upload-artifact@v4 @@ -162,7 +169,7 @@ jobs: - uses: actions/cache@v4 with: path: ${{ env.CODEQL_BUNDLE_CACHE_DIR }}/sources/${{ needs.plan.outputs.release }} - key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }} + key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }}-${{ runner.arch }} - name: Build and verify cache run: | @@ -189,10 +196,7 @@ jobs: strategy: fail-fast: false matrix: - os: - - ubuntu-latest - - macos-latest - - windows-latest + os: ${{ fromJSON(needs.plan.outputs.verify_os) }} runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -216,7 +220,7 @@ jobs: - uses: actions/cache@v4 with: path: ${{ env.CODEQL_BUNDLE_CACHE_DIR }}/sources/${{ needs.plan.outputs.release }} - key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }} + key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }}-${{ runner.arch }} - name: Verify all caches run: codeql-bundle-cache verify-all --plan release-plan.json --assets-dir dist @@ -299,13 +303,16 @@ jobs: - name: Create candidate catalog entry if: steps.recheck.outputs.skip != 'true' + shell: bash run: | + validated_platform_args=() + while IFS= read -r platform; do + validated_platform_args+=(--validated-platform "$platform") + done < <(jq -r '.source_assets[].platform' release-plan.json) codeql-bundle-cache catalog-entry \ --plan release-plan.json \ --assets-dir dist \ - --validated-platform linux64 \ - --validated-platform osx64 \ - --validated-platform win64 \ + "${validated_platform_args[@]}" \ --output catalog-entry.json codeql-bundle-cache verify-entry \ --entry catalog-entry.json \ diff --git a/README.md b/README.md index cdf2c8e..a71dec7 100644 --- a/README.md +++ b/README.md @@ -18,14 +18,14 @@ For more details on CodeQL customization packs see the section [CodeQL customiza The CodeQL bundle application can be installed using `pip` with the command: ```bash -python3.11 -m pip install https://github.com/advanced-security/codeql-bundle/releases/download/v0.5.0/codeql_bundle-0.5.0-py3-none-any.whl +python3.11 -m pip install https://github.com/advanced-security/codeql-bundle/releases/download/v0.6.0/codeql_bundle-0.6.0-py3-none-any.whl ``` ## Usage The source bundle can be an existing local archive or directory, a -`github/codeql-action` release tag, or an HTTP(S) URL. Release tags and URLs are -downloaded into a persistent local cache. +`github/codeql-action` release tag, or an HTTP(S) URL. Release tags select the +current platform's bundle, and downloads are stored in a persistent local cache. The CodeQL bundle application requires a [CodeQL workspace](https://codeql.github.com/docs/codeql-cli/about-codeql-workspaces/) to locate the packs you want to include in a custom bundle. You can see the packs available in your workspace by running `codeql pack ls -- ` where `` is the root directory of your CodeQL workspace. @@ -37,11 +37,14 @@ with the command: codeql-bundle --bundle codeql-bundle-v2.26.1 --output codeql-custom-bundle.tar.gz --workspace --log INFO ``` -If the source bundle is the platform agnostic bundle then you can create platform specific bundles to reduce the size of the used bundle(s). -The following example creates platform specific bundles for all the currently supported platforms. +Because the upstream all-platform bundle is +[deprecated](https://github.blog/changelog/2026-09-22-deprecation-notice-all-platform-codeql-bundle/), +release tags only build for the current target (`linux64`, `linux-arm64`, +`osx64`, or `win64`). Run once per target; local all-platform archives still +support multiple targets. ```bash -codeql-bundle --bundle --output --workspace --log INFO -p linux64 -p osx64 -p win64 +codeql-bundle --bundle codeql-bundle-v2.27.0 --output --workspace --log INFO -p linux64 ``` ### Compilation caches diff --git a/codeql_bundle/cache.py b/codeql_bundle/cache.py index 2aeeee7..470e277 100644 --- a/codeql_bundle/cache.py +++ b/codeql_bundle/cache.py @@ -36,6 +36,7 @@ CACHE_FORMAT_VERSION = 1 DOWNLOAD_CHUNK_SIZE = 1024 * 1024 CATALOG_REFRESH_SECONDS = 60 * 60 +BUNDLE_PLATFORMS = ("linux64", "linux-arm64", "osx64", "win64") RELEASE_PATTERN = re.compile(r"^codeql-bundle-v\d+\.\d+\.\d+$") CACHE_RELEASE_PATTERN = re.compile( r"^codeql-compilation-cache-v\d+\.\d+\.\d+(?:-[A-Za-z0-9._-]+)?$" @@ -172,7 +173,7 @@ def to_dict(self) -> dict[str, Any]: } def source_asset_for_platform(self, platform_name: str) -> Optional[SourceAsset]: - return next( + asset = next( ( asset for asset in self.source_assets @@ -180,6 +181,12 @@ def source_asset_for_platform(self, platform_name: str) -> Optional[SourceAsset] ), None, ) + if asset is not None: + return asset + return next( + (asset for asset in self.source_assets if asset.platform == "all"), + None, + ) class BundleCatalog: @@ -323,7 +330,12 @@ def default_cache_dir() -> Path: def current_bundle_platform() -> str: system = platform.system() if system == "Linux": - return "linux64" + machine = platform.machine().lower() + if machine in {"x86_64", "amd64"}: + return "linux64" + if machine in {"aarch64", "arm64"}: + return "linux-arm64" + raise CacheException(f"Unsupported Linux architecture: {machine}") if system == "Darwin": return "osx64" if system == "Windows": @@ -721,8 +733,6 @@ def _resolve_release( bundle = self.catalog.find_release(release) if bundle: asset = bundle.source_asset_for_platform(platform_name) - if asset is None and platform_name != "all": - asset = bundle.source_asset_for_platform("all") if asset is None: raise CatalogException( f"Bundle {release} has no source asset for {platform_name}." @@ -1009,19 +1019,20 @@ def write_json(path: Path, value: dict[str, Any]) -> None: def source_asset_name(platform_name: str) -> str: if platform_name == "all": return "codeql-bundle.tar.gz" - if platform_name not in {"linux64", "osx64", "win64"}: + if platform_name not in BUNDLE_PLATFORMS: raise CacheException(f"Unsupported bundle platform: {platform_name}") return f"codeql-bundle-{platform_name}.tar.gz" def source_platform_for_request(requested_platforms: Iterable[str]) -> str: - requested = tuple(requested_platforms) + requested = set(requested_platforms) current = current_bundle_platform() - if not requested: - return "all" - if requested == (current,): - return current - return "all" + if requested and requested != {current}: + raise CacheException( + f"Release tags can only build for the current platform ({current}); " + f"requested {', '.join(sorted(requested))}." + ) + return current def github_asset_digest(asset: dict[str, Any]) -> Optional[str]: diff --git a/codeql_bundle/cache_cli.py b/codeql_bundle/cache_cli.py index 5b92983..b4f5b0b 100644 --- a/codeql_bundle/cache_cli.py +++ b/codeql_bundle/cache_cli.py @@ -14,6 +14,7 @@ from semantic_version import Version from codeql_bundle.cache import ( + BUNDLE_PLATFORMS, CACHE_FORMAT_VERSION, CODEQL_ACTION_REPOSITORY, BundleCatalog, @@ -50,7 +51,7 @@ logger = logging.getLogger(__name__) -SOURCE_PLATFORMS = ("all", "linux64", "osx64", "win64") +REQUIRED_SOURCE_PLATFORMS = frozenset(("linux64", "osx64", "win64")) MAX_RELEASE_ASSET_SIZE = 2 * 1024 * 1024 * 1024 DEFAULT_COMPILATION_CACHE_SIZE_MB = 1536 @@ -142,7 +143,7 @@ def prune_cache(cache_dir: Path, max_age_days: float, dry_run: bool) -> None: "--platform", "platforms", multiple=True, - type=click.Choice(["linux64", "osx64", "win64"]), + type=click.Choice(BUNDLE_PLATFORMS), ) @click.option( "--cache-dir", @@ -212,11 +213,15 @@ def plan_release( client = GitHubReleaseClient() release_value = client.release(release) source_assets = _release_source_assets(release_value) + platform_name = current_bundle_platform() source_asset = next( - asset - for asset in source_assets - if asset.platform == current_bundle_platform() + (asset for asset in source_assets if asset.platform == platform_name), + None, ) + if source_asset is None: + raise click.ClickException( + f"Release {release} has no source bundle for {platform_name}." + ) source_path = cache_path(cache_dir, "sources", release, source_asset.name) download_file( source_asset.url, @@ -493,7 +498,7 @@ def _verify_cache_with_bundle( "validated_platforms", multiple=True, required=True, - type=click.Choice(["linux64", "osx64", "win64"]), + type=click.Choice(BUNDLE_PLATFORMS), ) @click.option( "--output", @@ -509,6 +514,15 @@ def catalog_entry( ) -> None: """Create a catalog entry for verified release assets.""" plan = _read_plan(plan_path) + if missing := sorted( + set(validated_platforms) + - {asset["platform"] for asset in plan["source_assets"]} + ): + raise click.ClickException( + "Release plan has no source bundle for validated platform(s): " + f"{', '.join(missing)}." + ) + compilation_caches = {} for target in plan["targets"]: asset_name = _cache_asset_name(target["language"]) @@ -581,10 +595,12 @@ def update_catalog( def _release_source_assets(release: dict[str, Any]) -> tuple[SourceAsset, ...]: assets = {asset["name"]: asset for asset in release.get("assets", [])} result = [] - for platform_name in SOURCE_PLATFORMS: + for platform_name in BUNDLE_PLATFORMS: name = source_asset_name(platform_name) asset = assets.get(name) if asset is None: + if platform_name not in REQUIRED_SOURCE_PLATFORMS: + continue raise click.ClickException( f"Upstream release {release['tag_name']} has no {name}." ) @@ -800,10 +816,11 @@ def _read_plan(path: Path) -> dict[str, Any]: validate_remote_url(source.url) source_platforms.add(source.platform) if ( - source_platforms != set(SOURCE_PLATFORMS) - or len(value["source_assets"]) != len(SOURCE_PLATFORMS) + not REQUIRED_SOURCE_PLATFORMS.issubset(source_platforms) + or not source_platforms.issubset(BUNDLE_PLATFORMS) + or len(value["source_assets"]) != len(source_platforms) ): - raise ValueError("incomplete source platform inventory") + raise ValueError("invalid source platform inventory") target_names = set() languages = set() diff --git a/codeql_bundle/cli.py b/codeql_bundle/cli.py index d8d8c1b..5c03a76 100644 --- a/codeql_bundle/cli.py +++ b/codeql_bundle/cli.py @@ -12,6 +12,7 @@ from codeql_bundle.helpers.codeql import CodeQLException from codeql_bundle.helpers.bundle import CustomBundle, BundleException, BundlePlatform from codeql_bundle.cache import ( + BUNDLE_PLATFORMS, BundleCatalog, BundleSourceResolver, CacheException, @@ -65,7 +66,7 @@ "-p", "--platform", multiple=True, - type=click.Choice(["linux64", "osx64", "win64"], case_sensitive=False), + type=click.Choice(BUNDLE_PLATFORMS, case_sensitive=False), help="Target platform for the bundle", ) @click.option( diff --git a/codeql_bundle/helpers/bundle.py b/codeql_bundle/helpers/bundle.py index 24f2975..7bb252d 100644 --- a/codeql_bundle/helpers/bundle.py +++ b/codeql_bundle/helpers/bundle.py @@ -12,7 +12,7 @@ import os import subprocess from jsonschema import validate, ValidationError -from enum import Enum, verify, UNIQUE +from enum import Enum, StrEnum, verify, UNIQUE from dataclasses import dataclass from graphlib import TopologicalSorter import platform @@ -20,6 +20,7 @@ from codeql_bundle.cache import ( CompilationCacheManager, compute_pack_fingerprint, + current_bundle_platform, safe_extract_tar, ) @@ -194,32 +195,19 @@ def get_compilation_cache_targets( return targets -@verify(UNIQUE) -class BundlePlatform(Enum): - LINUX = 1 - WINDOWS = 2 - OSX = 3 - - @staticmethod - def from_string(platform: str) -> "BundlePlatform": - if platform.lower() == "linux" or platform.lower() == "linux64": - return BundlePlatform.LINUX - elif platform.lower() == "windows" or platform.lower() == "win64": - return BundlePlatform.WINDOWS - elif platform.lower() == "osx" or platform.lower() == "osx64": - return BundlePlatform.OSX - else: - raise BundleException(f"Invalid platform {platform}") - - def __str__(self): - if self == BundlePlatform.LINUX: - return "linux64" - elif self == BundlePlatform.WINDOWS: - return "win64" - elif self == BundlePlatform.OSX: - return "osx64" - else: - raise BundleException(f"Invalid platform {self}") +class BundlePlatform(StrEnum): + LINUX = "linux64" + LINUX_ARM64 = "linux-arm64" + WINDOWS = "win64" + OSX = "osx64" + + @classmethod + def from_string(cls, platform: str) -> "BundlePlatform": + aliases = {"linux": "linux64", "windows": "win64", "osx": "osx64"} + try: + return cls(aliases.get(platform.lower(), platform.lower())) + except ValueError: + raise BundleException(f"Invalid platform {platform}") from None class Bundle: @@ -242,39 +230,15 @@ def __init__(self, bundle_path: Path) -> None: else: raise BundleException("Invalid CodeQL bundle path") - def supports_linux() -> set[BundlePlatform]: - if (self.bundle_path / "cpp" / "tools" / "linux64").exists(): - return {BundlePlatform.LINUX} - else: - return set() - - def supports_macos() -> set[BundlePlatform]: - if (self.bundle_path / "cpp" / "tools" / "osx64").exists(): - return {BundlePlatform.OSX} - else: - return set() - - def supports_windows() -> set[BundlePlatform]: - if (self.bundle_path / "cpp" / "tools" / "win64").exists(): - return {BundlePlatform.WINDOWS} - else: - return set() - - self.platforms: set[BundlePlatform] = ( - supports_linux() | supports_macos() | supports_windows() - ) + self.platforms = { + platform + for platform in BundlePlatform + if (self.bundle_path / "cpp" / "tools" / platform).exists() + } - current_system = platform.system() - if not current_system in ["Linux", "Darwin", "Windows"]: - raise BundleException(f"Unsupported system: {current_system}") - if current_system == "Linux" and BundlePlatform.LINUX not in self.platforms: - raise BundleException("Bundle doesn't support Linux!") - elif current_system == "Darwin" and BundlePlatform.OSX not in self.platforms: - raise BundleException("Bundle doesn't support OSX!") - elif ( - current_system == "Windows" and BundlePlatform.WINDOWS not in self.platforms - ): - raise BundleException("Bundle doesn't support Windows!") + current_platform = BundlePlatform.from_string(current_bundle_platform()) + if current_platform not in self.platforms: + raise BundleException(f"Bundle doesn't support {current_platform}!") self.codeql = CodeQL(self.bundle_codeql_exe) @@ -785,28 +749,28 @@ def is_unsafe_path(basedir: Path, path: Path) -> bool: config = load_config(config_path) - if platform.system() == "Windows": - keytool = "tools/win64/java/bin/keytool.exe" - elif platform.system() == "Linux": - keytool = "tools/linux64/java/bin/keytool" - elif platform.system() == "Darwin": - keytool = "tools/osx64/java/bin/keytool" - else: - raise BundleException(f"Unsupported platform {platform.system()}") - - keytool = self.bundle_path / keytool - if not keytool.exists(): - raise BundleException(f"Keytool {keytool} does not exist.") - - keystores: list[str] = [ - "tools/win64/java/lib/security/cacerts", - "tools/linux64/java/lib/security/cacerts", - "tools/osx64/java/lib/security/cacerts", - "tools/osx64/java-aarch64/lib/security/cacerts", - ] - # Add the certificates to the Java keystores if "CodeQLBundleAdditionalCertificates" in config: + platform_name = current_bundle_platform() + keytool = ( + self.bundle_path + / "tools" + / platform_name + / "java" + / "bin" + / ("keytool.exe" if platform_name == "win64" else "keytool") + ) + if not keytool.exists(): + raise BundleException(f"Keytool {keytool} does not exist.") + + keystores = list( + (self.bundle_path / "tools").glob( + "*/java*/lib/security/cacerts" + ) + ) + if not keystores: + raise BundleException("The bundle contains no Java keystores.") + for cert in config["CodeQLBundleAdditionalCertificates"]: src = workspace_path / Path(cert["Source"]) src = src.resolve() @@ -818,9 +782,6 @@ def is_unsafe_path(basedir: Path, path: Path) -> bool: raise BundleException(f"Certificate file {src} does not exist.") for keystore in keystores: - keystore = self.bundle_path / keystore - if not keystore.exists(): - raise BundleException(f"Keystore {keystore} does not exist.") logging.info(f"Adding certificate {src} to keystore {keystore}") subprocess.run( [ @@ -917,32 +878,19 @@ def get_nonplatform_tool_paths( platform: BundlePlatform, ) -> List[Path]: """Get a list of paths to tools that are not for the specified platform relative to the root of a bundle.""" - specialize_path: Optional[Callable[[Path], List[Path]]] = None - linux64_subpaths = [Path("linux64"), Path("linux")] - osx64_subpaths = [Path("osx64"), Path("macos")] - win64_subpaths = [Path("win64"), Path("windows")] - if platform == BundlePlatform.LINUX: - specialize_path = lambda p: [ - p / subpath - for subpath in osx64_subpaths + win64_subpaths - ] - elif platform == BundlePlatform.WINDOWS: - specialize_path = lambda p: [ - p / subpath - for subpath in osx64_subpaths + linux64_subpaths - ] - elif platform == BundlePlatform.OSX: - specialize_path = lambda p: [ - p / subpath - for subpath in linux64_subpaths + win64_subpaths - ] - else: - raise BundleException(f"Unsupported platform {platform}.") + platform_subpaths = { + BundlePlatform.LINUX: ("linux64", "linux"), + BundlePlatform.LINUX_ARM64: ("linux-arm64",), + BundlePlatform.OSX: ("osx64", "macos"), + BundlePlatform.WINDOWS: ("win64", "windows"), + } return [ - candidate - for candidates in map(specialize_path, relative_tools_paths) - for candidate in candidates + tools_path / subpath + for tools_path in relative_tools_paths + for candidate_platform, subpaths in platform_subpaths.items() + if candidate_platform != platform + for subpath in subpaths ] def filter(tarinfo: tarfile.TarInfo) -> Optional[tarfile.TarInfo]: @@ -957,14 +905,29 @@ def filter(tarinfo: tarfile.TarInfo) -> Optional[tarfile.TarInfo]: exclusion_paths.append(Path("swift/qltest")) exclusion_paths.append(Path("swift/resource-dir")) - if platform == BundlePlatform.LINUX: + if platform in { + BundlePlatform.LINUX, + BundlePlatform.LINUX_ARM64, + }: exclusion_paths.append(Path("swift/qltest/osx64")) exclusion_paths.append(Path("swift/resource-dir/osx64")) - if platform == BundlePlatform.OSX: + if platform in { + BundlePlatform.LINUX_ARM64, + BundlePlatform.OSX, + }: exclusion_paths.append(Path("swift/qltest/linux64")) exclusion_paths.append(Path("swift/resource-dir/linux64")) + if platform in { + BundlePlatform.LINUX, + BundlePlatform.OSX, + }: + exclusion_paths.append(Path("swift/qltest/linux-arm64")) + exclusion_paths.append( + Path("swift/resource-dir/linux-arm64") + ) + tarfile_path_root = Path(tarfile_path.parts[0]) exclusion_paths = [ tarfile_path_root / path for path in exclusion_paths diff --git a/codeql_bundle/supported-codeql-bundles.schema.json b/codeql_bundle/supported-codeql-bundles.schema.json index d69afd4..af9458c 100644 --- a/codeql_bundle/supported-codeql-bundles.schema.json +++ b/codeql_bundle/supported-codeql-bundles.schema.json @@ -84,6 +84,7 @@ "enum": [ "all", "linux64", + "linux-arm64", "osx64", "win64" ] @@ -177,6 +178,7 @@ "items": { "enum": [ "linux64", + "linux-arm64", "osx64", "win64" ] diff --git a/pyproject.toml b/pyproject.toml index 664a87d..2022d0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "codeql-bundle" -version = "0.5.0" +version = "0.6.0" description = "Tool to create custom CodeQL bundles" authors = ["Remco Vermeulen "] readme = "README.md" diff --git a/tests/test_bundle.py b/tests/test_bundle.py new file mode 100644 index 0000000..2d7db84 --- /dev/null +++ b/tests/test_bundle.py @@ -0,0 +1,116 @@ +from pathlib import Path +from tempfile import TemporaryDirectory +import tarfile +import unittest +from unittest.mock import patch + +from codeql_bundle.helpers.bundle import BundlePlatform, CustomBundle + + +class BundleTests(unittest.TestCase): + def test_linux_arm64_bundle_uses_only_native_tools(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + certificate = root / "certificate.pem" + certificate.write_text("certificate") + config = root / "additional-data.json" + config.write_text( + '{"CodeQLBundleAdditionalCertificates":' + '[{"Source":"certificate.pem"}]}' + ) + + bundle = object.__new__(CustomBundle) + bundle.tmp_dir = None + bundle.bundle_path = root / "bundle" + bundle.languages = [] + bundle.platforms = {BundlePlatform.LINUX_ARM64} + keytool = bundle.bundle_path / "tools/linux-arm64/java/bin/keytool" + keystore = ( + bundle.bundle_path + / "tools/linux-arm64/java/lib/security/cacerts" + ) + for path in (keytool, keystore): + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + + with patch( + "codeql_bundle.helpers.bundle.current_bundle_platform", + return_value="linux-arm64", + ), patch( + "codeql_bundle.helpers.bundle.subprocess.run" + ) as run, patch( + "codeql_bundle.helpers.bundle.tarfile.open" + ) as open_archive: + bundle.add_files_and_certs(config, root) + bundle.bundle(root, {BundlePlatform.LINUX_ARM64}) + + run.assert_called_once() + command = run.call_args.args[0] + self.assertEqual(str(keytool), command[0]) + self.assertEqual(str(keystore), command[command.index("-keystore") + 1]) + archive_filter = ( + open_archive.return_value.__enter__.return_value.add.call_args.kwargs[ + "filter" + ] + ) + self.assertIsNone( + archive_filter(tarfile.TarInfo("codeql/tools/linux64/tool")) + ) + self.assertIsNone( + archive_filter(tarfile.TarInfo("codeql/swift/qltest/linux64/tool")) + ) + self.assertIsNone( + archive_filter( + tarfile.TarInfo("codeql/swift/resource-dir/linux64/tool") + ) + ) + self.assertIsNotNone( + archive_filter(tarfile.TarInfo("codeql/tools/linux-arm64/tool")) + ) + + def test_non_arm_bundles_exclude_linux_arm64_swift_tools(self) -> None: + for platform, native_swift_platform in ( + (BundlePlatform.LINUX, "linux64"), + (BundlePlatform.OSX, "osx64"), + ): + with self.subTest(platform=platform), TemporaryDirectory() as directory: + root = Path(directory) + bundle = object.__new__(CustomBundle) + bundle.tmp_dir = TemporaryDirectory() + bundle.bundle_path = root / "bundle" + bundle.languages = set() + bundle.platforms = {platform} + + with patch( + "codeql_bundle.helpers.bundle.tarfile.open" + ) as open_archive: + bundle.bundle(root, {platform}) + + archive_filter = ( + open_archive.return_value.__enter__.return_value.add.call_args.kwargs[ + "filter" + ] + ) + self.assertIsNone( + archive_filter( + tarfile.TarInfo("codeql/swift/qltest/linux-arm64/tool") + ) + ) + self.assertIsNone( + archive_filter( + tarfile.TarInfo( + "codeql/swift/resource-dir/linux-arm64/tool" + ) + ) + ) + self.assertIsNotNone( + archive_filter( + tarfile.TarInfo( + f"codeql/swift/qltest/{native_swift_platform}/tool" + ) + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cache.py b/tests/test_cache.py index 748e757..030ae80 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -30,6 +30,7 @@ current_bundle_platform, safe_extract_tar, sha256_file, + source_asset_name, source_platform_for_request, ) from codeql_bundle.helpers.codeql import CodeQLPack, CodeQLPackConfig @@ -228,16 +229,24 @@ def test_empty_catalog_does_not_hash_local_archive(self) -> None: def test_release_source_is_runnable_on_current_platform(self) -> None: current = current_bundle_platform() - other = next( - platform - for platform in ("linux64", "osx64", "win64") - if platform != current - ) - self.assertEqual("all", source_platform_for_request(())) + other = "win64" if current != "win64" else "linux64" + self.assertEqual(current, source_platform_for_request(())) self.assertEqual(current, source_platform_for_request((current,))) - self.assertEqual("all", source_platform_for_request((other,))) + with self.assertRaisesRegex( + CacheException, "only build for the current platform" + ): + source_platform_for_request((other,)) + + def test_linux_arm64_platform(self) -> None: + with patch( + "codeql_bundle.cache.platform.system", return_value="Linux" + ), patch( + "codeql_bundle.cache.platform.machine", return_value="aarch64" + ): + self.assertEqual("linux-arm64", current_bundle_platform()) self.assertEqual( - "all", source_platform_for_request(("linux64", "win64")) + "codeql-bundle-linux-arm64.tar.gz", + source_asset_name("linux-arm64"), ) def test_local_archive_matches_catalog_digest(self) -> None: @@ -265,6 +274,36 @@ def test_local_archive_matches_catalog_digest(self) -> None: self.assertEqual(bundle, resolved.supported_bundle) + def test_cataloged_release_uses_legacy_all_platform_asset(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + served = root / "served" + served.mkdir() + archive = served / "codeql-bundle.tar.gz" + archive.write_bytes(b"bundle") + with serve(served) as base_url: + source = SourceAsset( + name=archive.name, + url=f"{base_url}/{archive.name}", + sha256=sha256_file(archive), + size=archive.stat().st_size, + platform="all", + ) + cache = ReleaseAsset( + name="cache.tar.gz", + url=f"{base_url}/cache.tar.gz", + sha256="2" * 64, + size=10, + ) + bundle = bundle_with_assets(source, cache) + resolved = BundleSourceResolver( + BundleCatalog([bundle]), root / "downloads" + ).resolve(bundle.release) + + self.assertEqual(bundle, resolved.supported_bundle) + self.assertEqual(source.sha256, resolved.digest) + self.assertEqual(archive.read_bytes(), resolved.path.read_bytes()) + def test_url_download_is_reused(self) -> None: with TemporaryDirectory() as directory: root = Path(directory) @@ -345,10 +384,7 @@ def release(self, tag: str) -> dict[str, object]: BundleCatalog([]), root / "downloads", release_client=ReleaseClient(), - ).resolve( - "codeql-bundle-v1.2.3", - [current_bundle_platform()], - ) + ).resolve("codeql-bundle-v1.2.3") self.assertEqual(sha256_file(archive), resolved.digest) diff --git a/tests/test_cache_cli.py b/tests/test_cache_cli.py index 9bbe664..2b2b23d 100644 --- a/tests/test_cache_cli.py +++ b/tests/test_cache_cli.py @@ -6,8 +6,8 @@ from click.testing import CliRunner -from codeql_bundle.cache import CatalogLoader -from codeql_bundle.cache_cli import main +from codeql_bundle.cache import BUNDLE_PLATFORMS, CatalogLoader, source_asset_name +from codeql_bundle.cache_cli import _release_source_assets, main class CacheCliTests(unittest.TestCase): @@ -61,7 +61,32 @@ def test_build_rejects_invalid_release_plan(self) -> None: self.assertNotEqual(0, result.exit_code) self.assertIn("Invalid release plan", result.output) - def test_catalog_entry_can_be_added_to_catalog(self) -> None: + def test_release_source_assets_include_arm64_when_available(self) -> None: + for platforms in ( + ("linux64", "osx64", "win64"), + BUNDLE_PLATFORMS, + ): + with self.subTest(platforms=platforms): + release = { + "tag_name": "codeql-bundle-v1.2.3", + "assets": [ + { + "browser_download_url": f"https://example.test/{source_asset_name(platform)}", + "digest": f"sha256:{index:064x}", + "name": source_asset_name(platform), + "size": 100, + } + for index, platform in enumerate(platforms, start=1) + ], + } + + assets = _release_source_assets(release) + + self.assertEqual( + platforms, tuple(asset.platform for asset in assets) + ) + + def test_arm64_catalog_entry_can_be_added_to_catalog(self) -> None: runner = CliRunner() with runner.isolated_filesystem(): root = Path.cwd() @@ -69,6 +94,7 @@ def test_catalog_entry_can_be_added_to_catalog(self) -> None: assets.mkdir() cache_asset = assets / "codeql-compilation-cache-cpp.tar.gz" cache_asset.write_bytes(b"cache") + plan_path = root / "plan.json" plan = { "cache_format": 1, "cache_release": "codeql-compilation-cache-v1.2.3", @@ -77,19 +103,14 @@ def test_catalog_entry_can_be_added_to_catalog(self) -> None: "release": "codeql-bundle-v1.2.3", "source_assets": [ { - "name": name, + "name": source_asset_name(platform), "platform": platform, "sha256": str(index) * 64, "size": 100, - "url": f"https://example.test/{name}", + "url": f"https://example.test/{source_asset_name(platform)}", } - for index, (platform, name) in enumerate( - [ - ("all", "codeql-bundle.tar.gz"), - ("linux64", "codeql-bundle-linux64.tar.gz"), - ("osx64", "codeql-bundle-osx64.tar.gz"), - ("win64", "codeql-bundle-win64.tar.gz"), - ], + for index, platform in enumerate( + BUNDLE_PLATFORMS, start=1, ) ], @@ -102,28 +123,25 @@ def test_catalog_entry_can_be_added_to_catalog(self) -> None: } ], } - plan_path = root / "plan.json" plan_path.write_text(json.dumps(plan)) catalog_path = root / "catalog.json" catalog_path.write_text( json.dumps({"schema_version": 1, "bundles": []}) ) entry_path = root / "entry.json" + catalog_entry_args = [ + "catalog-entry", + "--plan", + str(plan_path), + "--assets-dir", + str(assets), + "--validated-platform", + "linux-arm64", + "--output", + str(entry_path), + ] - result = runner.invoke( - main, - [ - "catalog-entry", - "--plan", - str(plan_path), - "--assets-dir", - str(assets), - "--validated-platform", - "linux64", - "--output", - str(entry_path), - ], - ) + result = runner.invoke(main, catalog_entry_args) self.assertEqual(0, result.exit_code, result.output) result = runner.invoke( @@ -139,7 +157,24 @@ def test_catalog_entry_can_be_added_to_catalog(self) -> None: self.assertEqual(0, result.exit_code, result.output) catalog = CatalogLoader().load(str(catalog_path)) - self.assertIsNotNone(catalog.find_release("codeql-bundle-v1.2.3")) + bundle = catalog.find_release("codeql-bundle-v1.2.3") + self.assertIsNotNone(bundle) + self.assertEqual(("linux-arm64",), bundle.validated_platforms) + + plan["source_assets"] = [ + asset + for asset in plan["source_assets"] + if asset["platform"] != "linux-arm64" + ] + plan_path.write_text(json.dumps(plan)) + result = runner.invoke(main, catalog_entry_args) + + self.assertNotEqual(0, result.exit_code) + self.assertIn( + "Release plan has no source bundle for validated platform(s): " + "linux-arm64.", + result.output, + ) if __name__ == "__main__":