From 0fb7232746a35a97a938e1b23fcbbee79982bb5d Mon Sep 17 00:00:00 2001 From: Alex Reinking Date: Sat, 8 Aug 2026 18:33:48 -0400 Subject: [PATCH 1/4] Share release and package metadata helpers --- generate_index.py | 24 +++------------ scripts/check_deps.py | 26 ++++------------ scripts/check_llvm.py | 34 +++++++------------- scripts/github_releases.py | 40 ++++++++++++++++++++++++ scripts/package_metadata.py | 22 +++++++++++++ scripts/validate_package_metadata.py | 7 ++--- tests/test_github_releases.py | 41 +++++++++++++++++++++++++ tests/test_validate_package_metadata.py | 8 +++++ 8 files changed, 134 insertions(+), 68 deletions(-) create mode 100644 scripts/github_releases.py create mode 100644 scripts/package_metadata.py create mode 100644 tests/test_github_releases.py diff --git a/generate_index.py b/generate_index.py index 861b016..e1fc893 100755 --- a/generate_index.py +++ b/generate_index.py @@ -12,13 +12,14 @@ from __future__ import annotations -import json import os import re import sys -import urllib.request from pathlib import Path +sys.path.insert(0, str(Path(__file__).with_name("scripts"))) +from github_releases import GitHubReleases + REPO = os.environ.get("GITHUB_REPOSITORY", "halide/pypi") TOKEN = os.environ.get("GITHUB_TOKEN") OUT_DIR = Path(os.environ.get("OUT_DIR", "_site")) @@ -29,25 +30,8 @@ def normalize(name: str) -> str: return re.sub(r"[-_.]+", "-", name).lower() -def api_get(path: str): - req = urllib.request.Request(f"https://api.github.com{path}") - req.add_header("Accept", "application/vnd.github+json") - if TOKEN: - req.add_header("Authorization", f"Bearer {TOKEN}") - with urllib.request.urlopen(req, timeout=30) as resp: - return json.load(resp) - - def list_all_releases() -> list[dict]: - releases = [] - page = 1 - while True: - batch = api_get(f"/repos/{REPO}/releases?per_page=100&page={page}") - if not batch: - break - releases.extend(batch) - page += 1 - return releases + return list(GitHubReleases(REPO, TOKEN).releases()) def project_for_tag(tag: str) -> str | None: diff --git a/scripts/check_deps.py b/scripts/check_deps.py index 9f5ff0a..f2f0621 100755 --- a/scripts/check_deps.py +++ b/scripts/check_deps.py @@ -6,12 +6,11 @@ import json import os import sys -import urllib.error -import urllib.request from collections.abc import Callable, Iterable from pathlib import Path -import tomllib +from github_releases import GitHubReleases +from package_metadata import project_identity as metadata_project_identity PLATFORMS = ( { @@ -52,9 +51,7 @@ def project_identity(package_dir: Path) -> tuple[str, str]: - with (package_dir / "pyproject.toml").open("rb") as file: - project = tomllib.load(file)["project"] - return project["name"], project["version"] + return metadata_project_identity(package_dir / "pyproject.toml") def missing_package_matrix( @@ -72,20 +69,9 @@ def missing_package_matrix( def github_release_exists(tag: str) -> bool: - request = urllib.request.Request( - f"https://api.github.com/repos/{os.environ['GITHUB_REPOSITORY']}/releases/tags/{tag}", - headers={ - "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}", - "Accept": "application/vnd.github+json", - }, - ) - try: - urllib.request.urlopen(request, timeout=30) - return True - except urllib.error.HTTPError as error: - if error.code == 404: - return False - raise + return GitHubReleases( + os.environ["GITHUB_REPOSITORY"], os.environ.get("GITHUB_TOKEN") + ).release_exists(tag) def main(arguments: list[str] | None = None) -> None: diff --git a/scripts/check_llvm.py b/scripts/check_llvm.py index fb09cd9..353c2fc 100755 --- a/scripts/check_llvm.py +++ b/scripts/check_llvm.py @@ -3,14 +3,13 @@ from __future__ import annotations -import json import os import sys -import urllib.request from collections.abc import Callable, Iterable from pathlib import Path -import tomllib +from github_releases import GitHubReleases +from package_metadata import project_metadata sys.path.insert(0, str(Path(__file__).parents[1] / "packages" / "halide-llvm")) from _version_provider import get_commit_info, version_from_tag @@ -35,31 +34,20 @@ def should_build( def package_name() -> str: - with ( + return project_metadata( Path(__file__).parents[1] / "packages" / "halide-llvm" / "pyproject.toml" - ).open("rb") as file: - return tomllib.load(file)["project"]["name"] + )["name"] def github_release_asset_names(project: str) -> list[str]: names: list[str] = [] - page = 1 - while True: - request = urllib.request.Request( - f"https://api.github.com/repos/{os.environ['GITHUB_REPOSITORY']}/releases?per_page=100&page={page}", - headers={ - "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}", - "Accept": "application/vnd.github+json", - }, - ) - with urllib.request.urlopen(request, timeout=30) as response: - releases = json.load(response) - if not releases: - return names - for release in releases: - if release["tag_name"].startswith(f"{project}@"): - names.extend(asset["name"] for asset in release.get("assets", [])) - page += 1 + releases = GitHubReleases( + os.environ["GITHUB_REPOSITORY"], os.environ.get("GITHUB_TOKEN") + ) + for release in releases.releases(): + if release["tag_name"].startswith(f"{project}@"): + names.extend(asset["name"] for asset in release.get("assets", [])) + return names def main() -> None: diff --git a/scripts/github_releases.py b/scripts/github_releases.py new file mode 100644 index 0000000..531f4ac --- /dev/null +++ b/scripts/github_releases.py @@ -0,0 +1,40 @@ +"""Small GitHub Releases API client shared by release tooling.""" + +from __future__ import annotations + +import json +import urllib.error +import urllib.request +from collections.abc import Iterator +from typing import Any + + +class GitHubReleases: + def __init__(self, repository: str, token: str | None = None) -> None: + self.repository = repository + self.token = token + + def get(self, path: str) -> Any: + request = urllib.request.Request(f"https://api.github.com{path}") + request.add_header("Accept", "application/vnd.github+json") + if self.token: + request.add_header("Authorization", f"Bearer {self.token}") + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + + def releases(self) -> Iterator[dict]: + page = 1 + while batch := self.get( + f"/repos/{self.repository}/releases?per_page=100&page={page}" + ): + yield from batch + page += 1 + + def release_exists(self, tag: str) -> bool: + try: + self.get(f"/repos/{self.repository}/releases/tags/{tag}") + except urllib.error.HTTPError as error: + if error.code == 404: + return False + raise + return True diff --git a/scripts/package_metadata.py b/scripts/package_metadata.py new file mode 100644 index 0000000..3696f55 --- /dev/null +++ b/scripts/package_metadata.py @@ -0,0 +1,22 @@ +"""Read and validate static Python project metadata.""" + +from __future__ import annotations + +from pathlib import Path + +import tomllib + + +def project_metadata(path: Path) -> dict: + """Return the ``[project]`` table from a pyproject.toml file.""" + with path.open("rb") as file: + return tomllib.load(file)["project"] + + +def project_identity(path: Path) -> tuple[str, str]: + """Return a project’s static name and version, or raise a clear error.""" + project = project_metadata(path) + name, version = project.get("name"), project.get("version") + if not name or not version: + raise ValueError(f"{path}: project name and version are required") + return name, version diff --git a/scripts/validate_package_metadata.py b/scripts/validate_package_metadata.py index ab01e46..a9dc60b 100755 --- a/scripts/validate_package_metadata.py +++ b/scripts/validate_package_metadata.py @@ -6,14 +6,11 @@ import sys from pathlib import Path -import tomllib +from package_metadata import project_identity def validate(path: Path) -> None: - with path.open("rb") as file: - project = tomllib.load(file)["project"] - if not project.get("name") or not project.get("version"): - raise ValueError(f"{path}: project name and version are required") + project_identity(path) def main(arguments: list[str] | None = None) -> None: diff --git a/tests/test_github_releases.py b/tests/test_github_releases.py new file mode 100644 index 0000000..e554eb9 --- /dev/null +++ b/tests/test_github_releases.py @@ -0,0 +1,41 @@ +import sys +import unittest +import urllib.error +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[1] / "scripts")) +from github_releases import GitHubReleases + + +class GitHubReleasesTest(unittest.TestCase): + def test_releases_paginates_until_an_empty_page(self): + client = GitHubReleases("halide/pypi") + paths = [] + + def get(path): + paths.append(path) + return [[{"tag_name": "one"}], [{"tag_name": "two"}], []][len(paths) - 1] + + client.get = get + + self.assertEqual( + [release["tag_name"] for release in client.releases()], ["one", "two"] + ) + self.assertEqual(len(paths), 3) + + def test_release_exists_only_swallows_not_found(self): + client = GitHubReleases("halide/pypi") + error = urllib.error.HTTPError( + "https://example.test", 404, "not found", {}, None + ) + + def get(_): + raise error + + client.get = get + self.assertFalse(client.release_exists("missing@1")) + error.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_validate_package_metadata.py b/tests/test_validate_package_metadata.py index f8a9a5a..276b906 100644 --- a/tests/test_validate_package_metadata.py +++ b/tests/test_validate_package_metadata.py @@ -18,6 +18,14 @@ def test_requires_name_and_version(self): path.write_text('[project]\nname = "example"\nversion = "1.0"\n') validate_package_metadata.validate(path) + def test_reads_project_identity(self): + with tempfile.TemporaryDirectory() as temporary_directory: + path = Path(temporary_directory) / "pyproject.toml" + path.write_text('[project]\nname = "example"\nversion = "1.0"\n') + self.assertEqual( + validate_package_metadata.project_identity(path), ("example", "1.0") + ) + if __name__ == "__main__": unittest.main() From a6432e631968c226db7f8881f2a7c169afaf28c4 Mon Sep 17 00:00:00 2001 From: Alex Reinking Date: Sat, 8 Aug 2026 19:03:25 -0400 Subject: [PATCH 2/4] Centralize wheel build platforms and steps --- .github/actions/build-wheel/action.yml | 136 +++++++++++++++++++++++++ .github/workflows/build-dependency.yml | 93 ++--------------- .github/workflows/build-llvm.yml | 132 +++--------------------- scripts/check_deps.py | 40 +------- scripts/check_llvm.py | 5 + scripts/platforms.py | 61 +++++++++++ tests/test_check_deps.py | 4 +- tests/test_check_llvm.py | 6 ++ 8 files changed, 237 insertions(+), 240 deletions(-) create mode 100644 .github/actions/build-wheel/action.yml create mode 100644 scripts/platforms.py diff --git a/.github/actions/build-wheel/action.yml b/.github/actions/build-wheel/action.yml new file mode 100644 index 0000000..3b391c0 --- /dev/null +++ b/.github/actions/build-wheel/action.yml @@ -0,0 +1,136 @@ +name: Build wheel +description: Build, retag, repair, and upload a wheel for one platform. + +inputs: + package: + required: true + platform: + required: true + artifact-name: + required: true + container: + required: false + default: "" + docker-image: + required: false + default: "" + manylinux-plat: + required: false + default: "" + msvc-arch: + required: false + default: "" + wheel-plat: + required: false + default: "" + pin-gcc12: + required: false + default: "false" + toolchain: + required: false + default: "" + llvm-ref: + required: false + default: "" + +runs: + using: composite + steps: + - name: Set up GCC 12 and Python (manylinux) + if: inputs.container != '' + shell: bash + run: | + yum install -y gcc-toolset-12-gcc-c++ + echo "/opt/rh/gcc-toolset-12/root/usr/bin" >> "$GITHUB_PATH" + echo "/opt/python/cp312-cp312/bin" >> "$GITHUB_PATH" + + - name: Set up Python + if: inputs.container == '' && inputs.docker-image == '' + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Set up MSVC + if: inputs.msvc-arch != '' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: ${{ inputs.msvc-arch }} + + - name: Build wheel + if: inputs.docker-image == '' + shell: bash + env: + PACKAGE: ${{ inputs.package }} + TOOLCHAIN: ${{ inputs.toolchain }} + HALIDE_LLVM_REF: ${{ inputs.llvm-ref }} + run: | + pip install cmake ninja + arguments=("$PACKAGE" -w dist/ -v) + if [[ -n "$TOOLCHAIN" ]]; then + arguments+=("--config-settings=cmake.define.CMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/$PACKAGE/toolchains/$TOOLCHAIN") + fi + pip wheel "${arguments[@]}" + + - name: Build wheel (docker) + if: inputs.docker-image != '' + shell: bash + env: + PACKAGE: ${{ inputs.package }} + DOCKER_IMAGE: ${{ inputs.docker-image }} + PIN_GCC12: ${{ inputs.pin-gcc12 }} + TOOLCHAIN: ${{ inputs.toolchain }} + HALIDE_LLVM_REF: ${{ inputs.llvm-ref }} + GITHUB_TOKEN: ${{ github.token }} + run: | + docker run --rm -v "${{ github.workspace }}:/project" -w /project \ + -e PACKAGE -e TOOLCHAIN -e HALIDE_LLVM_REF -e GITHUB_TOKEN -e PIN_GCC12 \ + "$DOCKER_IMAGE" bash -c ' + set -euo pipefail + if [ "$PIN_GCC12" = true ]; then + yum install -y gcc-toolset-12-gcc-c++; + export PATH="/opt/rh/gcc-toolset-12/root/usr/bin:$PATH"; + fi + export PATH="/opt/python/cp312-cp312/bin:$PATH" + pip install cmake ninja + arguments=("$PACKAGE" -w dist/ -v) + if [ -n "$TOOLCHAIN" ]; then + arguments+=("--config-settings=cmake.define.CMAKE_TOOLCHAIN_FILE=/project/$PACKAGE/toolchains/$TOOLCHAIN") + fi + pip wheel "${arguments[@]}" + ' + + - name: Retag wheel + if: inputs.wheel-plat != '' + shell: bash + run: | + pip install wheel + wheel tags --platform-tag ${{ inputs.wheel-plat }} --remove dist/*.whl + + - name: Repair wheel (auditwheel) + if: inputs.manylinux-plat != '' && inputs.docker-image == '' + shell: bash + run: | + pip install auditwheel + auditwheel repair --plat ${{ inputs.manylinux-plat }} -w dist/ dist/*.whl + rm -f dist/*-linux_*.whl + + - name: Repair wheel (auditwheel/docker) + if: inputs.manylinux-plat != '' && inputs.docker-image != '' + shell: bash + env: + DOCKER_IMAGE: ${{ inputs.docker-image }} + run: | + docker run --rm -v "${{ github.workspace }}:/project" -w /project \ + "$DOCKER_IMAGE" bash -c ' + set -euo pipefail + export PATH="/opt/python/cp312-cp312/bin:$PATH" + pip install auditwheel + auditwheel repair --plat ${{ inputs.manylinux-plat }} -w dist/ dist/*.whl + rm -f dist/*-linux_*.whl + ' + + - name: Upload wheel + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact-name }} + path: dist/*.whl diff --git a/.github/workflows/build-dependency.yml b/.github/workflows/build-dependency.yml index 181fc3a..bf1aca6 100644 --- a/.github/workflows/build-dependency.yml +++ b/.github/workflows/build-dependency.yml @@ -44,90 +44,17 @@ jobs: with: submodules: recursive - - name: Set up GCC 12 and Python (manylinux) - if: matrix.container - run: | - yum install -y gcc-toolset-12-gcc-c++ - echo "/opt/rh/gcc-toolset-12/root/usr/bin" >> "$GITHUB_PATH" - echo "/opt/python/cp312-cp312/bin" >> "$GITHUB_PATH" - - - name: Set up Python - if: ${{ !matrix.container && !matrix.docker_image }} - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Set up MSVC - if: matrix.msvc_arch - uses: ilammy/msvc-dev-cmd@v1 - with: - arch: ${{ matrix.msvc_arch }} - - - name: Install CMake and Ninja - if: ${{ !matrix.container && !matrix.docker_image }} - run: pip install cmake ninja - - - name: Build wheel - if: ${{ !matrix.docker_image }} - run: pip wheel ${{ inputs.package }} -w dist/ - - - name: Build wheel (docker) - if: matrix.docker_image - run: | - docker run --rm -v "${{ github.workspace }}:/project" -w /project \ - -e PIN_GCC12=${{ matrix.pin_gcc12 && 'true' || 'false' }} \ - ${{ matrix.docker_image }} bash -c ' - set -euo pipefail - if [ "$PIN_GCC12" = true ]; then - yum install -y gcc-toolset-12-gcc-c++ - export PATH="/opt/rh/gcc-toolset-12/root/usr/bin:$PATH" - fi - export PATH="/opt/python/cp312-cp312/bin:$PATH" - pip install cmake ninja - pip wheel ${{ inputs.package }} -w dist/ - ' - - - name: Retag wheel - if: matrix.wheel_plat - shell: bash - run: | - pip install wheel - wheel tags --platform-tag ${{ matrix.wheel_plat }} --remove dist/*.whl - - - name: Repair wheel (auditwheel) - if: ${{ matrix.manylinux_plat && !matrix.docker_image }} - run: | - pip install auditwheel - OUTPUT=$(auditwheel repair --plat ${{ matrix.manylinux_plat }} -w dist/ dist/*.whl 2>&1) || { - echo "$OUTPUT" - echo "$OUTPUT" | grep -q "does not look like a platform wheel" \ - && pip install wheel && wheel tags --platform-tag ${{ matrix.manylinux_plat }} --remove dist/*.whl \ - || exit 1 - } - rm -f dist/*-linux_*.whl - - - name: Repair wheel (auditwheel/docker) - if: ${{ matrix.manylinux_plat && matrix.docker_image }} - run: | - docker run --rm -v "${{ github.workspace }}:/project" -w /project \ - ${{ matrix.docker_image }} bash -c ' - set -euo pipefail - export PATH="/opt/python/cp312-cp312/bin:$PATH" - pip install auditwheel - OUTPUT=$(auditwheel repair --plat ${{ matrix.manylinux_plat }} -w dist/ dist/*.whl 2>&1) || { - echo "$OUTPUT" - echo "$OUTPUT" | grep -q "does not look like a platform wheel" \ - && pip install wheel && wheel tags --platform-tag ${{ matrix.manylinux_plat }} --remove dist/*.whl \ - || exit 1 - } - rm -f dist/*-linux_*.whl - ' - - - name: Upload wheel - uses: actions/upload-artifact@v4 + - uses: ./.github/actions/build-wheel with: - name: wheel-${{ matrix.pkg }}-${{ matrix.platform }} - path: dist/*.whl + package: ${{ inputs.package }} + platform: ${{ matrix.platform }} + artifact-name: wheel-${{ matrix.pkg }}-${{ matrix.platform }} + container: ${{ matrix.container }} + docker-image: ${{ matrix.docker_image }} + manylinux-plat: ${{ matrix.manylinux_plat }} + msvc-arch: ${{ matrix.msvc_arch }} + wheel-plat: ${{ matrix.wheel_plat }} + pin-gcc12: ${{ matrix.pin_gcc12 }} publish: name: Publish dependency release diff --git a/.github/workflows/build-llvm.yml b/.github/workflows/build-llvm.yml index 4ea0e89..a201560 100644 --- a/.github/workflows/build-llvm.yml +++ b/.github/workflows/build-llvm.yml @@ -24,6 +24,7 @@ jobs: outputs: should_build: ${{ steps.check.outputs.should_build }} llvm_ref: ${{ steps.check.outputs.llvm_ref }} + matrix: ${{ steps.check.outputs.matrix }} steps: - uses: actions/checkout@v4 @@ -44,47 +45,7 @@ jobs: contents: read strategy: fail-fast: false - matrix: - include: - - platform: x86-64-linux - runner: ubuntu-latest - container: quay.io/pypa/manylinux_2_28_x86_64 - toolchain: x86-64-linux.cmake - manylinux_plat: manylinux_2_28_x86_64 - - platform: x86-32-linux - runner: ubuntu-latest - toolchain: x86-32-linux.cmake - docker_image: quay.io/pypa/manylinux_2_28_i686 - manylinux_plat: manylinux_2_28_i686 - pin_gcc12: true - - platform: arm-64-linux - runner: ubuntu-24.04-arm - container: quay.io/pypa/manylinux_2_28_aarch64 - toolchain: arm-64-linux.cmake - manylinux_plat: manylinux_2_28_aarch64 - - platform: arm-32-linux - runner: ubuntu-24.04-arm - toolchain: arm-32-linux.cmake - docker_image: quay.io/pypa/manylinux_2_31_armv7l - manylinux_plat: manylinux_2_31_armv7l - - platform: x86-64-macos - runner: macos-15-intel - toolchain: x86-64-macos.cmake - - platform: arm-64-macos - runner: macos-15 - toolchain: arm-64-macos.cmake - - platform: x86-64-windows - runner: windows-2022 - toolchain: x86-64-windows.cmake - msvc_arch: amd64 - - platform: x86-32-windows - runner: windows-2022 - toolchain: x86-32-windows.cmake - msvc_arch: amd64_x86 - wheel_plat: win32 - defaults: - run: - working-directory: packages/halide-llvm + matrix: ${{ fromJSON(needs.check.outputs.matrix) }} env: HALIDE_LLVM_REF: ${{ needs.check.outputs.llvm_ref }} GITHUB_TOKEN: ${{ github.token }} @@ -95,83 +56,20 @@ jobs: # Keep GCC 12 for manylinux builds: newer libstdc++ symbols are not # repaired by auditwheel and break consumers with older toolchains. - - name: Set up GCC 12 and Python (manylinux) - if: matrix.container - run: | - yum install -y gcc-toolset-12-gcc-c++ - echo "/opt/rh/gcc-toolset-12/root/usr/bin" >> "$GITHUB_PATH" - echo "/opt/python/cp312-cp312/bin" >> "$GITHUB_PATH" - - - name: Set up Python - if: ${{ !matrix.container && !matrix.docker_image }} - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - # VS 2022 is intentionally pinned to match the toolset used by Halide. - - name: Set up MSVC - if: matrix.msvc_arch - uses: ilammy/msvc-dev-cmd@v1 - with: - arch: ${{ matrix.msvc_arch }} - - - name: Build wheel - if: ${{ !matrix.docker_image }} - run: > - pip wheel . -w dist/ -v - --config-settings=cmake.define.CMAKE_TOOLCHAIN_FILE=toolchains/${{ matrix.toolchain }} - - - name: Retag wheel - if: matrix.wheel_plat - shell: bash - run: | - pip install wheel - wheel tags --platform-tag ${{ matrix.wheel_plat }} --remove dist/*.whl - - # i686 and armv7l use Docker because the manylinux images cannot run - # actions/checkout's Node runtime on those architectures. - - name: Build wheel (docker) - if: matrix.docker_image - run: | - docker run --rm \ - -v "${{ github.workspace }}:/project" \ - -w /project/packages/halide-llvm \ - -e HALIDE_LLVM_REF -e GITHUB_TOKEN \ - -e PIN_GCC12=${{ matrix.pin_gcc12 && 'true' || 'false' }} \ - ${{ matrix.docker_image }} bash -c ' - set -euo pipefail - if [ "$PIN_GCC12" = true ]; then - yum install -y gcc-toolset-12-gcc-c++ - export PATH="/opt/rh/gcc-toolset-12/root/usr/bin:$PATH" - fi - export PATH="/opt/python/cp312-cp312/bin:$PATH" - pip wheel . -w dist/ -v --config-settings=cmake.define.CMAKE_TOOLCHAIN_FILE=toolchains/${{ matrix.toolchain }} - ' - - - name: Repair wheel (auditwheel) - if: ${{ matrix.manylinux_plat && !matrix.docker_image }} - run: | - pip install auditwheel - auditwheel repair --plat ${{ matrix.manylinux_plat }} -w dist/ dist/*.whl - rm -f dist/*-linux_*.whl - - - name: Repair wheel (auditwheel/docker) - if: ${{ matrix.manylinux_plat && matrix.docker_image }} - run: | - docker run --rm -v "${{ github.workspace }}:/project" \ - -w /project/packages/halide-llvm ${{ matrix.docker_image }} bash -c ' - set -euo pipefail - export PATH="/opt/python/cp312-cp312/bin:$PATH" - pip install auditwheel - auditwheel repair --plat ${{ matrix.manylinux_plat }} -w dist/ dist/*.whl - rm -f dist/*-linux_*.whl - ' - - - name: Upload wheel - uses: actions/upload-artifact@v4 + # VS 2022 remains pinned through the canonical LLVM matrix. + - uses: ./.github/actions/build-wheel with: - name: wheel-${{ matrix.platform }} - path: packages/halide-llvm/dist/*.whl + package: packages/halide-llvm + platform: ${{ matrix.platform }} + artifact-name: wheel-${{ matrix.pkg }}-${{ matrix.platform }} + container: ${{ matrix.container }} + docker-image: ${{ matrix.docker_image }} + manylinux-plat: ${{ matrix.manylinux_plat }} + msvc-arch: ${{ matrix.msvc_arch }} + wheel-plat: ${{ matrix.wheel_plat }} + pin-gcc12: ${{ matrix.pin_gcc12 }} + toolchain: ${{ matrix.toolchain }} + llvm-ref: ${{ needs.check.outputs.llvm_ref }} publish: name: Publish LLVM release diff --git a/scripts/check_deps.py b/scripts/check_deps.py index f2f0621..b4984d0 100755 --- a/scripts/check_deps.py +++ b/scripts/check_deps.py @@ -11,43 +11,7 @@ from github_releases import GitHubReleases from package_metadata import project_identity as metadata_project_identity - -PLATFORMS = ( - { - "platform": "x86-64-linux", - "runner": "ubuntu-latest", - "container": "quay.io/pypa/manylinux_2_28_x86_64", - "manylinux_plat": "manylinux_2_28_x86_64", - }, - { - "platform": "x86-32-linux", - "runner": "ubuntu-latest", - "docker_image": "quay.io/pypa/manylinux_2_28_i686", - "manylinux_plat": "manylinux_2_28_i686", - "pin_gcc12": True, - }, - { - "platform": "arm-64-linux", - "runner": "ubuntu-24.04-arm", - "container": "quay.io/pypa/manylinux_2_28_aarch64", - "manylinux_plat": "manylinux_2_28_aarch64", - }, - { - "platform": "arm-32-linux", - "runner": "ubuntu-24.04-arm", - "docker_image": "quay.io/pypa/manylinux_2_31_armv7l", - "manylinux_plat": "manylinux_2_31_armv7l", - }, - {"platform": "x86-64-macos", "runner": "macos-15-intel"}, - {"platform": "arm-64-macos", "runner": "macos-15"}, - {"platform": "x86-64-windows", "runner": "windows-latest", "msvc_arch": "amd64"}, - { - "platform": "x86-32-windows", - "runner": "windows-latest", - "msvc_arch": "amd64_x86", - "wheel_plat": "win32", - }, -) +from platforms import wheel_matrix def project_identity(package_dir: Path) -> tuple[str, str]: @@ -64,7 +28,7 @@ def missing_package_matrix( needed = not release_exists(tag) print(f"{tag}: {'needed' if needed else 'already released'}") if needed: - matrix.extend({"pkg": package.name, **platform} for platform in PLATFORMS) + matrix.extend(wheel_matrix(package.name)) return matrix diff --git a/scripts/check_llvm.py b/scripts/check_llvm.py index 353c2fc..65ebde8 100755 --- a/scripts/check_llvm.py +++ b/scripts/check_llvm.py @@ -3,6 +3,7 @@ from __future__ import annotations +import json import os import sys from collections.abc import Callable, Iterable @@ -10,6 +11,7 @@ from github_releases import GitHubReleases from package_metadata import project_metadata +from platforms import wheel_matrix sys.path.insert(0, str(Path(__file__).parents[1] / "packages" / "halide-llvm")) from _version_provider import get_commit_info, version_from_tag @@ -57,6 +59,9 @@ def main() -> None: print(f"{resolved_ref}: should_build={str(build).lower()}") with open(os.environ["GITHUB_OUTPUT"], "a") as output: output.write(f"should_build={str(build).lower()}\nllvm_ref={resolved_ref}\n") + output.write( + f"matrix={json.dumps({'include': wheel_matrix(package_name(), llvm=True)}, separators=(',', ':'))}\n" + ) if __name__ == "__main__": diff --git a/scripts/platforms.py b/scripts/platforms.py new file mode 100644 index 0000000..311077d --- /dev/null +++ b/scripts/platforms.py @@ -0,0 +1,61 @@ +"""Canonical wheel-build platform definitions.""" + +from __future__ import annotations + +PLATFORMS = ( + { + "platform": "x86-64-linux", + "runner": "ubuntu-latest", + "container": "quay.io/pypa/manylinux_2_28_x86_64", + "manylinux_plat": "manylinux_2_28_x86_64", + }, + { + "platform": "x86-32-linux", + "runner": "ubuntu-latest", + "docker_image": "quay.io/pypa/manylinux_2_28_i686", + "manylinux_plat": "manylinux_2_28_i686", + "pin_gcc12": True, + }, + { + "platform": "arm-64-linux", + "runner": "ubuntu-24.04-arm", + "container": "quay.io/pypa/manylinux_2_28_aarch64", + "manylinux_plat": "manylinux_2_28_aarch64", + }, + { + "platform": "arm-32-linux", + "runner": "ubuntu-24.04-arm", + "docker_image": "quay.io/pypa/manylinux_2_31_armv7l", + "manylinux_plat": "manylinux_2_31_armv7l", + }, + {"platform": "x86-64-macos", "runner": "macos-15-intel"}, + {"platform": "arm-64-macos", "runner": "macos-15"}, + {"platform": "x86-64-windows", "runner": "windows-latest", "msvc_arch": "amd64"}, + { + "platform": "x86-32-windows", + "runner": "windows-latest", + "msvc_arch": "amd64_x86", + "wheel_plat": "win32", + }, +) + + +def wheel_matrix(package: str, *, llvm: bool = False) -> list[dict[str, object]]: + """Return the CI matrix, with the LLVM toolchain-specific additions.""" + matrix = [] + for entry in PLATFORMS: + item = {"pkg": package, **entry} + if llvm: + item["toolchain"] = f"{item['platform']}.cmake" + if item["platform"].endswith("windows"): + item["runner"] = "windows-2022" + matrix.append(item) + return matrix + + +def platform(platform_name: str) -> dict[str, object]: + """Return one canonical platform definition.""" + for entry in PLATFORMS: + if entry["platform"] == platform_name: + return dict(entry) + raise ValueError(f"unknown platform: {platform_name}") diff --git a/tests/test_check_deps.py b/tests/test_check_deps.py index f95a81e..29591a2 100644 --- a/tests/test_check_deps.py +++ b/tests/test_check_deps.py @@ -27,11 +27,11 @@ def test_only_missing_projects_get_platform_matrix_entries(self): lambda tag: tag == "halide-flatbuffers@23.5.26", ) - self.assertEqual(len(check_deps.PLATFORMS), len(matrix)) + self.assertEqual(len(check_deps.wheel_matrix("unused")), len(matrix)) self.assertEqual({entry["pkg"] for entry in matrix}, {"wabt"}) self.assertEqual( {entry["platform"] for entry in matrix}, - {platform["platform"] for platform in check_deps.PLATFORMS}, + {platform["platform"] for platform in check_deps.wheel_matrix("unused")}, ) diff --git a/tests/test_check_llvm.py b/tests/test_check_llvm.py index aa80cd6..7b3997f 100644 --- a/tests/test_check_llvm.py +++ b/tests/test_check_llvm.py @@ -25,6 +25,12 @@ def test_dev_ref_uses_resolved_commit_prefix(self): self.assertFalse(build) self.assertEqual(resolved, "deadbeef0123456789") + def test_llvm_matrix_uses_the_pinned_windows_runner_and_toolchains(self): + matrix = check_llvm.wheel_matrix("halide-llvm", llvm=True) + windows = next(item for item in matrix if item["platform"] == "x86-64-windows") + self.assertEqual(windows["runner"], "windows-2022") + self.assertEqual(windows["toolchain"], "x86-64-windows.cmake") + if __name__ == "__main__": unittest.main() From 8cf8c0d72c6c5cbbcf7c7628104c1d167feeb177 Mon Sep 17 00:00:00 2001 From: Alex Reinking Date: Sat, 8 Aug 2026 19:05:12 -0400 Subject: [PATCH 3/4] Reuse CI platform settings for local LLVM builds --- .github/workflows/validate.yml | 4 ++-- README.md | 5 +++-- packages/halide-llvm/local-build.sh | 32 +++++++++++------------------ scripts/platforms.py | 28 +++++++++++++++++++++++++ tests/test_check_deps.py | 10 +++++++++ 5 files changed, 55 insertions(+), 24 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 30df443..c4d3997 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -4,7 +4,7 @@ on: workflow_call: {} push: paths: - - .github/actions/publish-releases/** + - .github/actions/** - .github/workflows/** - .gitmodules - generate_index.py @@ -15,7 +15,7 @@ on: - packages/**/toolchains/** pull_request: paths: - - .github/actions/publish-releases/** + - .github/actions/** - .github/workflows/** - .gitmodules - generate_index.py diff --git a/README.md b/README.md index 0391788..8db8163 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,9 @@ them. each link's hash fragment. - `scripts/` contains the release checks, metadata validation, and shared publisher logic; `tests/` covers those scripts and index generation. -- `.github/workflows/build-dependency.yml` is the reusable build/publish - workflow used by the FlatBuffers and WABT trigger workflows. +- `scripts/platforms.py` defines the canonical platform matrix for CI and local + LLVM builds; `.github/actions/build-wheel/` performs the common build, + repair, and artifact-upload steps. All release tags have the form `@`, for example `halide-llvm@22.1.7`. The index generator discovers those tags automatically, diff --git a/packages/halide-llvm/local-build.sh b/packages/halide-llvm/local-build.sh index 6361332..2a1c6f1 100755 --- a/packages/halide-llvm/local-build.sh +++ b/packages/halide-llvm/local-build.sh @@ -106,27 +106,12 @@ run_local_macos_build() { run_linux_docker_build() { local platform="$1" - local image dist_dir + local image dist_dir manylinux_plat needs_gcc12 local toolchain="toolchains/$platform.cmake" - case "$platform" in - x86-64-linux) - image="quay.io/pypa/manylinux_2_28_x86_64" - ;; - x86-32-linux) - image="quay.io/pypa/manylinux_2_28_i686" - ;; - arm-64-linux) - image="quay.io/pypa/manylinux_2_28_aarch64" - ;; - arm-32-linux) - image="quay.io/pypa/manylinux_2_31_armv7l" - ;; - *) - echo "error: unsupported Linux Docker platform: $platform" >&2 - exit 1 - ;; - esac + image="$(python3 "$REPO_ROOT/scripts/platforms.py" "$platform" image)" + manylinux_plat="$(python3 "$REPO_ROOT/scripts/platforms.py" "$platform" manylinux_plat)" + needs_gcc12="$(python3 "$REPO_ROOT/scripts/platforms.py" "$platform" needs_gcc12)" dist_dir="dist/$platform" mkdir -p "$dist_dir" @@ -141,16 +126,22 @@ run_linux_docker_build() { -v "$(pwd):/project" \ -w /project \ -e "HALIDE_LLVM_REF=$HALIDE_LLVM_REF" \ + -e "NEEDS_GCC12=$needs_gcc12" \ "$image" \ bash -c " set -euo pipefail + if [ \"\$NEEDS_GCC12\" = true ]; then + yum install -y gcc-toolset-12-gcc-c++ + export PATH=/opt/rh/gcc-toolset-12/root/usr/bin:\$PATH + fi export PATH=/opt/python/cp312-cp312/bin:\$PATH + pip install cmake ninja pip wheel . -w $dist_dir/ -v \ --config-settings=cmake.define.CMAKE_TOOLCHAIN_FILE=$toolchain pip install auditwheel - auditwheel repair -w $dist_dir/ $dist_dir/*.whl + auditwheel repair --plat $manylinux_plat -w $dist_dir/ $dist_dir/*.whl rm -f $dist_dir/*-linux_*.whl echo @@ -168,6 +159,7 @@ if [[ -z "$REF" || "$#" -gt 2 ]]; then fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" cd "$SCRIPT_DIR" export HALIDE_LLVM_REF="$REF" diff --git a/scripts/platforms.py b/scripts/platforms.py index 311077d..cc570c3 100644 --- a/scripts/platforms.py +++ b/scripts/platforms.py @@ -2,6 +2,8 @@ from __future__ import annotations +import argparse + PLATFORMS = ( { "platform": "x86-64-linux", @@ -59,3 +61,29 @@ def platform(platform_name: str) -> dict[str, object]: if entry["platform"] == platform_name: return dict(entry) raise ValueError(f"unknown platform: {platform_name}") + + +def platform_value(platform_name: str, field: str) -> str: + """Return a shell-friendly platform property for local build tooling.""" + entry = platform(platform_name) + if field == "image": + value = entry.get("container") or entry.get("docker_image") + elif field == "needs_gcc12": + value = bool(entry.get("container") or entry.get("pin_gcc12")) + else: + value = entry.get(field) + if value is None: + raise ValueError(f"{platform_name}: no {field}") + return str(value).lower() if isinstance(value, bool) else str(value) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Query canonical wheel platforms") + parser.add_argument("platform") + parser.add_argument("field", choices=("image", "manylinux_plat", "needs_gcc12")) + arguments = parser.parse_args() + print(platform_value(arguments.platform, arguments.field)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_check_deps.py b/tests/test_check_deps.py index 29591a2..5dfc1ee 100644 --- a/tests/test_check_deps.py +++ b/tests/test_check_deps.py @@ -5,6 +5,7 @@ sys.path.insert(0, str(Path(__file__).parents[1] / "scripts")) import check_deps +import platforms class CheckDependenciesTest(unittest.TestCase): @@ -34,6 +35,15 @@ def test_only_missing_projects_get_platform_matrix_entries(self): {platform["platform"] for platform in check_deps.wheel_matrix("unused")}, ) + def test_local_build_properties_come_from_the_canonical_platform(self): + self.assertEqual( + platforms.platform_value("x86-64-linux", "image"), + "quay.io/pypa/manylinux_2_28_x86_64", + ) + self.assertEqual( + platforms.platform_value("x86-32-linux", "needs_gcc12"), "true" + ) + if __name__ == "__main__": unittest.main() From 8714a6bec6a423acb41bacbc6a283b0f79806774 Mon Sep 17 00:00:00 2001 From: Alex Reinking Date: Sat, 8 Aug 2026 19:06:04 -0400 Subject: [PATCH 4/4] Share manylinux wheel repair policy --- .github/actions/build-wheel/action.yml | 6 ++---- packages/halide-llvm/local-build.sh | 3 +-- scripts/repair-wheel.sh | 19 +++++++++++++++++++ 3 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 scripts/repair-wheel.sh diff --git a/.github/actions/build-wheel/action.yml b/.github/actions/build-wheel/action.yml index 3b391c0..3455af3 100644 --- a/.github/actions/build-wheel/action.yml +++ b/.github/actions/build-wheel/action.yml @@ -111,8 +111,7 @@ runs: shell: bash run: | pip install auditwheel - auditwheel repair --plat ${{ inputs.manylinux-plat }} -w dist/ dist/*.whl - rm -f dist/*-linux_*.whl + bash scripts/repair-wheel.sh ${{ inputs.manylinux-plat }} dist - name: Repair wheel (auditwheel/docker) if: inputs.manylinux-plat != '' && inputs.docker-image != '' @@ -125,8 +124,7 @@ runs: set -euo pipefail export PATH="/opt/python/cp312-cp312/bin:$PATH" pip install auditwheel - auditwheel repair --plat ${{ inputs.manylinux-plat }} -w dist/ dist/*.whl - rm -f dist/*-linux_*.whl + bash scripts/repair-wheel.sh ${{ inputs.manylinux-plat }} dist ' - name: Upload wheel diff --git a/packages/halide-llvm/local-build.sh b/packages/halide-llvm/local-build.sh index 2a1c6f1..4ec0d5c 100755 --- a/packages/halide-llvm/local-build.sh +++ b/packages/halide-llvm/local-build.sh @@ -141,8 +141,7 @@ run_linux_docker_build() { --config-settings=cmake.define.CMAKE_TOOLCHAIN_FILE=$toolchain pip install auditwheel - auditwheel repair --plat $manylinux_plat -w $dist_dir/ $dist_dir/*.whl - rm -f $dist_dir/*-linux_*.whl + bash /project/scripts/repair-wheel.sh $manylinux_plat $dist_dir echo echo 'Built wheels:' diff --git a/scripts/repair-wheel.sh b/scripts/repair-wheel.sh new file mode 100644 index 0000000..19c056b --- /dev/null +++ b/scripts/repair-wheel.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Repair a wheel for manylinux, retagging pure wheels when auditwheel declines. + +set -euo pipefail + +platform="$1" +dist_dir="$2" + +if output="$(auditwheel repair --plat "$platform" -w "$dist_dir" "$dist_dir"/*.whl 2>&1)"; then + echo "$output" +else + echo "$output" + if ! grep -q "does not look like a platform wheel" <<<"$output"; then + exit 1 + fi + pip install wheel + wheel tags --platform-tag "$platform" --remove "$dist_dir"/*.whl +fi +rm -f "$dist_dir"/*-linux_*.whl