diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 771687b..0479d57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,9 +87,13 @@ jobs: name: native distributions uses: ./.github/workflows/wheels.yml + conda: + name: native conda distributions + uses: ./.github/workflows/conda.yml + ci-required: name: ci-required - needs: [lint, test-cpu, package] + needs: [lint, test-cpu, package, conda] if: ${{ always() }} runs-on: ubuntu-24.04 timeout-minutes: 5 @@ -99,7 +103,8 @@ jobs: LINT_RESULT: ${{ needs.lint.result }} CPU_RESULT: ${{ needs.test-cpu.result }} PACKAGE_RESULT: ${{ needs.package.result }} + CONDA_RESULT: ${{ needs.conda.result }} run: | - for result in "${LINT_RESULT}" "${CPU_RESULT}" "${PACKAGE_RESULT}"; do + for result in "${LINT_RESULT}" "${CPU_RESULT}" "${PACKAGE_RESULT}" "${CONDA_RESULT}"; do test "${result}" = success || exit 1 done diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml new file mode 100644 index 0000000..1088e86 --- /dev/null +++ b/.github/workflows/conda.yml @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +name: native conda packages + +on: + workflow_call: + workflow_dispatch: + +permissions: + contents: read + +jobs: + source: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + fetch-depth: 0 + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.12.1" + enable-cache: false + - run: CUPHOTON_XDR_BUILD_EXT=0 uv build --sdist + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cuphoton-conda-source + path: dist/*.tar.gz + if-no-files-found: error + + build: + needs: source + strategy: + fail-fast: false + matrix: + python: ["3.12", "3.13", "3.14"] + platform: + - runner: ubuntu-24.04 + subdir: linux-64 + pixi-target: x86_64-unknown-linux-musl + pixi-sha256: a40e0cb519c681b76cf5ae3a2bdb72e98edde6b1a72cc6a2469edaa4f2c68dca + - runner: ubuntu-24.04-arm + subdir: linux-aarch64 + pixi-target: aarch64-unknown-linux-musl + pixi-sha256: bac5949b12ec65d76ea5a9d22b67340b2447cf6ccaa34deac1386430b45b5231 + runs-on: ${{ matrix.platform.runner }} + timeout-minutes: 30 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - name: Install pinned Pixi + env: + PIXI_TARGET: ${{ matrix.platform.pixi-target }} + PIXI_SHA256: ${{ matrix.platform.pixi-sha256 }} + run: | + mkdir -p "${RUNNER_TEMP}/pixi-bin" + curl --fail --location --retry 3 \ + "https://github.com/prefix-dev/pixi/releases/download/v0.62.2/pixi-${PIXI_TARGET}" \ + --output "${RUNNER_TEMP}/pixi-bin/pixi" + printf '%s %s\n' "${PIXI_SHA256}" "${RUNNER_TEMP}/pixi-bin/pixi" | sha256sum --check + chmod +x "${RUNNER_TEMP}/pixi-bin/pixi" + echo "${RUNNER_TEMP}/pixi-bin" >> "${GITHUB_PATH}" + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cuphoton-conda-source + path: dist + - name: Build and test in isolated conda environments + env: + PYTHON_VERSION: ${{ matrix.python }} + run: >- + pixi exec --spec rattler-build=0.76.1 --spec python=3.12 -- + python scripts/conda/build.py dist/*.tar.gz + --python "$PYTHON_VERSION" --output-dir dist/conda + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cuphoton-conda-${{ matrix.platform.subdir }}-py${{ matrix.python }} + path: | + dist/conda/${{ matrix.platform.subdir }}/cuphoton-*.conda + dist/conda/provenance.json + if-no-files-found: error + retention-days: 30 diff --git a/MANIFEST.in b/MANIFEST.in index b00237b..518d840 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -14,6 +14,8 @@ prune .gitlab prune scripts include scripts/cuphoton-openmpi-rank-exec recursive-include scripts/wheels *.sh *.py *.txt +recursive-include scripts/conda *.py +recursive-include packaging/conda *.yaml *.sh include LICENSE include CITATION.cff include Makefile diff --git a/Makefile b/Makefile index 3fbde64..d065480 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -.PHONY: sync sync-gpu sync-cutile lock lock-check lint format test test-cpu test-core test-xdr test-xfit test-xfit-real test-xpois test-xscan test-xrep test-xray test-gpu test-cutile clean-dist build package-check wheels release-check ci-lint ci-test-cpu hooks +.PHONY: sync sync-gpu sync-cutile lock lock-check lint format test test-cpu test-core test-xdr test-xfit test-xfit-real test-xpois test-xscan test-xrep test-xray test-gpu test-cutile clean-dist build package-check wheels conda release-check ci-lint ci-test-cpu hooks CPU_EXTRAS = --extra dev --extra torch --extra viz --extra photometry GPU_EXTRAS = --extra dev --extra gpu --extra viz @@ -72,14 +72,20 @@ test-cutile: clean-dist: rm -rf dist -build: clean-dist +build: + rm -f dist/cuphoton-*.tar.gz CUPHOTON_XDR_BUILD_EXT=0 uv build --sdist wheels: build + rm -f dist/cuphoton-*.whl uv tool run --from cibuildwheel==4.2.1 cibuildwheel --platform linux --output-dir dist dist/*.tar.gz +conda: build + rm -rf dist/conda + pixi exec --spec rattler-build=0.76.1 --spec python=3.12 -- python scripts/conda/build.py dist/*.tar.gz + package-check: build - uvx --isolated --from twine==6.2.0 twine check --strict dist/* + uvx --isolated --from twine==6.2.0 twine check --strict dist/*.tar.gz release-check: $(MAKE) lock-check @@ -87,7 +93,7 @@ release-check: $(MAKE) ci-test-cpu $(MAKE) wheels python scripts/wheels/check_distributions.py dist --arch "$$(uname -m)" - uvx --isolated --from twine==6.2.0 twine check --strict dist/* + uvx --isolated --from twine==6.2.0 twine check --strict dist/*.tar.gz dist/*.whl ci-lint: $(MAKE) lint diff --git a/ROADMAP.md b/ROADMAP.md index 6d6550e..74faa22 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -19,7 +19,8 @@ institutional adaptation. - Publish pip packages after release automation and provenance checks are established. -- Add conda packaging with the same dependency-profile boundaries. +- Ship conda packaging for the native I/O profile. A CPU-only conda package + and separate dependency profiles remain future work. - Keep experimental toolchain integrations, such as cuTile, isolated from the core install. diff --git a/docs/packaging.md b/docs/packaging.md index 950b3ed..12f3326 100644 --- a/docs/packaging.md +++ b/docs/packaging.md @@ -1,21 +1,23 @@ -# Native wheels +# Native packages Release artifacts are six Linux wheels (CPython 3.12, 3.13, and 3.14 on x86-64 and ARM64) plus one source archive. Wheels target glibc 2.28 or later. The runtime dependencies may impose a newer glibc floor; the installed-wheel -CI tests use Debian 12. Free-threaded Python, Windows, macOS, and conda builds -are outside this matrix. +CI tests use Debian 12. Native conda builds cover the same six Python and +architecture combinations, as described below. Free-threaded Python, Windows, +and macOS are outside both matrices. Each wheel contains `cuphoton.xdr._nvcomp_batch_ext` and a privately renamed, reentrant CFITSIO 4.7.0 shared library. CUDA, cuFile, KvikIO, and nvCOMP remain in their upstream wheels, installed through `cuphoton[io]`. `cuphoton[gpu]` -also includes the photometry, PyTorch, and Numba backends. Photutils currently -requires a source build and C compiler on ARM64; `cuphoton` and `cuphoton[io]` -do not install it. +also includes the photometry, PyTorch, and Numba backends. Installing Photutils +from PyPI currently requires a source build and C compiler on ARM64; +`cuphoton` and `cuphoton[io]` do not install it. Conda provides ARM64 Photutils +binaries. The native KvikIO ABI is restricted to the 26.6 release family. nvCOMP is restricted to 5.2. Updating either family requires rebuilding and qualifying -the native wheels. The CUDA SDK build inputs are pinned to 13.0 so a newer +the native packages. The CUDA SDK build inputs are pinned to 13.0 so a newer build environment cannot silently raise the runtime floor. ## Versions and release candidates @@ -51,14 +53,14 @@ Add `viz` for visualization dependencies. Use `--pre` when selecting the newest available prerelease instead of pinning one. Each changed candidate needs a new RC number: PyPI does not allow replacing an uploaded filename. -## Build +## Build wheels From the checkout, on each native Linux architecture with Docker and uv: ```bash make wheels python scripts/wheels/check_distributions.py dist --arch "$(uname -m)" -uvx --from twine==6.2.0 twine check --strict dist/* +uvx --from twine==6.2.0 twine check --strict dist/*.tar.gz dist/*.whl ``` `make wheels` builds a source archive, then uses cibuildwheel 4.2.1 to build @@ -73,6 +75,12 @@ editable installs remain Python-only unless `CUPHOTON_XDR_BUILD_EXT=1` is set. See [XDR source installation](components/xdr.md#native-extension-availability) for the explicit native development build. +`make build` refreshes only the source archive. `make wheels` replaces cuPhoton +wheels while preserving conda outputs; `make conda` replaces `dist/conda` while +preserving wheels. Use `make clean-dist` explicitly to remove all distributions. +To reuse the wheel build's exact archive for conda, use the direct conda build +command below. + The reusable `wheels.yml` workflow builds from one source archive on native x86-64 and ARM64 runners. It checks base imports inside cibuildwheel, then installs each wheel and its `io` dependencies in a clean Python container @@ -88,7 +96,99 @@ These workers solve generated xPOIS inputs on the CPU and check numerical results, distinct processes, and MPI collectives. JSON receipts are retained as CI artifacts. These checks do not establish GPU executor correctness. -## GPU qualification +## Build and install conda packages + +The `cuphoton` conda package includes the compiled XDR extension and depends on +upstream conda packages for CFITSIO 4.7, CUDA 13, KvikIO 26.6, and nvCOMP 5.2. +Conda installs those libraries into the environment; no manual CUDA paths +are needed. +Conda has no pip-style extras: this package provides the native I/O profile +only and requires CUDA 13 libraries. It does not provide a CPU-only install; +use the base pip package for that profile. + +With uv and pixi installed, run on each native Linux architecture: + +```bash +make conda +``` + +This builds the SCM-versioned source archive and uses rattler-build 0.76.1 to +produce Python 3.12, 3.13, and 3.14 packages. The archive supplies the same +version used by wheels, including RC versions, without requiring Git during +the conda build. To build one Python version from an existing source archive: + +```bash +pixi exec --spec rattler-build=0.76.1 --spec python=3.12 -- \ + python scripts/conda/build.py 'dist/cuphoton-.tar.gz' \ + --python 3.12 --output-dir dist/conda +``` + +Use an output directory without existing cuPhoton packages. Outputs go under +`linux-64` or `linux-aarch64`, with a `provenance.json` recording the source +archive and package SHA256 values. Builds use strict channel priority with +`rapidsai` before `conda-forge`, excluding `defaults`. This order is required +by the current installed-package solver; it also gives RAPIDS precedence for +other packages present in both channels. Review the resolved environment +before qualifying an artifact. +The qualified conda nvCOMP build is 5.2.0.10, while the wheel environment uses +5.2.0.13. Both stay within the required 5.2 ABI family and need independent +artifact qualification; their patch versions are not interchangeable evidence. + +Create a local channel index so conda resolves the artifact's runtime +dependencies. For downloaded CI artifacts, place the packages under +`dist/conda/linux-64` or `dist/conda/linux-aarch64` first. Select the version +and build string from the package filename: + +```bash +mkdir -p dist/conda/noarch +pixi exec --spec conda-index=0.13.0 -- python -m conda_index dist/conda +conda create --prefix ./conda-xdr --override-channels \ + --strict-channel-priority -c "$PWD/dist/conda" -c rapidsai -c conda-forge \ + 'python=3.12' 'cuphoton==' +``` + +Direct installation of a `.conda` filename skips dependency resolution; use +the indexed channel directory above. See [conda's installation +guidance](https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/installing-with-conda.html). +For broader GPU and photometry functionality, add the following packages. +The PyTorch build selector chooses its CUDA 13.0 build: + +```bash +conda install --prefix ./conda-xdr --override-channels \ + --strict-channel-priority -c "$PWD/dist/conda" -c rapidsai -c conda-forge \ + 'photutils>=3' 'numba>=0.61,<0.66' 'numba-cuda>=0.30,<0.31' \ + 'pytorch>=2.13,<3' 'pytorch=*=cuda130*' 'cuda-version=13.0' +``` + +Install development tools such as `pytest`, `ruff`, and `pre-commit` through +conda. Keep the repository's uv development environment separate: the current +conda PyTorch package requires `setuptools<82`, while the pip `dev` extra +requires `setuptools>=83`. For Python-only editable work in a separate conda +environment with the runtime dependencies and pip installed, run +`CUPHOTON_XDR_BUILD_EXT=0 conda run --prefix ./conda-dev python -m pip install --no-deps -e .`. +Native builds should use the recipe above. + +The reusable `conda.yml` workflow builds all six variants from one source +archive and retains the packages and provenance as CI artifacts. Its isolated +installation tests load the native extension and exercise CFITSIO planning +without a GPU or driver. Upstream conda GPU packages retain some pip-only +dependency names in their Python metadata, so `pip check` is not a conda +validation gate. The actual native and GPU tests are required. + +Qualify each exact conda artifact on its architecture and Python version with +a CUDA 13-compatible driver, using the installed-package GPU check: + +```bash +conda run --prefix ./conda-xdr python -I scripts/wheels/test_installed.py \ + --mode gpu --output conda-gpu.json +``` + +The check runs outside the source tree internally and exercises the same +decoding, ordering, concurrency, and buffer-lifetime cases as wheels. Retain +its JSON receipt and the tested artifact hash. Conda channel publication is +not configured; release-tag publishing below applies to PyPI artifacts. + +## Wheel GPU qualification CI CPU checks do not establish GPU correctness. Download the exact `cuphoton-distributions` artifact and test each wheel on its architecture and diff --git a/packaging/conda/build.sh b/packaging/conda/build.sh new file mode 100755 index 0000000..176a038 --- /dev/null +++ b/packaging/conda/build.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +export CUPHOTON_XDR_BUILD_EXT=1 +export CUPHOTON_XDR_NATIVE_PREFIX="$PREFIX" +export SETUPTOOLS_SCM_PRETEND_VERSION_FOR_CUPHOTON="$PKG_VERSION" +"$PYTHON" -m pip install . --no-deps --no-build-isolation --no-index -v diff --git a/packaging/conda/recipe.yaml b/packaging/conda/recipe.yaml new file mode 100644 index 0000000..e550859 --- /dev/null +++ b/packaging/conda/recipe.yaml @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +context: + version: ${{ env.get("CUPHOTON_CONDA_VERSION") }} + +package: + name: cuphoton + version: ${{ version }} + +# scripts/conda/build.py stages the versioned source archive here. +source: + path: ../source + +build: + number: 0 + skip: not linux + script: build.sh + +requirements: + build: + - ${{ compiler('cxx') }} + - ${{ stdlib('c') }} + host: + - python + - pip + - setuptools >=83 + - setuptools-scm ==10.3.4 + - wheel + - pybind11 >=3,<4 + - cuda-version ==13.0 + - cuda-cudart-dev + - cuda-crt-dev_${{ target_platform }} + - libcufile-dev + - cfitsio >=4.7,<5 + - libkvikio >=26.06,<26.07 + - libnvcomp-dev >=5.2,<5.3 + run: + - python + - astropy-base >=6.1.4 + - h5py >=3.10 + - numexpr >=2.10 + - numpy >=2.0,<2.6 + - pandas >=2.2 + - pyarrow >=23 + - pyyaml >=6 + - scipy >=1.13 + - cuda-version >=13,<14 + - cupy >=14,<15 + - kvikio >=26.06,<26.07 + - libkvikio >=26.06,<26.07 + - libnvcomp >=5.2,<5.3 + - nvcomp >=5.2,<5.3 + +tests: + - python: + imports: + - cuphoton + - cuphoton.xdr + # Upstream GPU packages retain pip-only dependency names in METADATA. + # Conda solves those dependencies; the next test loads the actual libs. + pip_check: false + - script: + - python -I scripts/wheels/test_installed.py --mode native + files: + source: + - scripts/wheels/test_installed.py + +about: + homepage: https://github.com/NVIDIA/cuPhoton + repository: https://github.com/NVIDIA/cuPhoton + documentation: https://github.com/NVIDIA/cuPhoton/blob/main/docs/packaging.md + summary: GPU-accelerated astronomy and imaging tools with native XDR I/O + license: Apache-2.0 + license_file: + - LICENSE + - THIRD_PARTY_NOTICES.md diff --git a/packaging/conda/variants.yaml b/packaging/conda/variants.yaml new file mode 100644 index 0000000..5825c92 --- /dev/null +++ b/packaging/conda/variants.yaml @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +python: + - "3.12" + - "3.13" + - "3.14" +cxx_compiler: + - gxx +cxx_compiler_version: + - "14" +c_stdlib: + - sysroot +c_stdlib_version: + - "2.28" diff --git a/scripts/conda/build.py b/scripts/conda/build.py new file mode 100644 index 0000000..29819e9 --- /dev/null +++ b/scripts/conda/build.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Build native conda packages from a versioned cuPhoton source archive.""" + +from __future__ import annotations + +import argparse +import email.parser +import hashlib +import json +import os +import shutil +import subprocess +import tarfile +import tempfile +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("sdist", type=Path) + parser.add_argument("--output-dir", type=Path, default=Path("dist/conda")) + parser.add_argument("--python", choices=("3.12", "3.13", "3.14")) + args = parser.parse_args() + archive = args.sdist.resolve() + output = args.output_dir.resolve() + if any(output.glob("linux-*/cuphoton-*.conda")): + parser.error( + "Use an output directory without existing cuPhoton packages" + ) + # Conda records build prefixes in its recipe, bytecode, and relocation + # metadata. Keep user-specific checkout/output paths out of the package. + with tempfile.TemporaryDirectory( + prefix="cuphoton-conda-", dir="/tmp" + ) as temporary: + stage = Path(temporary) + build_output = stage / "output" + with tarfile.open(archive) as source: + source.extractall(stage / "archive", filter="data") + (source_dir,) = (stage / "archive").iterdir() + metadata = email.parser.Parser().parsestr( + (source_dir / "PKG-INFO").read_text() + ) + if metadata["Name"] != "cuphoton" or not metadata["Version"]: + raise ValueError("Expected a versioned cuphoton source archive") + version = metadata["Version"] + source_dir.rename(stage / "source") + shutil.copytree(stage / "source/packaging/conda", stage / "recipe") + environment = os.environ.copy() + environment["CUPHOTON_CONDA_VERSION"] = version + # Solving/testing native planning does not require a physical GPU. + environment.setdefault("CONDA_OVERRIDE_CUDA", "13.0") + command = [ + "rattler-build", + "build", + "--recipe", + str(stage / "recipe"), + "--output-dir", + str(build_output), + "--no-config", + "--channel", + "rapidsai", + "--channel", + "conda-forge", + "--test", + "native", + ] + if args.python: + command.extend(("--variant", f"python={args.python}")) + subprocess.run(command, env=environment, check=True) + packages = sorted( + build_output.glob(f"linux-*/cuphoton-{version}-*.conda") + ) + expected_count = 1 if args.python else 3 + if len(packages) != expected_count: + raise RuntimeError( + f"Expected {expected_count} cuPhoton packages, " + f"found {len(packages)}" + ) + receipt = { + "version": version, + "source_archive": archive.name, + "source_sha256": hashlib.sha256(archive.read_bytes()).hexdigest(), + "packages": {}, + } + for package in packages: + relative_path = package.relative_to(build_output) + destination = output / relative_path + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(package, destination) + receipt["packages"][str(relative_path)] = hashlib.sha256( + destination.read_bytes() + ).hexdigest() + (output / "provenance.json").write_text( + json.dumps(receipt, indent=2, sort_keys=True) + "\n" + ) + + +if __name__ == "__main__": + main() diff --git a/src/cuphoton/xdr/nvcomp_batch.py b/src/cuphoton/xdr/nvcomp_batch.py index 3eed7c3..94e3154 100644 --- a/src/cuphoton/xdr/nvcomp_batch.py +++ b/src/cuphoton/xdr/nvcomp_batch.py @@ -19,7 +19,9 @@ import ctypes import importlib.util import os +import platform import struct +import sys from pathlib import Path from typing import Sequence @@ -72,6 +74,7 @@ def _find_library_path( *, env_var: str, description: str, + recursive: bool = True, ) -> Path: env_value = os.environ.get(env_var) if env_value: @@ -89,14 +92,16 @@ def _find_library_path( if path is not None: return path - for name in names: - matches = sorted(package_base.rglob(name)) - if matches: - return matches[0] + if recursive: + for name in names: + matches = sorted(package_base.rglob(name)) + if matches: + return matches[0] raise ImportError( f"Could not find {description} under {package_base}. " - "Searched lib64, lib, and recursive matches for: " + "Searched lib64, lib" + f"{' and subdirectories' if recursive else ''} for: " f"{', '.join(names)}. " f"Set {env_var} to the directory containing the library." ) @@ -118,6 +123,17 @@ def _candidate_cuda_homes(): if path is not None and path not in seen: seen.add(path) yield path + prefix = Path(sys.prefix) + if (prefix / "conda-meta").is_dir(): + machine = platform.machine() + target = "sbsa" if machine == "aarch64" else machine + for path in ( + prefix, + prefix / "targets" / f"{target}-linux", + ): + if path not in seen: + seen.add(path) + yield path default = Path("/usr/local/cuda") if default not in seen: yield default @@ -155,35 +171,40 @@ def _preload_gpu_package_libraries() -> None: nvcomp_base = _package_dir("nvidia.libnvcomp") rapids_logger_base = _package_dir("rapids_logger") kvikio_base = _package_dir("libkvikio") + prefix = Path(sys.prefix) if ( nvcomp_base is None or rapids_logger_base is None or kvikio_base is None ): - raise ImportError( - "Could not find libkvikio, rapids-logger, or " - "nvidia-libnvcomp Python packages required by " - "cuphoton.xdr._nvcomp_batch_ext." - ) + if not (prefix / "conda-meta").is_dir(): + raise ImportError( + "Could not find libkvikio, rapids-logger, or " + "nvidia-libnvcomp Python packages required by " + "cuphoton.xdr._nvcomp_batch_ext." + ) libraries = [ _find_library_path( - nvcomp_base, + nvcomp_base or prefix, ("libnvcomp.so.5", "libnvcomp.so"), env_var=_NVCOMP_LIB_ENV, description="nvCOMP library", + recursive=nvcomp_base is not None, ), _find_library_path( - rapids_logger_base, + rapids_logger_base or prefix, ("librapids_logger.so",), env_var=_RAPIDS_LOGGER_LIB_ENV, description="RAPIDS logger library", + recursive=rapids_logger_base is not None, ), _find_library_path( - kvikio_base, + kvikio_base or prefix, ("libkvikio.so",), env_var=_KVIKIO_LIB_ENV, description="KvikIO library", + recursive=kvikio_base is not None, ), ] for library in libraries: diff --git a/src/cuphoton/xdr/setup_package.py b/src/cuphoton/xdr/setup_package.py index 6570198..2e0ae5c 100644 --- a/src/cuphoton/xdr/setup_package.py +++ b/src/cuphoton/xdr/setup_package.py @@ -6,6 +6,7 @@ import importlib.util import os +import platform import shutil import subprocess from pathlib import Path @@ -17,6 +18,7 @@ _NVCOMP_LIB_ENV = "CUPHOTON_XDR_NVCOMP_LIB_DIR" _KVIKIO_LIB_ENV = "CUPHOTON_XDR_KVIKIO_LIB_DIR" _CFITSIO_ROOT_ENV = "CUPHOTON_XDR_CFITSIO_ROOT" +_NATIVE_PREFIX_ENV = "CUPHOTON_XDR_NATIVE_PREFIX" def _unique(paths): @@ -65,6 +67,7 @@ def _find_library_dir( *, env_var: str, description: str, + recursive: bool = True, ) -> Path: env_value = os.environ.get(env_var) if env_value: @@ -81,20 +84,30 @@ def _find_library_dir( if _library_in_dir(lib_dir, names): return lib_dir - for name in names: - matches = sorted(package_base.rglob(name)) - if matches: - return matches[0].parent + if recursive: + for name in names: + matches = sorted(package_base.rglob(name)) + if matches: + return matches[0].parent raise RuntimeError( f"Could not find {description} under {package_base}. " - "Searched lib64, lib, and recursive matches for: " + "Searched lib64, lib" + f"{' and subdirectories' if recursive else ''} for: " f"{', '.join(names)}. " f"Set {env_var} to the directory containing the library." ) def _cuda_home_candidates(): + native_prefix = os.environ.get(_NATIVE_PREFIX_ENV) + if native_prefix: + root = Path(native_prefix) + yield root + machine = platform.machine() + target = "sbsa" if machine == "aarch64" else machine + yield root / "targets" / f"{target}-linux" + return seen = set() for value in ( os.environ.get("CUDA_HOME"), @@ -149,14 +162,19 @@ def _find_gpu_package_paths(): import pybind11 pybind11_include = Path(pybind11.get_include()) - kvikio_base = _package_dir("libkvikio") - nvcomp_base = _package_dir("nvidia.libnvcomp") + native_prefix = os.environ.get(_NATIVE_PREFIX_ENV) + if native_prefix: + kvikio_base = nvcomp_base = Path(native_prefix) + else: + kvikio_base = _package_dir("libkvikio") + nvcomp_base = _package_dir("nvidia.libnvcomp") kvikio_include = kvikio_base / "include" kvikio_lib = _find_library_dir( kvikio_base, ("libkvikio.so",), env_var=_KVIKIO_LIB_ENV, description="KvikIO library", + recursive=not native_prefix, ) nvcomp_include = nvcomp_base / "include" nvcomp_lib = _find_library_dir( @@ -164,6 +182,7 @@ def _find_gpu_package_paths(): ("libnvcomp.so.5", "libnvcomp.so"), env_var=_NVCOMP_LIB_ENV, description="nvCOMP library", + recursive=not native_prefix, ) return { @@ -185,6 +204,15 @@ def _find_cufile_include(cuda_include: Path) -> Path: if (cuda_include / "cufile.h").is_file(): return cuda_include + if os.environ.get(_NATIVE_PREFIX_ENV): + for root in _cuda_home_candidates(): + candidate = root / "include" + if (candidate / "cufile.h").is_file(): + return candidate + raise RuntimeError( + f"{_NATIVE_PREFIX_ENV} must contain the cuFile header cufile.h" + ) + for module_name in ("nvidia.cu13", "nvidia.cufile"): try: candidate = _package_dir(module_name) / "include" @@ -209,7 +237,12 @@ def _split_pkg_config_flags(flags: str, prefix: str) -> list[str]: def _find_cfitsio_paths(): - env_value = os.environ.get(_CFITSIO_ROOT_ENV) + root_env = ( + _CFITSIO_ROOT_ENV + if os.environ.get(_CFITSIO_ROOT_ENV) + else _NATIVE_PREFIX_ENV + ) + env_value = os.environ.get(root_env) if env_value: root = Path(env_value) include_candidates = [root / "include", root] @@ -235,8 +268,7 @@ def _find_cfitsio_paths(): ) if include_dir is None or lib_dir is None: raise RuntimeError( - f"{_CFITSIO_ROOT_ENV}={root} must contain fitsio.h and " - "libcfitsio" + f"{root_env}={root} must contain fitsio.h and libcfitsio" ) return { "include_dirs": [str(include_dir)], diff --git a/tests/test_conda_build.py b/tests/test_conda_build.py new file mode 100644 index 0000000..33e84b5 --- /dev/null +++ b/tests/test_conda_build.py @@ -0,0 +1,244 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Check source identity and artifact promotion without a conda build.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import io +import json +import os +import subprocess +import sys +import tarfile +import textwrap +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts/conda/build.py" +SPEC = importlib.util.spec_from_file_location("conda_build", SCRIPT) +conda_build = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(conda_build) + + +def source_archive(directory, version): + directory.mkdir(parents=True, exist_ok=True) + archive = directory / "release-source.tar.gz" + contents = { + "PKG-INFO": f"Name: cuphoton\nVersion: {version}\n".encode(), + "packaging/conda/recipe.yaml": b"source:\n path: ../source\n", + "src/cuphoton/__init__.py": b"# Packaged source, without Git.\n", + } + with tarfile.open(archive, "w:gz") as target: + for name, data in contents.items(): + member = tarfile.TarInfo(f"cuphoton-source/{name}") + member.size = len(data) + target.addfile(member, io.BytesIO(data)) + return archive, contents + + +def invoke(monkeypatch, archive, output, python=None): + arguments = [str(SCRIPT), str(archive), "--output-dir", str(output)] + if python is not None: + arguments.extend(("--python", python)) + monkeypatch.setattr(sys, "argv", arguments) + conda_build.main() + + +def write_packages(directory, version, versions): + packages = {} + for python in versions: + tag = f"py{python.replace('.', '')}h123_0" + relative = f"linux-64/cuphoton-{version}-{tag}.conda" + payload = f"{version}: native package for Python {python}\n".encode() + destination = directory / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(payload) + packages[relative] = payload + return packages + + +@pytest.mark.parametrize("version", ["0.1.3", "0.1.3rc0"]) +@pytest.mark.parametrize("python", [None, "3.14"]) +def test_sdist_identity_and_bytes_survive_neutral_build_staging( + monkeypatch, tmp_path, version, python +): + private = tmp_path / "private-home" + private_tmp = private / "temporary" + private_tmp.mkdir(parents=True) + monkeypatch.setenv("TMPDIR", str(private_tmp)) + monkeypatch.setattr(conda_build.tempfile, "tempdir", str(private_tmp)) + monkeypatch.setenv("CUPHOTON_CONDA_VERSION", "0.0.0") + archive, contents = source_archive(private / "input", version) + output = private / "artifacts" + builds = [] + expected = {} + + def build(command, *, env, check): + assert env["CUPHOTON_CONDA_VERSION"] == version + recipe = Path(command[command.index("--recipe") + 1]) + build_output = Path(command[command.index("--output-dir") + 1]) + stage = recipe.parent + assert stage.parent == Path("/tmp") + assert build_output.is_relative_to(stage) + assert not stage.is_relative_to(private) + assert not (stage / "source/.git").exists() + for name, data in contents.items(): + assert (stage / "source" / name).read_bytes() == data + assert (recipe / "recipe.yaml").read_bytes() == contents[ + "packaging/conda/recipe.yaml" + ] + builds.append(stage) + expected.update( + write_packages( + build_output, + version, + [python] if python else ["3.12", "3.13", "3.14"], + ) + ) + + monkeypatch.setattr(conda_build.subprocess, "run", build) + + invoke(monkeypatch, archive, output, python) + + assert len(builds) == 1 + assert not builds[0].exists() + assert { + str(path.relative_to(output)): path.read_bytes() + for path in output.glob("linux-*/*.conda") + } == expected + receipt = json.loads((output / "provenance.json").read_text()) + assert receipt == { + "version": version, + "source_archive": archive.name, + "source_sha256": hashlib.sha256(archive.read_bytes()).hexdigest(), + "packages": { + name: hashlib.sha256(data).hexdigest() + for name, data in expected.items() + }, + } + + +def test_stale_output_is_rejected_before_building(monkeypatch, tmp_path): + archive, _ = source_archive(tmp_path / "input", "0.1.3rc0") + output = tmp_path / "output" + original = write_packages(output, "0.1.2", ["3.12"]) + receipt = output / "provenance.json" + receipt.write_bytes(b"previous build receipt") + + def unexpected_build(*args, **kwargs): + pytest.fail( + "A stale output directory must be rejected before building" + ) + + monkeypatch.setattr(conda_build.subprocess, "run", unexpected_build) + + with pytest.raises(SystemExit) as error: + invoke(monkeypatch, archive, output) + + assert error.value.code == 2 + assert receipt.read_bytes() == b"previous build receipt" + assert { + str(path.relative_to(output)): path.read_bytes() + for path in output.glob("linux-*/*.conda") + } == original + + +@pytest.mark.parametrize("python", [None, "3.14"]) +def test_incomplete_matrix_is_not_promoted(monkeypatch, tmp_path, python): + archive, _ = source_archive(tmp_path / "input", "0.1.3rc0") + output = tmp_path / "output" + + def build(command, *, env, check): + build_output = Path(command[command.index("--output-dir") + 1]) + versions = ["3.12", "3.13"] if python is None else [] + write_packages(build_output, "0.1.3rc0", versions) + # A stale version must not fill the gap in this build's matrix. + write_packages(build_output, "0.1.2", ["3.14"]) + + monkeypatch.setattr(conda_build.subprocess, "run", build) + + with pytest.raises(RuntimeError, match="Expected .* cuPhoton packages"): + invoke(monkeypatch, archive, output, python) + + assert not output.exists() + + +@pytest.mark.parametrize( + "targets", + [ + ("wheels", "conda"), + ("conda", "wheels"), + ("wheels", "conda", "conda"), + ("conda", "wheels", "wheels"), + ], +) +def test_packaging_targets_preserve_both_formats(tmp_path, targets): + makefile = SCRIPT.parents[2] / "Makefile" + (tmp_path / "Makefile").write_text(makefile.read_text()) + commands = tmp_path / "bin" + commands.mkdir() + wheel = "dist/cuphoton-0.1.3-cp312-cp312-manylinux_2_28_x86_64.whl" + conda = "dist/conda/linux-64/cuphoton-0.1.3-py312_0.conda" + scripts = { + "uv": f"""\ + #!/bin/sh + set -eu + mkdir -p dist + if test "$1" = build; then + printf source > dist/cuphoton-0.1.3.tar.gz + else + test -f dist/cuphoton-0.1.3.tar.gz + test ! -e {wheel} + printf wheel > {wheel} + fi + """, + "pixi": f"""\ + #!/bin/sh + set -eu + test -f dist/cuphoton-0.1.3.tar.gz + test ! -e {conda} + mkdir -p dist/conda/linux-64 + printf conda > {conda} + """, + "uvx": """\ + #!/bin/sh + set -eu + for argument do + case "$argument" in dist/*) test -f "$argument" ;; esac + done + """, + } + for name, content in scripts.items(): + command = commands / name + command.write_text(textwrap.dedent(content)) + command.chmod(0o755) + environment = {**os.environ, "PATH": f"{commands}:{os.environ['PATH']}"} + (tmp_path / "dist").mkdir() + stale_source = tmp_path / "dist/cuphoton-0.0.0.tar.gz" + stale_source.write_text("stale source") + stale_wheel = tmp_path / "dist/cuphoton-0.0.0-old.whl" + stale_wheel.write_text("stale wheel") + stale_conda = tmp_path / "dist/conda/linux-64/cuphoton-0.0.0-old.conda" + stale_conda.parent.mkdir(parents=True) + stale_conda.write_text("stale conda") + + for target in (*targets, "package-check"): + subprocess.run( + ["make", target], + cwd=tmp_path, + env=environment, + check=True, + capture_output=True, + ) + + assert (tmp_path / wheel).read_text() == "wheel" + assert (tmp_path / conda).read_text() == "conda" + assert not stale_source.exists() + assert not stale_wheel.exists() + assert not stale_conda.exists() + assert len(list((tmp_path / "dist").glob("*.tar.gz"))) == 1 diff --git a/tests/xdr/test_cuda_discovery.py b/tests/xdr/test_cuda_discovery.py index e9cf4f7..926a51f 100644 --- a/tests/xdr/test_cuda_discovery.py +++ b/tests/xdr/test_cuda_discovery.py @@ -4,9 +4,10 @@ from __future__ import annotations +import platform import sys from pathlib import Path -from types import ModuleType +from types import ModuleType, SimpleNamespace import pytest @@ -34,6 +35,7 @@ def cuda_wheel(monkeypatch, tmp_path): monkeypatch.delitem(sys.modules, name, raising=False) monkeypatch.delenv("CUDA_HOME", raising=False) monkeypatch.delenv("CUDA_PATH", raising=False) + monkeypatch.delenv("CUPHOTON_XDR_NATIVE_PREFIX", raising=False) return root @@ -164,3 +166,193 @@ def test_build_skips_runtime_wheel_without_crt(monkeypatch, tmp_path): toolkit / "include", toolkit / "lib", ) + + +@pytest.fixture +def conda_native_prefix(monkeypatch, tmp_path): + prefix = tmp_path / "conda" + (prefix / "conda-meta").mkdir(parents=True) + (prefix / "lib").mkdir() + (prefix / "include").mkdir() + for name in ( + "libcudart.so.13", + "libnvcomp.so.5", + "librapids_logger.so", + "libkvikio.so", + "libcfitsio.so", + ): + (prefix / "lib" / name).touch() + for name in ("fitsio.h", "cufile.h"): + (prefix / "include" / name).touch() + monkeypatch.setattr(sys, "prefix", str(prefix)) + monkeypatch.setattr(nvcomp_batch, "_package_dir", lambda _: None) + monkeypatch.setitem( + sys.modules, + "pybind11", + SimpleNamespace(get_include=lambda: str(prefix / "include")), + ) + for name in ( + "CUDA_HOME", + "CUDA_PATH", + "CUPHOTON_XDR_NATIVE_PREFIX", + "CUPHOTON_XDR_CFITSIO_ROOT", + "CUPHOTON_XDR_NVCOMP_LIB_DIR", + "CUPHOTON_XDR_KVIKIO_LIB_DIR", + "CUPHOTON_XDR_RAPIDS_LOGGER_LIB_DIR", + ): + monkeypatch.delenv(name, raising=False) + return prefix + + +@pytest.mark.parametrize( + ("architecture", "target_name"), + [("x86_64", "x86_64-linux"), ("aarch64", "sbsa-linux")], +) +def test_conda_build_uses_prefix_headers_and_libraries( + monkeypatch, conda_native_prefix, architecture, target_name +): + monkeypatch.setattr(platform, "machine", lambda: architecture) + prefix = conda_native_prefix + target = _cuda_tree(prefix / "targets" / target_name) + monkeypatch.setenv("CUPHOTON_XDR_NATIVE_PREFIX", str(prefix)) + + def unexpected_wheel_lookup(name): + pytest.fail(f"Conda build looked for a wheel package: {name}") + + monkeypatch.setattr( + setup_package, "_package_dir", unexpected_wheel_lookup + ) + + (extension,) = setup_package.get_extensions() + + assert str(prefix / "include") in extension.include_dirs + assert str(target / "include") in extension.include_dirs + assert str(prefix / "lib") in extension.library_dirs + assert str(target / "lib") in extension.library_dirs + assert "-lcfitsio" in extension.extra_link_args + + +def test_explicit_native_prefix_cannot_fall_back_to_cuda_wheel( + monkeypatch, tmp_path, cuda_wheel +): + monkeypatch.setenv("CUPHOTON_XDR_NATIVE_PREFIX", str(tmp_path / "empty")) + + with pytest.raises(RuntimeError, match="libcudart.so.13"): + setup_package._find_cuda_toolkit() + + +@pytest.mark.parametrize("target_layout", [False, True]) +@pytest.mark.parametrize( + ("architecture", "target_name"), + [("x86_64", "x86_64-linux"), ("aarch64", "sbsa-linux")], +) +def test_conda_runtime_uses_python_prefix_not_shell_prefix( + monkeypatch, + tmp_path, + conda_native_prefix, + target_layout, + architecture, + target_name, +): + monkeypatch.setattr(platform, "machine", lambda: architecture) + monkeypatch.setenv("CONDA_PREFIX", str(tmp_path / "unrelated")) + loaded = [] + monkeypatch.setattr(nvcomp_batch, "_load_shared_library", loaded.append) + runtime = conda_native_prefix / "lib" / "libcudart.so.13" + if target_layout: + target_runtime = ( + conda_native_prefix + / "targets" + / target_name + / "lib" + / runtime.name + ) + target_runtime.parent.mkdir(parents=True) + runtime.rename(target_runtime) + runtime = target_runtime + + nvcomp_batch._preload_cudart() + nvcomp_batch._preload_gpu_package_libraries() + + assert loaded == [runtime] + [ + conda_native_prefix / "lib" / name + for name in ( + "libnvcomp.so.5", + "librapids_logger.so", + "libkvikio.so", + ) + ] + + +def test_conda_runtime_honors_explicit_library_override( + monkeypatch, tmp_path, conda_native_prefix +): + override = tmp_path / "selected" + override.mkdir() + (override / "libnvcomp.so.5").touch() + monkeypatch.setenv("CUPHOTON_XDR_NVCOMP_LIB_DIR", str(override)) + loaded = [] + monkeypatch.setattr(nvcomp_batch, "_load_shared_library", loaded.append) + + nvcomp_batch._preload_gpu_package_libraries() + + assert loaded[0] == override / "libnvcomp.so.5" + + +def test_conda_missing_library_does_not_search_nested_environments( + monkeypatch, conda_native_prefix +): + prefix = conda_native_prefix + library = prefix / "lib" / "libnvcomp.so.5" + nested_library = prefix / "envs" / "unrelated" / "lib" / library.name + nested_library.parent.mkdir(parents=True) + library.rename(nested_library) + monkeypatch.setenv("CUPHOTON_XDR_NATIVE_PREFIX", str(prefix)) + + with pytest.raises(ImportError, match="Could not find nvCOMP library"): + nvcomp_batch._preload_gpu_package_libraries() + with pytest.raises(RuntimeError, match="Could not find nvCOMP library"): + setup_package._find_gpu_package_paths() + + +def test_plain_python_prefix_is_not_treated_as_conda(conda_native_prefix): + (conda_native_prefix / "conda-meta").rmdir() + + with pytest.raises(ImportError, match="Python packages required"): + nvcomp_batch._preload_gpu_package_libraries() + + +@pytest.mark.parametrize( + "missing_package", + [None, "nvidia.libnvcomp", "rapids_logger", "libkvikio"], +) +def test_gpu_wheels_remain_usable_inside_conda( + monkeypatch, tmp_path, conda_native_prefix, missing_package +): + package_dirs = {} + expected = [] + for module, library in ( + ("nvidia.cu13", "libcudart.so.13"), + ("nvidia.libnvcomp", "libnvcomp.so.5"), + ("rapids_logger", "librapids_logger.so"), + ("libkvikio", "libkvikio.so"), + ): + if module == missing_package: + expected.append(conda_native_prefix / "lib" / library) + continue + package = tmp_path / "site-packages" / module + (package / "lib").mkdir(parents=True) + path = package / "lib" / library + path.touch() + package_dirs[module] = package + expected.append(path) + monkeypatch.setattr( + nvcomp_batch, "_package_dir", lambda name: package_dirs.get(name) + ) + loaded = [] + monkeypatch.setattr(nvcomp_batch, "_load_shared_library", loaded.append) + + nvcomp_batch._preload_cudart() + nvcomp_batch._preload_gpu_package_libraries() + + assert loaded == expected