Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,13 @@ jobs:
name: native distributions
uses: ./.github/workflows/wheels.yml

conda:
name: native conda distributions
uses: ./.github/workflows/conda.yml

ci-required:
name: ci-required
needs: [lint, test-cpu, package]
needs: [lint, test-cpu, package, conda]
if: ${{ always() }}
runs-on: ubuntu-24.04
timeout-minutes: 5
Expand All @@ -99,7 +103,8 @@ jobs:
LINT_RESULT: ${{ needs.lint.result }}
CPU_RESULT: ${{ needs.test-cpu.result }}
PACKAGE_RESULT: ${{ needs.package.result }}
CONDA_RESULT: ${{ needs.conda.result }}
run: |
for result in "${LINT_RESULT}" "${CPU_RESULT}" "${PACKAGE_RESULT}"; do
for result in "${LINT_RESULT}" "${CPU_RESULT}" "${PACKAGE_RESULT}" "${CONDA_RESULT}"; do
test "${result}" = success || exit 1
done
85 changes: 85 additions & 0 deletions .github/workflows/conda.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0

name: native conda packages

on:
workflow_call:
workflow_dispatch:

permissions:
contents: read

jobs:
source:
runs-on: ubuntu-24.04
timeout-minutes: 10
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
persist-credentials: false
fetch-depth: 0
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
with:
version: "0.12.1"
enable-cache: false
- run: CUPHOTON_XDR_BUILD_EXT=0 uv build --sdist
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: cuphoton-conda-source
path: dist/*.tar.gz
if-no-files-found: error

build:
needs: source
strategy:
fail-fast: false
matrix:
python: ["3.12", "3.13", "3.14"]
platform:
- runner: ubuntu-24.04
subdir: linux-64
pixi-target: x86_64-unknown-linux-musl
pixi-sha256: a40e0cb519c681b76cf5ae3a2bdb72e98edde6b1a72cc6a2469edaa4f2c68dca
- runner: ubuntu-24.04-arm
subdir: linux-aarch64
pixi-target: aarch64-unknown-linux-musl
pixi-sha256: bac5949b12ec65d76ea5a9d22b67340b2447cf6ccaa34deac1386430b45b5231
runs-on: ${{ matrix.platform.runner }}
timeout-minutes: 30
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
persist-credentials: false
- name: Install pinned Pixi
env:
PIXI_TARGET: ${{ matrix.platform.pixi-target }}
PIXI_SHA256: ${{ matrix.platform.pixi-sha256 }}
run: |
mkdir -p "${RUNNER_TEMP}/pixi-bin"
curl --fail --location --retry 3 \
"https://github.com/prefix-dev/pixi/releases/download/v0.62.2/pixi-${PIXI_TARGET}" \
--output "${RUNNER_TEMP}/pixi-bin/pixi"
printf '%s %s\n' "${PIXI_SHA256}" "${RUNNER_TEMP}/pixi-bin/pixi" | sha256sum --check
chmod +x "${RUNNER_TEMP}/pixi-bin/pixi"
echo "${RUNNER_TEMP}/pixi-bin" >> "${GITHUB_PATH}"
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: cuphoton-conda-source
path: dist
- name: Build and test in isolated conda environments
env:
PYTHON_VERSION: ${{ matrix.python }}
run: >-
pixi exec --spec rattler-build=0.76.1 --spec python=3.12 --
python scripts/conda/build.py dist/*.tar.gz
--python "$PYTHON_VERSION" --output-dir dist/conda
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: cuphoton-conda-${{ matrix.platform.subdir }}-py${{ matrix.python }}
path: |
dist/conda/${{ matrix.platform.subdir }}/cuphoton-*.conda
dist/conda/provenance.json
if-no-files-found: error
retention-days: 30
2 changes: 2 additions & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ prune .gitlab
prune scripts
include scripts/cuphoton-openmpi-rank-exec
recursive-include scripts/wheels *.sh *.py *.txt
recursive-include scripts/conda *.py
recursive-include packaging/conda *.yaml *.sh
include LICENSE
include CITATION.cff
include Makefile
Expand Down
14 changes: 10 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0

.PHONY: sync sync-gpu sync-cutile lock lock-check lint format test test-cpu test-core test-xdr test-xfit test-xfit-real test-xpois test-xscan test-xrep test-xray test-gpu test-cutile clean-dist build package-check wheels release-check ci-lint ci-test-cpu hooks
.PHONY: sync sync-gpu sync-cutile lock lock-check lint format test test-cpu test-core test-xdr test-xfit test-xfit-real test-xpois test-xscan test-xrep test-xray test-gpu test-cutile clean-dist build package-check wheels conda release-check ci-lint ci-test-cpu hooks

CPU_EXTRAS = --extra dev --extra torch --extra viz --extra photometry
GPU_EXTRAS = --extra dev --extra gpu --extra viz
Expand Down Expand Up @@ -72,22 +72,28 @@ test-cutile:
clean-dist:
rm -rf dist

build: clean-dist
build:
rm -f dist/cuphoton-*.tar.gz
CUPHOTON_XDR_BUILD_EXT=0 uv build --sdist

wheels: build
rm -f dist/cuphoton-*.whl
uv tool run --from cibuildwheel==4.2.1 cibuildwheel --platform linux --output-dir dist dist/*.tar.gz

conda: build
Comment thread
coderabbitai[bot] marked this conversation as resolved.
rm -rf dist/conda
pixi exec --spec rattler-build=0.76.1 --spec python=3.12 -- python scripts/conda/build.py dist/*.tar.gz
Comment thread
coderabbitai[bot] marked this conversation as resolved.

package-check: build
uvx --isolated --from twine==6.2.0 twine check --strict dist/*
uvx --isolated --from twine==6.2.0 twine check --strict dist/*.tar.gz

release-check:
$(MAKE) lock-check
$(MAKE) ci-lint
$(MAKE) ci-test-cpu
$(MAKE) wheels
python scripts/wheels/check_distributions.py dist --arch "$$(uname -m)"
uvx --isolated --from twine==6.2.0 twine check --strict dist/*
uvx --isolated --from twine==6.2.0 twine check --strict dist/*.tar.gz dist/*.whl

ci-lint:
$(MAKE) lint
Expand Down
3 changes: 2 additions & 1 deletion ROADMAP.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ institutional adaptation.

- Publish pip packages after release automation and provenance checks are
established.
- Add conda packaging with the same dependency-profile boundaries.
- Ship conda packaging for the native I/O profile. A CPU-only conda package
and separate dependency profiles remain future work.
- Keep experimental toolchain integrations, such as cuTile, isolated from the
core install.

Expand Down
120 changes: 110 additions & 10 deletions docs/packaging.md
Original file line number Diff line number Diff line change
@@ -1,21 +1,23 @@
# Native wheels
# Native packages

Release artifacts are six Linux wheels (CPython 3.12, 3.13, and 3.14 on
x86-64 and ARM64) plus one source archive. Wheels target glibc 2.28 or later.
The runtime dependencies may impose a newer glibc floor; the installed-wheel
CI tests use Debian 12. Free-threaded Python, Windows, macOS, and conda builds
are outside this matrix.
CI tests use Debian 12. Native conda builds cover the same six Python and
architecture combinations, as described below. Free-threaded Python, Windows,
and macOS are outside both matrices.

Each wheel contains `cuphoton.xdr._nvcomp_batch_ext` and a privately renamed,
reentrant CFITSIO 4.7.0 shared library. CUDA, cuFile, KvikIO, and nvCOMP remain
in their upstream wheels, installed through `cuphoton[io]`. `cuphoton[gpu]`
also includes the photometry, PyTorch, and Numba backends. Photutils currently
requires a source build and C compiler on ARM64; `cuphoton` and `cuphoton[io]`
do not install it.
also includes the photometry, PyTorch, and Numba backends. Installing Photutils
from PyPI currently requires a source build and C compiler on ARM64;
`cuphoton` and `cuphoton[io]` do not install it. Conda provides ARM64 Photutils
binaries.

The native KvikIO ABI is restricted to the 26.6 release family. nvCOMP is
restricted to 5.2. Updating either family requires rebuilding and qualifying
the native wheels. The CUDA SDK build inputs are pinned to 13.0 so a newer
the native packages. The CUDA SDK build inputs are pinned to 13.0 so a newer
build environment cannot silently raise the runtime floor.

## Versions and release candidates
Expand Down Expand Up @@ -51,14 +53,14 @@ Add `viz` for visualization dependencies. Use `--pre` when selecting the newest
available prerelease instead of pinning one. Each changed candidate needs a
new RC number: PyPI does not allow replacing an uploaded filename.

## Build
## Build wheels

From the checkout, on each native Linux architecture with Docker and uv:

```bash
make wheels
python scripts/wheels/check_distributions.py dist --arch "$(uname -m)"
uvx --from twine==6.2.0 twine check --strict dist/*
uvx --from twine==6.2.0 twine check --strict dist/*.tar.gz dist/*.whl
```

`make wheels` builds a source archive, then uses cibuildwheel 4.2.1 to build
Expand All @@ -73,6 +75,12 @@ editable installs remain Python-only unless `CUPHOTON_XDR_BUILD_EXT=1` is
set. See [XDR source installation](components/xdr.md#native-extension-availability)
for the explicit native development build.

`make build` refreshes only the source archive. `make wheels` replaces cuPhoton
wheels while preserving conda outputs; `make conda` replaces `dist/conda` while
preserving wheels. Use `make clean-dist` explicitly to remove all distributions.
To reuse the wheel build's exact archive for conda, use the direct conda build
command below.

The reusable `wheels.yml` workflow builds from one source archive on native
x86-64 and ARM64 runners. It checks base imports inside cibuildwheel, then
installs each wheel and its `io` dependencies in a clean Python container
Expand All @@ -88,7 +96,99 @@ These workers solve generated xPOIS inputs on the CPU and check numerical
results, distinct processes, and MPI collectives. JSON receipts are retained
as CI artifacts. These checks do not establish GPU executor correctness.

## GPU qualification
## Build and install conda packages

The `cuphoton` conda package includes the compiled XDR extension and depends on
upstream conda packages for CFITSIO 4.7, CUDA 13, KvikIO 26.6, and nvCOMP 5.2.
Conda installs those libraries into the environment; no manual CUDA paths
are needed.
Conda has no pip-style extras: this package provides the native I/O profile
only and requires CUDA 13 libraries. It does not provide a CPU-only install;
use the base pip package for that profile.

With uv and pixi installed, run on each native Linux architecture:

```bash
make conda
```

This builds the SCM-versioned source archive and uses rattler-build 0.76.1 to
produce Python 3.12, 3.13, and 3.14 packages. The archive supplies the same
version used by wheels, including RC versions, without requiring Git during
the conda build. To build one Python version from an existing source archive:

```bash
pixi exec --spec rattler-build=0.76.1 --spec python=3.12 -- \
python scripts/conda/build.py 'dist/cuphoton-<version>.tar.gz' \
--python 3.12 --output-dir dist/conda
```

Use an output directory without existing cuPhoton packages. Outputs go under
`linux-64` or `linux-aarch64`, with a `provenance.json` recording the source
archive and package SHA256 values. Builds use strict channel priority with
`rapidsai` before `conda-forge`, excluding `defaults`. This order is required
by the current installed-package solver; it also gives RAPIDS precedence for
other packages present in both channels. Review the resolved environment
before qualifying an artifact.
The qualified conda nvCOMP build is 5.2.0.10, while the wheel environment uses
5.2.0.13. Both stay within the required 5.2 ABI family and need independent
artifact qualification; their patch versions are not interchangeable evidence.

Create a local channel index so conda resolves the artifact's runtime
dependencies. For downloaded CI artifacts, place the packages under
`dist/conda/linux-64` or `dist/conda/linux-aarch64` first. Select the version
and build string from the package filename:

```bash
mkdir -p dist/conda/noarch
pixi exec --spec conda-index=0.13.0 -- python -m conda_index dist/conda
conda create --prefix ./conda-xdr --override-channels \
--strict-channel-priority -c "$PWD/dist/conda" -c rapidsai -c conda-forge \
'python=3.12' 'cuphoton=<version>=<build>'
```

Direct installation of a `.conda` filename skips dependency resolution; use
the indexed channel directory above. See [conda's installation
guidance](https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/installing-with-conda.html).
For broader GPU and photometry functionality, add the following packages.
The PyTorch build selector chooses its CUDA 13.0 build:

```bash
conda install --prefix ./conda-xdr --override-channels \
--strict-channel-priority -c "$PWD/dist/conda" -c rapidsai -c conda-forge \
'photutils>=3' 'numba>=0.61,<0.66' 'numba-cuda>=0.30,<0.31' \
'pytorch>=2.13,<3' 'pytorch=*=cuda130*' 'cuda-version=13.0'
```

Install development tools such as `pytest`, `ruff`, and `pre-commit` through
conda. Keep the repository's uv development environment separate: the current
conda PyTorch package requires `setuptools<82`, while the pip `dev` extra
requires `setuptools>=83`. For Python-only editable work in a separate conda
environment with the runtime dependencies and pip installed, run
`CUPHOTON_XDR_BUILD_EXT=0 conda run --prefix ./conda-dev python -m pip install --no-deps -e .`.
Native builds should use the recipe above.

The reusable `conda.yml` workflow builds all six variants from one source
archive and retains the packages and provenance as CI artifacts. Its isolated
installation tests load the native extension and exercise CFITSIO planning
without a GPU or driver. Upstream conda GPU packages retain some pip-only
dependency names in their Python metadata, so `pip check` is not a conda
validation gate. The actual native and GPU tests are required.

Qualify each exact conda artifact on its architecture and Python version with
a CUDA 13-compatible driver, using the installed-package GPU check:

```bash
conda run --prefix ./conda-xdr python -I scripts/wheels/test_installed.py \
--mode gpu --output conda-gpu.json
```

The check runs outside the source tree internally and exercises the same
decoding, ordering, concurrency, and buffer-lifetime cases as wheels. Retain
its JSON receipt and the tested artifact hash. Conda channel publication is
not configured; release-tag publishing below applies to PyPI artifacts.

## Wheel GPU qualification

CI CPU checks do not establish GPU correctness. Download the exact
`cuphoton-distributions` artifact and test each wheel on its architecture and
Expand Down
11 changes: 11 additions & 0 deletions packaging/conda/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0

set -euo pipefail

export CUPHOTON_XDR_BUILD_EXT=1
export CUPHOTON_XDR_NATIVE_PREFIX="$PREFIX"
export SETUPTOOLS_SCM_PRETEND_VERSION_FOR_CUPHOTON="$PKG_VERSION"
"$PYTHON" -m pip install . --no-deps --no-build-isolation --no-index -v
Loading
Loading