diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18b6bbe3..60bb14cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -284,22 +284,90 @@ jobs: run: pytest tests/test_cedspike2_integration.py tests/test_cedspike2_sonpipe.py -v --tb=short install-without-git: - # Guards the git-free install. The double-click installer is built on - # `uv tool install`, and uv shells out to a system `git` for every - # `git+` dependency URL -- which many end users (Windows and macOS in - # particular) do not have. Every runtime dependency is therefore pulled - # from a GitHub archive (zip) URL, fetched over plain HTTPS. This job - # installs NDI with `git` made unusable, so a `git+` URL sneaking back - # into pyproject.toml (here, or through an extra we install) fails CI + # Guards the git-free, compiler-free install on every platform the + # double-click installer targets. + # + # git: the installer is built on `uv tool install`, and uv shells out to a + # system `git` for every `git+` dependency URL -- which many end users + # (Windows and macOS in particular) do not have. Every runtime dependency + # is therefore pulled from a GitHub archive (zip) URL over plain HTTPS. + # This job installs NDI with `git` made unusable, so a `git+` URL sneaking + # back into pyproject.toml (here, or through an extra we install) fails CI # instead of failing on a user's machine. - name: Install without git - runs-on: ubuntu-latest + # + # Pyraview: not on PyPI, and a C++ build from its sdist. It must come from + # the pre-built wheels on its GitHub release, found through the committed + # flat index install/pyraview-wheels.html, with hashes enforced through + # install/pyraview-constraints.txt. `--only-binary pyraview` makes a + # missing wheel an error rather than a compile, CMake and the C/C++ + # compilers are hidden as well, and tools/check_pyraview_wheel.py then + # proves the installed WHEEL tags are exactly those of a release wheel. + # + # The Linux x86_64 leg keeps the job's original check name, so a branch + # rule that requires "Install without git" still finds it. + name: ${{ matrix.check_name }} + runs-on: ${{ matrix.os }} + # Bounds a hung install. (It does not bound time spent QUEUED for a + # runner label: if macos-15-intel is ever retired, drop that leg.) + timeout-minutes: 45 concurrency: - group: ci-install-without-git-${{ github.ref }} + group: ci-install-without-git-${{ matrix.leg }}-${{ github.ref }} cancel-in-progress: true + strategy: + fail-fast: false + matrix: + include: + - leg: linux-x86_64 + check_name: Install without git + os: ubuntu-latest + python: "3.12" + machine: x86_64 + tags: "manylinux_ x86_64" + - leg: linux-aarch64 + check_name: Install without git (linux-aarch64) + os: ubuntu-24.04-arm + python: "3.12" + machine: aarch64 + tags: "manylinux_ aarch64" + - leg: windows-amd64 + check_name: Install without git (windows-amd64) + os: windows-latest + python: "3.12" + machine: AMD64 + tags: "win_amd64" + - leg: macos-arm64 + check_name: Install without git (macos-arm64) + os: macos-latest + python: "3.12" + machine: arm64 + tags: "macosx_ arm64" + - leg: macos-intel + check_name: Install without git (macos-intel) + os: macos-15-intel + python: "3.12" + machine: x86_64 + tags: "macosx_ x86_64" + # An Intel Mac, as Apple Silicon sees one: an x86_64 CPython under + # Rosetta 2. uv picks wheels for the interpreter it installs into, + # so this resolves exactly what an Intel Mac would. + - leg: macos-rosetta-x86_64 + check_name: Install without git (macos-rosetta-x86_64) + os: macos-latest + python: cpython-3.12-macos-x86_64 + rosetta: true + machine: x86_64 + tags: "macosx_ x86_64" + defaults: + run: + shell: bash + env: + # Raw URL of THIS commit's index: uv reads --find-links only from a URL + # or a directory (a local .html file is refused), and the raw URL is + # what the double-click installer will use on main. + PYRAVIEW_FIND_LINKS: https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name || github.repository }}/${{ github.event.pull_request.head.sha || github.sha }}/install/pyraview-wheels.html steps: # The checkout itself uses git; that is fine -- the user's installer - # ships the source too. Only the install step below must not need it. + # ships the source too. Only the install steps below must not need it. - uses: actions/checkout@v4 - name: Install uv @@ -307,40 +375,193 @@ jobs: with: version: "0.11.32" # No cache: a warm cache could satisfy a dependency without ever - # reaching the code path that would call git. + # reaching the code path that would call git, or skip a download + # whose hash we mean to check. enable-cache: false - - name: Install NDI with git unavailable + - name: Make sure Rosetta 2 is available + if: matrix.rosetta + run: | + /usr/bin/pgrep -q oahd || sudo softwareupdate --install-rosetta --agree-to-license + arch -x86_64 /usr/bin/true + + - name: Create the Python ${{ matrix.python }} environment + run: | + uv python install "${{ matrix.python }}" + uv venv --python "${{ matrix.python }}" "$RUNNER_TEMP/venv" + if [ "$RUNNER_OS" = Windows ]; then + PY="$RUNNER_TEMP/venv/Scripts/python.exe" + else + PY="$RUNNER_TEMP/venv/bin/python" + fi + echo "PY=$PY" >> "$GITHUB_ENV" + "$PY" -c "import platform, sys; print(sys.version); print('machine:', platform.machine())" + "$PY" -c "import platform, sys; m = platform.machine(); sys.exit(0 if m == '${{ matrix.machine }}' else f'machine is {m}, expected ${{ matrix.machine }}')" + + - name: Hide git, CMake and the C/C++ compilers + run: | + # Writes a script the install steps source. On Linux and macOS it + # SHADOWS each tool with one that always fails, ahead of the real + # one on PATH (removing /usr/bin would remove the whole toolchain). + # On Windows a shadow script would be ignored -- native programs + # look for git.exe, not a shell script -- so every PATH directory + # holding one of these tools is dropped instead. + tools="git cmake cc c++ gcc g++ clang clang++ cl" + hide="$RUNNER_TEMP/hide-tools.sh" + if [ "$RUNNER_OS" = Windows ]; then + cat > "$hide" <&2\nexit 127\n' "$t" \ + > "$RUNNER_TEMP/notools/$t" + chmod +x "$RUNNER_TEMP/notools/$t" + done + echo "export PATH=\"$RUNNER_TEMP/notools:\$PATH\"" > "$hide" + fi + echo "HIDE_TOOLS=$hide" >> "$GITHUB_ENV" + + # Prove the hiding works before trusting the steps that rely on it. + source "$hide" + for t in git cmake c++ gcc; do + if "$t" --version >/dev/null 2>&1; then + echo "::error::$t is still usable; this job would prove nothing" + exit 1 + fi + done + echo "git, cmake, c++ and gcc are all unusable" + + - name: "Negative check: no find-links means no pyraview (and no compile)" + run: | + source "$HIDE_TOOLS" + # pyraview is not on PyPI, and the sdist is in no index we name, so + # without --find-links this has nothing to install and must say so. + # It must not fall back to building anything. + if uv pip install --python "$PY" --only-binary pyraview \ + --dry-run ".[pyraview]" > "$RUNNER_TEMP/neg.log" 2>&1; then + cat "$RUNNER_TEMP/neg.log" + echo "::error::installing .[pyraview] without --find-links succeeded" + exit 1 + fi + cat "$RUNNER_TEMP/neg.log" + grep -q "pyraview" "$RUNNER_TEMP/neg.log" + if grep -qiE "building pyraview|cmake" "$RUNNER_TEMP/neg.log"; then + echo "::error::the failed install tried to build pyraview" + exit 1 + fi + + - name: "Negative check: a wrong hash is refused" run: | - # Shadow git with a script that always fails, ahead of the real one - # on PATH. (Removing /usr/bin from PATH would also remove most of - # the toolchain, so shadowing is the cleaner way to hide it.) - mkdir -p "$RUNNER_TEMP/nogit" - printf '#!/bin/sh\necho "git is deliberately unavailable in this job" >&2\nexit 127\n' \ - > "$RUNNER_TEMP/nogit/git" - chmod +x "$RUNNER_TEMP/nogit/git" - export PATH="$RUNNER_TEMP/nogit:$PATH" - - # Prove the shadow works before trusting the install below. - if git --version; then - echo "::error::git is still usable; this job would prove nothing" + source "$HIDE_TOOLS" + # uv does NOT check the #sha256 fragments in a find-links page; the + # constraints file is what makes it verify. Prove that it does by + # corrupting every hash in a copy and expecting a refusal. + sed -E 's/sha256:[0-9a-f]{64}/sha256:0000000000000000000000000000000000000000000000000000000000000000/' \ + install/pyraview-constraints.txt > "$RUNNER_TEMP/bad-constraints.txt" + uv venv --python "${{ matrix.python }}" "$RUNNER_TEMP/hashcheck" + if uv pip install --python "$RUNNER_TEMP/hashcheck" --only-binary pyraview \ + -f "$PYRAVIEW_FIND_LINKS" -c "$RUNNER_TEMP/bad-constraints.txt" \ + "pyraview==0.4.2" > "$RUNNER_TEMP/hash.log" 2>&1; then + cat "$RUNNER_TEMP/hash.log" + echo "::error::a pyraview wheel with the wrong sha256 was installed" exit 1 fi + cat "$RUNNER_TEMP/hash.log" + grep -qi "hash mismatch" "$RUNNER_TEMP/hash.log" - uv venv --python 3.12 "$RUNNER_TEMP/venv" + - name: Install NDI with git and compilers unavailable + run: | + source "$HIDE_TOOLS" # [tutorials] brings pandas and matplotlib, which `ndi check` checks. - uv pip install --python "$RUNNER_TEMP/venv/bin/python" ".[tutorials]" + # [pyraview] is the extra the double-click installer will request. + # -v so the log names the wheel uv selected; only those lines are + # printed, the whole log only if the install fails. + if ! uv pip install -v --python "$PY" \ + -f "$PYRAVIEW_FIND_LINKS" \ + -c install/pyraview-constraints.txt \ + --only-binary pyraview \ + ".[tutorials,pyraview]" > "$RUNNER_TEMP/install.log" 2>&1; then + tail -n 200 "$RUNNER_TEMP/install.log" + exit 1 + fi + grep -m 20 -E "Selecting: pyraview|/pyraview-[0-9.]+-[^ ]*\.whl|\+ pyraview" "$RUNNER_TEMP/install.log" \ + || echo "(no pyraview lines in the uv log; the next step is the proof)" + if grep -qiE "building pyraview|pyraview-[0-9.]+\.tar\.gz" "$RUNNER_TEMP/install.log"; then + echo "::error::pyraview was built from source" + exit 1 + fi + + - name: Prove pyraview is a release wheel for this platform + # From outside the checkout, so imports resolve to the installed tree. + working-directory: ${{ runner.temp }} + run: | + "$PY" -c "import pyraview, os; print(pyraview.__file__, pyraview._lib_path)" + tag_args=() + for t in ${{ matrix.tags }}; do tag_args+=(--expect-tag "$t"); done + "$PY" "$GITHUB_WORKSPACE/tools/check_pyraview_wheel.py" \ + --expect-machine "${{ matrix.machine }}" "${tag_args[@]}" + + - name: The pip path (ndi_install.py's) also takes the wheel + run: | + source "$HIDE_TOOLS" + # pip, unlike uv, reads a local find-links .html file and checks the + # #sha256 fragment on each link -- which is how ndi_install.py uses + # the index. A seeded venv gives this interpreter its own pip. + uv venv --seed --python "${{ matrix.python }}" "$RUNNER_TEMP/pipvenv" + if [ "$RUNNER_OS" = Windows ]; then PIPPY="$RUNNER_TEMP/pipvenv/Scripts/python.exe" + else PIPPY="$RUNNER_TEMP/pipvenv/bin/python"; fi + "$PIPPY" -m pip install --no-cache-dir --only-binary pyraview \ + --find-links install/pyraview-wheels.html "pyraview==0.4.2" + cd "$RUNNER_TEMP" + tag_args=() + for t in ${{ matrix.tags }}; do tag_args+=(--expect-tag "$t"); done + "$PIPPY" "$GITHUB_WORKSPACE/tools/check_pyraview_wheel.py" \ + --expect-machine "${{ matrix.machine }}" "${tag_args[@]}" + + - name: The installer's command form (uv tool install) also takes the wheel + if: matrix.leg == 'linux-x86_64' + run: | + source "$HIDE_TOOLS" + # What the double-click installer runs, against this checkout. + # uv tool install does not verify hashes (neither the #sha256 + # fragments nor a constraints file's), so this leg proves only the + # wheel-not-sdist half; the steps above prove the hash half. + export UV_TOOL_DIR="$RUNNER_TEMP/uvtools" UV_TOOL_BIN_DIR="$RUNNER_TEMP/uvbin" + uv tool install --python 3.12 -f "$PYRAVIEW_FIND_LINKS" \ + --no-build-package pyraview "$GITHUB_WORKSPACE[pyraview]" + cd "$RUNNER_TEMP" + "$UV_TOOL_DIR/ndi/bin/python" "$GITHUB_WORKSPACE/tools/check_pyraview_wheel.py" \ + --expect-machine x86_64 --expect-tag manylinux_ + + - name: The committed index matches the GitHub release + if: matrix.leg == 'linux-x86_64' + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + # Regenerates both install/ files from the GitHub API in memory and + # compares: the committed sha256s are GitHub's own asset digests. + tag="v$(sed -nE 's/^ *"pyraview==([^"]+)",/\1/p' pyproject.toml)" + python3 tools/make_pyraview_index.py "$tag" --check - name: Import NDI and its sibling packages run: | - "$RUNNER_TEMP/venv/bin/python" -c "import ndi, did, ndr, vlt, vhlib, ndicompress, ndi_ontology, ndicalc_vis_matlab" + "$PY" -c "import ndi, did, ndr, vlt, vhlib, ndicompress, ndi_ontology, ndicalc_vis_matlab, pyraview" - name: Validate installation # Run from outside the checkout so `import ndi` resolves to the # installed wheel, not the source tree. working-directory: ${{ runner.temp }} run: | - "$RUNNER_TEMP/venv/bin/python" -m ndi check + "$PY" -m ndi check - name: Resolve an NDIcalc document definition # NDIcalc-vis-matlab's definitions are not in this repo: they come @@ -351,7 +572,7 @@ jobs: # the same reason as the step above. working-directory: ${{ runner.temp }} run: | - "$RUNNER_TEMP/venv/bin/python" - <<'PY' + "$PY" - <<'PY' from did.validate import resolve_definition_path from ndi.common import ndi_common_PathConstants from ndi.document import ndi_document diff --git a/CHANGELOG.md b/CHANGELOG.md index 4aedb30f..09a89594 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added +- **Pyraview from pre-built wheels, never from source.** New `pyraview` + extra (`pyraview==0.4.2`). Pyraview is not on PyPI, so the wheels on its + GitHub release are named in a committed flat index, + `install/pyraview-wheels.html` (each link with its `#sha256`; the sdist left + out), with the same hashes in `install/pyraview-constraints.txt` for uv. + `tools/make_pyraview_index.py` regenerates both from a release tag. + `ndi_install.py` no longer clones Pyraview and compiles its C++; it installs + the wheel with `--only-binary pyraview`. CI installs it without git, CMake + or a compiler on Linux x86_64/aarch64, Windows, macOS arm64, macOS Intel and + x86_64-under-Rosetta. + - **File CRC-32 checksums for NDI Cloud** (ndi-cloud-node#162, mirroring NDI-matlab#1025), so an importer can resume an interrupted upload by re-uploading only the files whose cloud copy does not match. diff --git a/README.md b/README.md index 9dcb6aca..0e934c72 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ NDI-Python requires these VH-Lab packages (installed automatically by `ndi_insta |---------|-----------|---------| | [DID-python](https://github.com/VH-Lab/DID-python) | VH-Lab/DID-python | Document database backend (SQLite, queries) | | [vhlab-toolbox-python](https://github.com/VH-Lab/vhlab-toolbox-python) | VH-Lab/vhlab-toolbox-python | Data utilities, file formats, signal processing | +| [Pyraview](https://github.com/VH-Lab/Pyraview) | VH-Lab/Pyraview | Multi-resolution signal pyramids for `ndi.gui.app.pyraview` (pre-built wheels, `pyraview` extra) | Additional dependencies (installed automatically): `numpy`, `networkx`, `jsonschema`, `requests`, `scipy`, `pandas`, `matplotlib`. @@ -81,6 +82,34 @@ pip install scipy pandas matplotlib opencv-python-headless portalocker openminds +### Pyraview (signal pyramids) + +[Pyraview](https://github.com/VH-Lab/Pyraview) is not on PyPI. Its GitHub +release ships a pre-built wheel for every platform and CPython version (Linux +x86_64/aarch64, macOS Apple Silicon *and* Intel, Windows), with the compiled +library inside, so no compiler is ever needed. `ndi_install.py` installs it +for you. In your own environment, from the repo root: + +```bash +pip install -e ".[pyraview]" --find-links install/pyraview-wheels.html --only-binary pyraview +``` + +or, with uv (which needs the index as a URL, and checks hashes only through +the constraints file): + +```bash +uv pip install -e ".[pyraview]" --only-binary pyraview -c install/pyraview-constraints.txt \ + -f https://raw.githubusercontent.com/Waltham-Data-Science/NDI-python/main/install/pyraview-wheels.html +``` + +`install/pyraview-wheels.html` lists each release wheel with its sha256 and +deliberately leaves out the source archive; `--only-binary pyraview` makes a +missing wheel an error instead of a C++ build. The extra is not part of `gui`, +so `pip install ".[gui]"` keeps working without these flags. On a new Pyraview +release, bump the pin in `pyproject.toml` and run +`python tools/make_pyraview_index.py vX.Y.Z` to regenerate both files in +`install/`. + ## Quick Start ```python diff --git a/install/pyraview-constraints.txt b/install/pyraview-constraints.txt new file mode 100644 index 00000000..db4ada04 --- /dev/null +++ b/install/pyraview-constraints.txt @@ -0,0 +1,58 @@ +# Generated by tools/make_pyraview_index.py from VH-Lab/Pyraview release v0.4.2. +# Do not edit by hand. One --hash per wheel in install/pyraview-wheels.html; +# passed as --constraint, uv refuses any pyraview file not listed here. +pyraview==0.4.2 \ + --hash=sha256:6d2b3d23fa256c9dd109bc163885271e4ab976ebf271c95152dee2d7347a8bb2 \ + --hash=sha256:a42b43c8e767a7fe817cef4cb0c52ceed25261e5be7de0067a206b43d73e3c35 \ + --hash=sha256:d17765bdd6da2b8b07e107b679eb3d36ad78178a7ffa692de7cda2eb1af6afdc \ + --hash=sha256:d3e411ce9ec07ed13170abd78286ec4f9b6cf048a4e5bb28797171837226ba25 \ + --hash=sha256:f49acf7b8b8045ca6d923f3727987f4ff9e86d966b24be81d4eb2335218d7cbd \ + --hash=sha256:a2c73e3bd4fb295c4c25ac142359312d02ae7fc2c57e08ec822275eb2429b5eb \ + --hash=sha256:562ef3139545b2c739983b8d8dc97e3f5a96fede40c12a1d8d35bbe12b896b95 \ + --hash=sha256:d914925de98f1436dac7ab1010cba3ef9a3ce036043c957fc3fb3793d910c01e \ + --hash=sha256:fabeca334872749733f9ab737826c171078ab45a787805c7451027040986a98d \ + --hash=sha256:bf07bc0c44ea8e6a7746a36650d23acd0a0725b86957ef1cbd0e190b03b11575 \ + --hash=sha256:64502646e3bc947468c25836691c303df169d2cb40a2b51fc20197259fe199b0 \ + --hash=sha256:3436f70f5fe6ff30cac860eb7c0b05b3bdd7bfec2527aacaa568e4bc29c2b307 \ + --hash=sha256:68d8f401b2d59ee89065514e5d6446db5d2220e425884e192d0d68fe4f071e06 \ + --hash=sha256:55c06a89846c7abce208c1b984b16e1b74ea2b03291e9d5b386f0aa935c18377 \ + --hash=sha256:3ad3bcf8f921f1700037b7007155fc05acc3822b60df18cb410c085d2c290de4 \ + --hash=sha256:2a550481d87ac65352845938e10c2204eb593945bef836a1a19bf9c02814a9ca \ + --hash=sha256:7944371ff263351c7bca47752f9c2ff3ad03bae1ff34221fa447bab03b52f54f \ + --hash=sha256:df12ad189e1d61a6c92dae78114fe002feb7d3e964c893f8392850a93c413031 \ + --hash=sha256:befd683939157d403cbc7638b92fd20773906512317c341e6c42171bc531fb07 \ + --hash=sha256:ddc41d3a2ceeab165f7877c36cb2ac8b73f07de5ae702a53eadc573cb7057d00 \ + --hash=sha256:8b55ee5eb36d9c21d3d7fc33c76c5becd225393c516da5eaa3beefdd0a5c8d84 \ + --hash=sha256:631fd383814a82de4a340c94c060cc93cedf8d7ca23c9ccc91762f3964034a57 \ + --hash=sha256:8447ee11d6e3c178c6f6fd033b63952f0d6e84dafd3c812f697a933c10d78d1e \ + --hash=sha256:43f210414b2a25ebe3722bd9d53530d133ff485bd73207eb143c26bac4590bd3 \ + --hash=sha256:f6883a716af3ca03821c9848196a7a431af3b1b4f0a86973177f6ddd00535ced \ + --hash=sha256:47c03390155f96de29b932df72bf31b0f56f5f5b8516b631716d01a82d820942 \ + --hash=sha256:27f556e53caee131358ae17f2d75302b8c4d1f1023d63cc1217d053ddb47926e \ + --hash=sha256:c90f274801e45ee8281d3aaa9addca5c4f922db5686553f056537f0b0d22aaea \ + --hash=sha256:f8e42a34a4dfa65f4eacbb7d5ae216ad8060caed12916d6709658847662cf09e \ + --hash=sha256:22f2fd5f811d68ba8679b082f78bc7577d24488196e7841035cbee44b2fa8aa1 \ + --hash=sha256:f212530d7d3cee7a5f76209342451d18bc641cdaab94a6b45c6f7ef4a518b401 \ + --hash=sha256:20bf3c85e4236dadc3a9edc7b39f0bc75858b25952ef6ba7b1b2d73d38229ad9 \ + --hash=sha256:63bc47a205f19ef7dac581ab0ff00324912868de8c95c438cb73c7331c270f7a \ + --hash=sha256:f5ba2a7b6060186e8523ce7a61e742fec566299ce5fcb3a5dfecfa7836e9b61b \ + --hash=sha256:2f08d9798e00f37d50c1601ae48a3197764e950facb757ad4452da96d3659bdb \ + --hash=sha256:7f08caeddaafc30529a38ed93aa492bf05116ce5340b465bb60ebe5a115b45aa \ + --hash=sha256:689db0f9014a3621b2373e7a926d7f54889cc68df70d77a22af4635e72f7a535 \ + --hash=sha256:c5f3cc644b24f4d40e6f57e8d49e1d81cf615fdd72e02225c5e10694df5cddb3 \ + --hash=sha256:943696a0c8ba3b6dd5b18dded1bb80d899615f5e892e3e02c7fdf93e0134403a \ + --hash=sha256:e7ea993c1c6dfa1b4e4a231bd6a88cad3065193305c3df7d829930c0999c2b4e \ + --hash=sha256:58b7a06023ce4c85cc96fec60fe203463b5dc23085fb1e792559cf1e1e3a334f \ + --hash=sha256:1e1883466eb002ff118625cec5e390057d5d7e9f1feda17e25ee10f2f8408c5b \ + --hash=sha256:f53dc9258c1c4ab68d528e93cc3168d0c0dcd18f293e6ec25a3df5d36651524e \ + --hash=sha256:aac9ed374c8517e772090a949a032006fe2cc91ff0f554d1395eafa89b5adc19 \ + --hash=sha256:0b130beef553bcc467160de15dc3a3d1b236bfca9c3d3815f5b79a641a323a97 \ + --hash=sha256:f50547f6418230c966ab0de8fa3fd010b97d4ec2bf4d7578ff93b421ae97d80d \ + --hash=sha256:57317c12f4a2403c5b164ae24a48fb5cb0a78d9911f6bdb2f74367498d34b1a3 \ + --hash=sha256:090a614d50494d30402039cc331dcca83ee157afdd5760facb58f72f11a0dcae \ + --hash=sha256:351d9a78a52962b29d2f7fa59229a119714c945bc2cc0d9363d040c2454f8831 \ + --hash=sha256:92a7dc1868c378805f688561adb7a283fce3c9689739a1e59ea46d0a021400ab \ + --hash=sha256:4aacc1c83c4922dd1150d9e7d54046c3cc6504710dc40c229fd2608d59955267 \ + --hash=sha256:a2d008942281e391586bf8e247c6d7ef50a24c0971fd2e13152fd6f4e1ff08fb \ + --hash=sha256:3b19700d0b2090832619376ea646dbf8ef13025a8f90a2fd80212dc07d94de56 \ + --hash=sha256:dcf41699112b5c7e3135739e693a6e2d2f1287864145b120950392e912610db3 diff --git a/install/pyraview-wheels.html b/install/pyraview-wheels.html new file mode 100644 index 00000000..2669fa2b --- /dev/null +++ b/install/pyraview-wheels.html @@ -0,0 +1,63 @@ + + + +pyraview wheels + +pyraview-0.4.2-cp310-cp310-macosx_10_13_x86_64.whl
+pyraview-0.4.2-cp310-cp310-macosx_11_0_arm64.whl
+pyraview-0.4.2-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl
+pyraview-0.4.2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl
+pyraview-0.4.2-cp310-cp310-win32.whl
+pyraview-0.4.2-cp310-cp310-win_amd64.whl
+pyraview-0.4.2-cp311-cp311-macosx_10_13_x86_64.whl
+pyraview-0.4.2-cp311-cp311-macosx_11_0_arm64.whl
+pyraview-0.4.2-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl
+pyraview-0.4.2-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl
+pyraview-0.4.2-cp311-cp311-win32.whl
+pyraview-0.4.2-cp311-cp311-win_amd64.whl
+pyraview-0.4.2-cp312-cp312-macosx_10_13_x86_64.whl
+pyraview-0.4.2-cp312-cp312-macosx_11_0_arm64.whl
+pyraview-0.4.2-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl
+pyraview-0.4.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl
+pyraview-0.4.2-cp312-cp312-win32.whl
+pyraview-0.4.2-cp312-cp312-win_amd64.whl
+pyraview-0.4.2-cp313-cp313-macosx_10_13_x86_64.whl
+pyraview-0.4.2-cp313-cp313-macosx_11_0_arm64.whl
+pyraview-0.4.2-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl
+pyraview-0.4.2-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl
+pyraview-0.4.2-cp313-cp313-win32.whl
+pyraview-0.4.2-cp313-cp313-win_amd64.whl
+pyraview-0.4.2-cp314-cp314-macosx_10_15_x86_64.whl
+pyraview-0.4.2-cp314-cp314-macosx_11_0_arm64.whl
+pyraview-0.4.2-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl
+pyraview-0.4.2-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl
+pyraview-0.4.2-cp314-cp314-win32.whl
+pyraview-0.4.2-cp314-cp314-win_amd64.whl
+pyraview-0.4.2-cp314-cp314t-macosx_10_15_x86_64.whl
+pyraview-0.4.2-cp314-cp314t-macosx_11_0_arm64.whl
+pyraview-0.4.2-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl
+pyraview-0.4.2-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl
+pyraview-0.4.2-cp314-cp314t-win32.whl
+pyraview-0.4.2-cp314-cp314t-win_amd64.whl
+pyraview-0.4.2-cp315-cp315-macosx_10_15_x86_64.whl
+pyraview-0.4.2-cp315-cp315-macosx_11_0_arm64.whl
+pyraview-0.4.2-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl
+pyraview-0.4.2-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl
+pyraview-0.4.2-cp315-cp315-win32.whl
+pyraview-0.4.2-cp315-cp315-win_amd64.whl
+pyraview-0.4.2-cp315-cp315t-macosx_10_15_x86_64.whl
+pyraview-0.4.2-cp315-cp315t-macosx_11_0_arm64.whl
+pyraview-0.4.2-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl
+pyraview-0.4.2-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl
+pyraview-0.4.2-cp315-cp315t-win32.whl
+pyraview-0.4.2-cp315-cp315t-win_amd64.whl
+pyraview-0.4.2-cp39-cp39-macosx_10_13_x86_64.whl
+pyraview-0.4.2-cp39-cp39-macosx_11_0_arm64.whl
+pyraview-0.4.2-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl
+pyraview-0.4.2-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl
+pyraview-0.4.2-cp39-cp39-win32.whl
+pyraview-0.4.2-cp39-cp39-win_amd64.whl
+ + diff --git a/ndi_install.py b/ndi_install.py index f7ff03d9..20bbc791 100644 --- a/ndi_install.py +++ b/ndi_install.py @@ -15,7 +15,8 @@ 1. Checks prerequisites (git, Python >= 3.10) 2. Clones vhlab-toolbox-python to ~/.ndi/tools/ (not on PyPI) 3. Writes a .pth file so Python finds it automatically - 4. Installs NDI-python and all pip dependencies (DID-python via pip) + 4. Installs NDI-python and all pip dependencies (DID-python via pip), + including Pyraview from its pre-built release wheels (never compiled) 5. Validates the installation """ @@ -41,23 +42,29 @@ "python_path": ".", "description": "VH-Lab data utilities and file formats (not on PyPI)", }, - { - "name": "Pyraview", - "repo": "https://github.com/VH-Lab/Pyraview.git", - "branch": "main", - # pip-installed rather than added to the path: the package carries a - # compiled library, so it has to be built and installed, not imported - # from a checkout. The clone is still what --update pulls. - "python_path": "", - "pip_install": True, - "description": "Multi-resolution signal pyramids (not on PyPI; builds a C++ library)", - }, ] DEFAULT_TOOLS_DIR = Path.home() / ".ndi" / "tools" PTH_FILENAME = "ndi-deps.pth" +# Pyraview (multi-resolution signal pyramids, a C++ core behind ctypes) is not +# on PyPI. Its release carries one wheel per platform and CPython version with +# the compiled library inside, so it is installed as the `pyraview` extra +# (pinned in pyproject.toml) from those wheels, found through the flat index +# committed at install/pyraview-wheels.html. That page gives every wheel's +# release URL with a #sha256 fragment, which pip checks on download. The sdist +# is deliberately not in it, and pyraview_pip_args() also passes --only-binary, +# so a platform with no wheel FAILS here rather than trying to compile C++. +# Regenerate the index with tools/make_pyraview_index.py on a new release. +PYRAVIEW_INDEX = Path("install") / "pyraview-wheels.html" + + +def pyraview_pip_args(ndi_root: Path) -> list[str]: + """The pip flags that let `pyraview` resolve, and only from a wheel.""" + return ["--find-links", str(ndi_root / PYRAVIEW_INDEX), "--only-binary", "pyraview"] + + # Seconds allowed for a subprocess that talks to the NETWORK: a git clone, a # git pull, or the editable install (which resolves six git-URL dependencies # from pyproject.toml, so pip clones as well as downloads). @@ -289,44 +296,6 @@ def clone_or_update(name: str, repo_url: str, target_dir: Path, branch: str, upd return git_clone(repo_url, target_dir, branch) -# --------------------------------------------------------------------------- -# Compiled dependencies -# --------------------------------------------------------------------------- - - -def pip_install_dependency(name: str, repo_dir: Path) -> bool: - """Install a cloned dependency that has to be built rather than imported. - - Pyraview is a C++ core with a ctypes binding: the Python package cannot - import until it can load libpyraview. Its own build (scikit-build-core - driving CMake) compiles the library into the installed package, so - installing the checkout is all that is needed -- and is why this one is - pip-installed instead of being added to ndi-deps.pth like the others. - - It is not on PyPI, which is why it is cloned here at all rather than - named in pyproject.toml; a direct git URL there would install the same - thing but could never be published to an index. - """ - info(f"Building and installing {name}...") - # Not NETWORK_TIMEOUT: this one COMPILES a C++ library rather than merely - # fetching, so its cost is the machine's, not the network's, and half an - # hour is the bound that suits it. - result = subprocess.run( - [sys.executable, "-m", "pip", "install", str(repo_dir)], - capture_output=True, - text=True, - timeout=1800, - ) - if result.returncode != 0: - fail(f"{name}: pip install failed") - detail(result.stderr.strip()[-2000:]) - warn(" This build needs a C++ compiler and CMake.") - return False - - detail(result.stdout.strip()[-500:]) - return True - - # --------------------------------------------------------------------------- # .pth file management # --------------------------------------------------------------------------- @@ -404,9 +373,9 @@ def install_ndi_and_deps(ndi_root: Path, include_dev: bool = False) -> bool: # Install NDI in editable mode (pip resolves DID-python from pyproject.toml) info("Installing NDI-python (editable)...") - extras = "tutorials" + extras = "pyraview,tutorials" if include_dev: - extras = "dev,tutorials" + extras = "dev,pyraview,tutorials" result = subprocess.run( [ sys.executable, @@ -416,7 +385,8 @@ def install_ndi_and_deps(ndi_root: Path, include_dev: bool = False) -> bool: "-e", f".[{extras}]", "--quiet", - ], + ] + + pyraview_pip_args(ndi_root), capture_output=True, text=True, timeout=NETWORK_TIMEOUT, @@ -424,6 +394,10 @@ def install_ndi_and_deps(ndi_root: Path, include_dev: bool = False) -> bool: ) if result.returncode != 0: fail(f"pip install -e failed: {result.stderr.strip()}") + if "pyraview" in result.stderr.lower(): + warn(" pyraview installs only from its pre-built release wheels, listed in") + warn(f" {PYRAVIEW_INDEX}; it is never compiled. Check that one matches this") + warn(" Python version and platform.") ok = False else: success("NDI-python installed") @@ -588,8 +562,6 @@ def main() -> int: for dep in DEPENDENCIES: target = tools_dir / dep["name"] ok = clone_or_update(dep["name"], dep["repo"], target, dep["branch"], args.update) - if ok and dep.get("pip_install"): - ok = pip_install_dependency(dep["name"], target) if not ok: all_cloned = False diff --git a/pyproject.toml b/pyproject.toml index 6cb753bf..e3e3ea6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,11 @@ dependencies = [ # default. keyring is preferred when present but stays optional, # since it is the OS-keychain path rather than the portable one. "cryptography>=41.0", + # ...but cryptography 49 dropped its Intel-macOS (universal2) wheels, so + # on an Intel Mac a newer one is a Rust + C build from the sdist, which an + # end user without Xcode cannot do. 48.0.1 is the last with a wheel there. + # Lift this once a release ships an x86_64 macOS wheel again. + "cryptography<49; sys_platform == 'darwin' and platform_machine == 'x86_64'", # dask[array] powers ndi.pyramid.ImagePyramidLoader -- the # napari-independent multiscale loader. It used to live under # the napari extra because only napari-driven viewers touched @@ -108,6 +113,23 @@ napari = [ "napari[all]>=0.4.19", "dask[array]>=2023.1", ] +# Pyraview (multi-resolution signal pyramids; ndi.gui.app.pyraview imports it) +# is a C++ core behind ctypes, and is NOT on PyPI: its GitHub release carries +# a pre-built wheel per platform and CPython version, compiled library inside. +# So this extra only resolves with a find-links flat index naming those wheels +# -- install/pyraview-wheels.html, generated by tools/make_pyraview_index.py -- +# and every install path we control also forbids building it from source: +# +# pip install ".[pyraview]" --find-links install/pyraview-wheels.html --only-binary pyraview +# uv pip install ".[pyraview]" --only-binary pyraview -c install/pyraview-constraints.txt \ +# -f https://raw.githubusercontent.com/Waltham-Data-Science/NDI-python/main/install/pyraview-wheels.html +# +# It is NOT folded into `gui`: a plain `pip install ".[gui]"` must keep working +# without the extra flag. Exact pin, by name rather than URL, so the one wheel +# filename per platform is fixed and the hashes in install/ can vouch for it. +pyraview = [ + "pyraview==0.4.2", +] dev = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", diff --git a/src/ndi/gui/app/pyraview/__init__.py b/src/ndi/gui/app/pyraview/__init__.py index 40e880ce..af075442 100644 --- a/src/ndi/gui/app/pyraview/__init__.py +++ b/src/ndi/gui/app/pyraview/__init__.py @@ -14,7 +14,9 @@ library (https://github.com/VH-Lab/Pyraview) that builds and reads the multi-resolution pyramids the viewer draws: a C++ core with MATLAB and Python bindings. Inside this package that name always means the library, since -Python 3 resolves it absolutely; ``ndi_install.py`` clones and builds it. +Python 3 resolves it absolutely. It is the ``pyraview`` extra, installed from +the pre-built release wheels listed in ``install/pyraview-wheels.html`` +(``ndi_install.py`` does this; it is never compiled). Nothing here imports the library at module level, so a machine without it still imports ndi.gui.app and still discovers the other apps. The two diff --git a/tests/test_ndi_install_exit_codes.py b/tests/test_ndi_install_exit_codes.py index 803dc481..38c9f01b 100644 --- a/tests/test_ndi_install_exit_codes.py +++ b/tests/test_ndi_install_exit_codes.py @@ -57,13 +57,6 @@ def _run(module, argv: list[str], **overrides) -> tuple[int, str]: defaults = { "check_prerequisites": lambda: [], "clone_or_update": lambda *a, **k: True, - # Step 2 calls this immediately after clone_or_update, for every - # dependency with pip_install set. Leaving it unstubbed shelled out to - # a REAL pip install: green in CI, where there is a network and a - # writable site-packages, and red anywhere else -- so three cases - # failed for a reason that had nothing to do with what they assert. - # See NDI-python#267. - "pip_install_dependency": lambda *a, **k: True, "get_site_packages": lambda: Path("/tmp"), "write_pth_file": lambda *a, **k: Path("/tmp/ndi-test.pth"), "find_ndi_root": lambda: REPO_ROOT, @@ -150,24 +143,3 @@ def test_failed_prerequisites_still_exits_nonzero(self, installer): def test_failed_clone_still_exits_nonzero(self, installer): code, _ = _run(installer, [], clone_or_update=lambda *a, **k: False) assert code == 1 - - def test_failed_dependency_pip_install_exits_nonzero(self, installer): - """The path the stub added in #267 would otherwise hide. - - Step 2 treats a failed pip install of a dependency exactly as it - treats a failed clone -- all_cloned goes false and main returns 1. - Nothing covered that, which is part of why the unstubbed call went - unnoticed for so long. - """ - code, out = _run( - installer, - ["--no-validate"], - pip_install_dependency=lambda *a, **k: False, - # Stubbed so that a REGRESSION stays cheap. If step 2 ever stops - # returning 1 here, main() falls through to the later steps; with - # these unstubbed that means a real network install on the way to - # a failing assert -- the thing #267 was about. - install_ndi_and_deps=lambda *a, **k: True, - ) - assert code == 1 - assert "Some dependencies could not be cloned" in out diff --git a/tests/test_ndi_install_timeouts.py b/tests/test_ndi_install_timeouts.py index 18bbf43d..e256006d 100644 --- a/tests/test_ndi_install_timeouts.py +++ b/tests/test_ndi_install_timeouts.py @@ -13,8 +13,9 @@ into a network-bound step does. That is the mistake worth preventing, because 120 looks generous right up until you remember the six clones behind it. -Two literals are allowed, each for a stated reason, and adding a third means -deciding here whether it deserves one. +One literal is allowed, for a stated reason, and adding another means deciding +here whether it deserves one. (There used to be a second, 1800 s for building +Pyraview's C++ from a clone; Pyraview now installs from pre-built wheels.) """ from __future__ import annotations @@ -36,8 +37,6 @@ #: rather than skips. ALLOWED_LITERALS = { 10: "git --version: a version string, on no network", - 1800: "the Pyraview build: COMPILES a C++ library, so its cost is the " - "machine's rather than the network's", } diff --git a/tests/test_pyraview_wheel_index.py b/tests/test_pyraview_wheel_index.py new file mode 100644 index 00000000..efb55cdb --- /dev/null +++ b/tests/test_pyraview_wheel_index.py @@ -0,0 +1,183 @@ +"""Pyraview installs from its pre-built release wheels, and never compiles. + +Pyraview is not on PyPI. The `pyraview` extra in pyproject.toml pins it by +name, and two committed files in install/ say where its wheels are and what +they hash to: + +* ``pyraview-wheels.html`` -- the ``--find-links`` flat index, one link per + release wheel with a ``#sha256=`` fragment, and no sdist (building one needs + CMake and a C++ compiler); +* ``pyraview-constraints.txt`` -- the same hashes as a constraints file, which + is how uv is made to verify them. + +These tests hold the pieces to each other offline. CI proves the rest with a +network: the install-pyraview-wheels matrix in ci.yml installs from them on +every platform, and regenerates them from the GitHub API to show the committed +hashes are GitHub's own digests. +""" + +from __future__ import annotations + +import importlib.util +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent +INDEX = REPO_ROOT / "install" / "pyraview-wheels.html" +CONSTRAINTS = REPO_ROOT / "install" / "pyraview-constraints.txt" +PYPROJECT = REPO_ROOT / "pyproject.toml" + +_LINK = re.compile(r'([^<]+)') + + +def _load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def links() -> list[tuple[str, str, str]]: + """(url, sha256, filename) for every link in the committed index.""" + found = _LINK.findall(INDEX.read_text(encoding="utf-8")) + assert found, f"no hashed links in {INDEX}" + return found + + +@pytest.fixture(scope="module") +def pinned_version() -> str: + match = re.search(r'^pyraview = \[\s*"pyraview==([^"]+)",\s*\]', PYPROJECT.read_text(), re.M) + assert match, 'pyproject.toml has no `pyraview = ["pyraview==..."]` extra' + return match.group(1) + + +class TestTheIndex: + def test_every_link_is_a_wheel_of_the_pinned_version(self, links, pinned_version): + for url, _, name in links: + assert name.endswith(".whl"), f"{name}: only wheels belong in the index" + assert name.startswith(f"pyraview-{pinned_version}-"), name + assert url.endswith("/" + name), (url, name) + assert url.startswith( + f"https://github.com/VH-Lab/Pyraview/releases/download/v{pinned_version}/" + ), url + + def test_every_link_carries_a_hash(self): + """A link without a fragment would slip past the regex above.""" + text = INDEX.read_text(encoding="utf-8") + assert text.count("=3.10; ndi_install.py runs on whatever it is.""" + names = [name for _, _, name in links] + for minor in (10, 11, 12, 13, 14): + tag = f"-cp3{minor}-cp3{minor}-" + for plat in ("manylinux_2_28_x86_64", "macosx_11_0_arm64", "win_amd64"): + assert any(tag in n and plat in n for n in names), (tag, plat) + assert any(tag in n and "macosx_" in n and "x86_64" in n for n in names), tag + + +class TestTheConstraints: + def test_same_hashes_as_the_index(self, links): + text = CONSTRAINTS.read_text(encoding="utf-8") + assert set(re.findall(r"--hash=sha256:([0-9a-f]{64})", text)) == {h for _, h, _ in links} + + def test_pins_the_same_version(self, pinned_version): + assert f"pyraview=={pinned_version} \\" in CONSTRAINTS.read_text(encoding="utf-8") + + +class TestTheGenerator: + def test_it_reproduces_the_committed_files(self, links): + """Both files are generated; a hand edit shows up as a mismatch.""" + gen = _load("make_pyraview_index", REPO_ROOT / "tools" / "make_pyraview_index.py") + tag = links[0][0].split("/")[-2] + release = { + "tag_name": tag, + "assets": [ + {"name": name, "digest": f"sha256:{sha}", "browser_download_url": url} + for url, sha, name in links + ] + # The real release also carries these; the generator must drop them. + + [ + { + "name": f"pyraview-{tag[1:]}.tar.gz", + "digest": "sha256:" + "0" * 64, + "browser_download_url": "https://example.invalid/sdist", + }, + { + "name": "Pyraview.mltbx", + "digest": "sha256:" + "1" * 64, + "browser_download_url": "https://example.invalid/mltbx", + }, + ], + } + wheel_list = gen.wheels(release) + assert gen.render_index(tag, wheel_list) == INDEX.read_text(encoding="utf-8") + assert gen.render_constraints(tag, wheel_list) == CONSTRAINTS.read_text(encoding="utf-8") + + +class TestPyproject: + def test_pyraview_is_not_in_the_gui_extra(self): + """`pip install ".[gui]"` must not need a --find-links flag.""" + match = re.search(r"^gui = \[(.*?)\]", PYPROJECT.read_text(), re.M | re.S) + assert match and "pyraview" not in match.group(1) + + def test_it_is_named_not_a_url(self): + text = PYPROJECT.read_text() + assert "pyraview @" not in text.lower() + assert "Pyraview.git" not in text + + +class TestNdiInstall: + @pytest.fixture + def installer(self): + return _load("ndi_install_pyraview_under_test", REPO_ROOT / "ndi_install.py") + + def test_pyraview_is_no_longer_cloned(self, installer): + assert all(dep["name"].lower() != "pyraview" for dep in installer.DEPENDENCIES) + + @pytest.mark.parametrize("include_dev", [False, True]) + def test_the_editable_install_takes_pyraview_from_the_wheel_index( + self, installer, monkeypatch, include_dev + ): + calls = [] + + def fake_run(cmd, **kwargs): + calls.append(cmd) + return subprocess.CompletedProcess(cmd, 0, "", "") + + monkeypatch.setattr(installer.subprocess, "run", fake_run) + assert installer.install_ndi_and_deps(REPO_ROOT, include_dev=include_dev) + + editable = next(c for c in calls if "-e" in c) + assert editable[:4] == [sys.executable, "-m", "pip", "install"] + target = editable[editable.index("-e") + 1] + assert "pyraview" in target.split("[")[1].rstrip("]").split(",") + index = Path(editable[editable.index("--find-links") + 1]) + assert index == INDEX and index.is_file() + assert editable[editable.index("--only-binary") + 1] == "pyraview" diff --git a/tools/check_pyraview_wheel.py b/tools/check_pyraview_wheel.py new file mode 100644 index 00000000..c9b3c683 --- /dev/null +++ b/tools/check_pyraview_wheel.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Prove the installed pyraview came from one of the release wheels. + + python tools/check_pyraview_wheel.py [--expect-machine x86_64] + [--expect-tag macosx_ --expect-tag x86_64] + +Run with the interpreter of the environment to check. It fails unless: + +* ``import pyraview`` works and loads the library bundled INSIDE the package + (not one named by ``PYRAVIEW_LIB``); +* the installed ``.dist-info/WHEEL`` is a platform wheel (Root-Is-Purelib: + false) whose tag set is exactly that of a wheel listed in + install/pyraview-wheels.html. A wheel built locally from the sdist would + carry a tag no release wheel has (``linux_x86_64`` rather than + ``manylinux_*``, a different macOS deployment target) or a different + generator, so this is what tells "downloaded" from "compiled here". + +It prints the matching release wheel's filename and sha256, which is the line +to read in a CI log. +""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import os +import platform +import re +import sys +from pathlib import Path + +INDEX = Path(__file__).resolve().parent.parent / "install" / "pyraview-wheels.html" +_LINK = re.compile(r'([^<]+\.whl)') + + +def expand_tags(filename: str) -> set[str]: + """The tag set a wheel filename stands for (PEP 425 compressed tag sets).""" + stem = filename[: -len(".whl")] + parts = stem.split("-") + py, abi, plat = parts[-3], parts[-2], parts[-1] + return {f"{p}-{a}-{q}" for p in py.split(".") for a in abi.split(".") for q in plat.split(".")} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--expect-machine", help="required platform.machine()") + parser.add_argument( + "--expect-tag", action="append", default=[], help="substring every WHEEL tag must contain" + ) + args = parser.parse_args() + problems = [] + + print(f"python: {sys.version.split()[0]} on {platform.system()} {platform.machine()}") + if args.expect_machine and platform.machine() != args.expect_machine: + problems.append( + f"platform.machine() is {platform.machine()!r}, not {args.expect_machine!r}" + ) + + if os.environ.get("PYRAVIEW_LIB"): + problems.append("PYRAVIEW_LIB is set; the bundled library would not be what loads") + + import pyraview + + package_dir = Path(pyraview.__file__).resolve().parent + lib = Path(pyraview._lib_path).resolve() + print(f"pyraview: {pyraview.__file__}") + print(f"library: {lib}") + if lib.parent != package_dir or not lib.is_file(): + problems.append(f"library {lib} is not bundled in {package_dir}") + + dist = importlib.metadata.distribution("pyraview") + wheel_text = dist.read_text("WHEEL") or "" + tags = set(re.findall(r"^Tag:\s*(\S+)", wheel_text, re.M)) + print(f"version: {dist.version}") + print(f"installer: {(dist.read_text('INSTALLER') or '?').strip()}") + print("WHEEL:\n " + "\n ".join(wheel_text.strip().splitlines())) + if not re.search(r"^Root-Is-Purelib:\s*false", wheel_text, re.M | re.I): + problems.append("WHEEL does not say Root-Is-Purelib: false -- not a platform wheel") + if not tags or any(t.endswith("-any") for t in tags): + problems.append(f"WHEEL tags {sorted(tags)} are not platform tags") + for needle in args.expect_tag: + if not all(needle in t for t in tags): + problems.append(f"WHEEL tags {sorted(tags)} do not all contain {needle!r}") + + listed = {name: sha for sha, name in _LINK.findall(INDEX.read_text(encoding="utf-8"))} + matches = [ + name + for name in listed + if name.startswith(f"pyraview-{dist.version}-") and expand_tags(name) == tags + ] + if len(matches) == 1: + print(f"INSTALLED RELEASE WHEEL: {matches[0]}") + print(f" sha256 (from install/pyraview-wheels.html): {listed[matches[0]]}") + else: + problems.append( + f"WHEEL tags {sorted(tags)} match {len(matches)} release wheels in {INDEX.name}; " + "this pyraview was not installed from a release wheel" + ) + + for p in problems: + print(f"FAIL: {p}") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/make_pyraview_index.py b/tools/make_pyraview_index.py new file mode 100644 index 00000000..050b95db --- /dev/null +++ b/tools/make_pyraview_index.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Regenerate the committed Pyraview wheel index from a GitHub release. + + python tools/make_pyraview_index.py v0.4.2 # rewrite both files + python tools/make_pyraview_index.py v0.4.2 --check # exit 1 if stale + python tools/make_pyraview_index.py v0.4.2 --release-json release.json + +Pyraview is not on PyPI. Its wheels -- one per platform and CPython version, +each with the compiled library inside -- are assets on the GitHub release. +This writes two files from that release, both under install/: + +``pyraview-wheels.html`` + A flat index (PEP 503 "find-links" page): one ```` per wheel, its + ``href`` the release download URL with a ``#sha256=`` fragment. Hand it + to ``--find-links``, as a local path or by its raw.githubusercontent.com + URL. The sdist is left OUT on purpose: building Pyraview needs CMake and + a C++ compiler, and an install that cannot find a wheel must fail rather + than quietly compile. + +``pyraview-constraints.txt`` + ``pyraview==`` with one ``--hash`` per wheel. Passed as + ``--constraint`` (``uv pip install -c`` / ``uv tool install -c``), it + makes uv reject any pyraview artifact whose sha256 is not in the list -- + whichever index it came from. + +The hashes are the ``digest`` GitHub computes for each release asset, so +nothing is downloaded. ``--release-json`` reads a saved API response instead +of calling the API (for a machine that cannot reach api.github.com). +``GITHUB_TOKEN``, when set, is sent to lift the anonymous rate limit. + +``--check`` regenerates in memory and compares with what is committed; CI +runs it so the committed hashes are proved equal to GitHub's own digests. +""" + +from __future__ import annotations + +import argparse +import html +import json +import os +import sys +import urllib.request +from pathlib import Path + +REPO = "VH-Lab/Pyraview" +ROOT = Path(__file__).resolve().parent.parent +INDEX_PATH = ROOT / "install" / "pyraview-wheels.html" +CONSTRAINTS_PATH = ROOT / "install" / "pyraview-constraints.txt" + + +def fetch_release(tag: str) -> dict: + """Return the GitHub API description of release TAG.""" + url = f"https://api.github.com/repos/{REPO}/releases/tags/{tag}" + headers = {"Accept": "application/vnd.github+json"} + token = os.environ.get("GITHUB_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=60) as r: + return json.load(r) + + +def wheels(release: dict) -> list[tuple[str, str, str]]: + """Return (filename, sha256, url) for every wheel asset, sorted by name.""" + out = [] + for asset in release["assets"]: + name = asset["name"] + if not name.endswith(".whl"): + continue # the sdist, the MATLAB toolbox, the mex files, the zips + digest = asset.get("digest") or "" + if not digest.startswith("sha256:"): + raise SystemExit(f"{name}: release asset has no sha256 digest ({digest!r})") + out.append((name, digest.removeprefix("sha256:"), asset["browser_download_url"])) + if not out: + raise SystemExit(f"release {release.get('tag_name')} has no wheel assets") + return sorted(out) + + +def version_of(wheel_list: list[tuple[str, str, str]]) -> str: + """The single version every wheel in the list carries.""" + versions = {name.split("-")[1] for name, _, _ in wheel_list} + if len(versions) != 1: + raise SystemExit(f"wheels carry more than one version: {sorted(versions)}") + return versions.pop() + + +def render_index(tag: str, wheel_list: list[tuple[str, str, str]]) -> str: + lines = [ + "", + "", + "", + 'pyraview wheels', + "", + ] + for name, sha, url in wheel_list: + href = html.escape(f"{url}#sha256={sha}", quote=True) + lines.append(f'{html.escape(name)}
') + lines += ["", "", ""] + return "\n".join(lines) + + +def render_constraints(tag: str, wheel_list: list[tuple[str, str, str]]) -> str: + version = version_of(wheel_list) + lines = [ + f"# Generated by tools/make_pyraview_index.py from {REPO} release {tag}.", + "# Do not edit by hand. One --hash per wheel in install/pyraview-wheels.html;", + "# passed as --constraint, uv refuses any pyraview file not listed here.", + f"pyraview=={version} \\", + ] + hashes = [f" --hash=sha256:{sha}" for _, sha, _ in wheel_list] + lines += [h + " \\" for h in hashes[:-1]] + [hashes[-1], ""] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("tag", help="release tag, e.g. v0.4.2") + parser.add_argument("--release-json", type=Path, help="read the API response from a file") + parser.add_argument("--check", action="store_true", help="fail if the files are stale") + args = parser.parse_args(argv) + + if args.release_json: + release = json.loads(args.release_json.read_text()) + else: + release = fetch_release(args.tag) + if release.get("tag_name") != args.tag: + raise SystemExit(f"release JSON is for {release.get('tag_name')!r}, not {args.tag!r}") + + wheel_list = wheels(release) + wanted = { + INDEX_PATH: render_index(args.tag, wheel_list), + CONSTRAINTS_PATH: render_constraints(args.tag, wheel_list), + } + + if args.check: + stale = [p for p, text in wanted.items() if not p.exists() or p.read_text() != text] + for p in stale: + print(f"STALE: {p.relative_to(ROOT)} does not match release {args.tag}") + if not stale: + print(f"OK: {len(wheel_list)} wheels, hashes match GitHub's digests for {args.tag}") + return 1 if stale else 0 + + for p, text in wanted.items(): + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text, newline="\n") + print(f"wrote {p.relative_to(ROOT)}") + print(f"{len(wheel_list)} wheels from {args.tag}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())