From 7da5ae8f2e9614a571f9f5ee0a2875d969d22b4b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 00:22:08 +0000 Subject: [PATCH 1/3] Add a double-click end-user installer; make the viewers MATLAB-safe install/: install-ndi.sh (Linux, macOS), install-ndi.ps1 (Windows), the double-clickable "Install NDI.command" / "Install NDI.cmd", and a README for end users. A pinned uv (0.11.32) installs Python 3.12 and ndi[gui,napari,pyraview] as a uv tool under ~/.ndi/{bin,python,venvs} only, with Pyraview from its release wheels (--no-build-package pyraview). --uninstall removes exactly those folders and the marked PATH block. NDI_INSTALL_SOURCE / NDI_PYRAVIEW_* / NDI_INSTALL_PYTHON let CI install the code under test. napariViewGEF and napariViewLightsheet now go through src/_ndi_launch.py, a stdlib-only module beside the package: started from MATLAB it drops LD_LIBRARY_PATH, LD_PRELOAD, DYLD_*, PYTHONHOME and PYTHONPATH and re-execs BEFORE `import ndi` loads numpy (cleans PATH in place on Windows). Replaces the shell wrappers tools/napariViewGEF and scripts/napariViewLightsheet. GUI.GEFManager.ViewerLauncher now defaults to "" (find it automatically). .github/workflows/installer.yml runs the real installer with git and compilers hidden on Linux x86_64/aarch64, Windows, macOS arm64, macOS Intel and x86_64-under-Rosetta, then checks the commands, the MATLAB re-exec, the PATH block and a clean uninstall. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015KyLF4xAq4Sm4aT5gDyMTS --- .gitattributes | 7 + .github/workflows/installer.yml | 317 +++++++++++++++++ CHANGELOG.md | 23 ++ README.md | 11 + docs/getting-started.md | 6 + install/Install NDI.cmd | 23 ++ install/Install NDI.command | 36 ++ install/README.md | 98 +++++ install/install-ndi.ps1 | 255 +++++++++++++ install/install-ndi.sh | 291 +++++++++++++++ pyproject.toml | 19 +- scripts/napariViewLightsheet | 26 -- src/_ndi_launch.py | 335 ++++++++++++++++++ .../ndi_matlab_python_bridge.yaml | 14 + src/ndi/gui/app/genepyramid/cli.py | 16 +- .../genepyramid/ndi_matlab_python_bridge.yaml | 14 +- .../lightsheetZarr/README-lightsheet-zarr.md | 25 +- src/ndi/gui/app/lightsheetZarr/cli.py | 15 +- .../ndi_matlab_python_bridge.yaml | 27 +- src/ndi/ndi_matlab_python_bridge.yaml | 7 + src/ndi/preferences.py | 13 +- tests/test_ndi_launch.py | 298 ++++++++++++++++ tools/napariViewGEF | 46 --- 23 files changed, 1807 insertions(+), 115 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/installer.yml create mode 100644 install/Install NDI.cmd create mode 100755 install/Install NDI.command create mode 100644 install/README.md create mode 100644 install/install-ndi.ps1 create mode 100755 install/install-ndi.sh delete mode 100644 scripts/napariViewLightsheet create mode 100644 src/_ndi_launch.py create mode 100644 tests/test_ndi_launch.py delete mode 100755 tools/napariViewGEF diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..eb0cf7c9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# The Windows double-click installer is stored with CRLF line endings exactly +# as committed (-text: no conversion either way), so a copy downloaded from +# GitHub -- raw, zip or clone -- is a well-formed batch file. +install/*.cmd -text +# The macOS/Linux installer scripts must stay LF wherever they are checked out. +install/*.sh text eol=lf +install/*.command text eol=lf diff --git a/.github/workflows/installer.yml b/.github/workflows/installer.yml new file mode 100644 index 00000000..3ec27cd5 --- /dev/null +++ b/.github/workflows/installer.yml @@ -0,0 +1,317 @@ +name: Installer + +# Runs the REAL end-user installer (install/install-ndi.sh, install/install-ndi.ps1) +# on every platform it supports, with git and compilers unavailable, against +# THIS commit's code -- not main -- through the installer's documented +# NDI_INSTALL_SOURCE override. Then checks what MATLAB relies on: the three +# commands in ~/.ndi/bin, a MATLAB-polluted environment left behind by the +# viewers' launcher, and an uninstall that removes only what it installed. +# +# Each run downloads ~2 GB per leg, hence the path filter and weekly schedule +# rather than every push. The weekly run catches a dependency that stops +# installing cleanly (a wheel withdrawn for one platform, say) with no change +# here. + +on: + pull_request: + paths: + - "install/**" + - "pyproject.toml" + - "src/_ndi_launch.py" + - "src/ndi/gui/app/genepyramid/cli.py" + - "src/ndi/gui/app/lightsheetZarr/cli.py" + - ".github/workflows/installer.yml" + push: + branches: [main] + paths: + - "install/**" + - "pyproject.toml" + - "src/_ndi_launch.py" + - "src/ndi/gui/app/genepyramid/cli.py" + - "src/ndi/gui/app/lightsheetZarr/cli.py" + - ".github/workflows/installer.yml" + workflow_dispatch: + schedule: + - cron: "23 7 * * 1" # Mondays 07:23 UTC + +permissions: + contents: read + +jobs: + install: + name: Installer (${{ matrix.leg }}) + runs-on: ${{ matrix.os }} + # Bounds a hung install. (It does not bound time spent QUEUED for a + # runner label: if macos-15-intel is ever retired, drop that leg.) + timeout-minutes: 60 + concurrency: + group: installer-${{ matrix.leg }}-${{ github.ref }} + cancel-in-progress: true + strategy: + fail-fast: false + matrix: + include: + - leg: linux-x86_64 + os: ubuntu-latest + machine: x86_64 + tags: "manylinux_ x86_64" + - leg: linux-aarch64 + os: ubuntu-24.04-arm + machine: aarch64 + tags: "manylinux_ aarch64" + - leg: windows-amd64 + os: windows-latest + machine: AMD64 + tags: "win_amd64" + - leg: macos-arm64 + os: macos-latest + machine: arm64 + tags: "macosx_ arm64" + - leg: macos-intel + os: macos-15-intel + machine: x86_64 + tags: "macosx_ x86_64" + # An Intel Mac, as Apple Silicon sees one: an x86_64 CPython under + # Rosetta 2, requested through NDI_INSTALL_PYTHON (ci.yml's approach). + # uv picks wheels for the interpreter it installs into, so this + # resolves exactly what an Intel Mac would. (Running the installer + # under `arch -x86_64` would NOT do it: Astral's installer sees + # through Rosetta with sysctl and installs the arm64 uv, which would + # then fetch an arm64 Python.) + - leg: macos-rosetta-x86_64 + os: macos-latest + machine: x86_64 + tags: "macosx_ x86_64" + rosetta: true + python: cpython-3.12-macos-x86_64 + defaults: + run: + shell: bash + env: + # Raw URL of THIS commit's wheel index: uv reads --find-links only from a + # URL or a directory. Same as the install-without-git job in ci.yml. + NDI_PYRAVIEW_FIND_LINKS: https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name || github.repository }}/${{ github.event.pull_request.head.sha || github.sha }}/install/pyraview-wheels.html + steps: + # The checkout itself uses git; that is fine. Only the installer must + # not need it. + - uses: actions/checkout@v4 + + - name: Make sure Rosetta 2 is available + if: matrix.rosetta + run: | + /usr/bin/pgrep -q oahd || sudo softwareupdate --install-rosetta --agree-to-license + arch -x86_64 /usr/bin/true + + - name: Hide git, CMake and the C/C++ compilers + run: | + # Copied from ci.yml's install-without-git job. Writes a script the + # install step sources. On Linux and macOS it SHADOWS each tool with + # one that always fails; on Windows every PATH directory holding one + # of these tools is dropped instead (native programs look for + # git.exe, not a shell script). + tools="git cmake cc c++ gcc g++ clang clang++ cl" + hide="$RUNNER_TEMP/hide-tools.sh" + if [ "$RUNNER_OS" = Windows ]; then + cat > "$hide" <&2\nexit 127\n' "$t" \ + > "$RUNNER_TEMP/notools/$t" + chmod +x "$RUNNER_TEMP/notools/$t" + done + echo "export PATH=\"$RUNNER_TEMP/notools:\$PATH\"" > "$hide" + fi + echo "HIDE_TOOLS=$hide" >> "$GITHUB_ENV" + # shellcheck source=/dev/null + ( source "$hide" + for t in git cmake c++ gcc; do + if "$t" --version >/dev/null 2>&1; then + echo "::error::$t is still usable; this job would prove nothing" + exit 1 + fi + done ) + echo "git, cmake, c++ and gcc are all unusable" + + - name: Set up what the installer must leave alone + run: | + # Files in ~/.ndi the installer does not own (NDI-matlab shares the + # folder), plus the shell start-up files and uv's update receipt, + # recorded so the uninstall step can prove they are as they were. + mkdir -p "$HOME/.ndi/tools/somerepo" + echo '{"dummy": "do not touch"}' > "$HOME/.ndi/NDI_Preferences.json" + echo 'keep me' > "$HOME/.ndi/tools/somerepo/README" + mkdir -p "$RUNNER_TEMP/before" + cp -R "$HOME/.ndi" "$RUNNER_TEMP/before/ndi" + if [ "$RUNNER_OS" != Windows ]; then + touch "$HOME/.bashrc" # so the bash block has a file to go in + for f in .zshrc .zprofile .bashrc .profile .bash_profile; do + if [ -f "$HOME/$f" ]; then cp "$HOME/$f" "$RUNNER_TEMP/before/$f"; fi + done + receipt="${XDG_CONFIG_HOME:-$HOME/.config}/uv/uv-receipt.json" + else + receipt="$LOCALAPPDATA/uv/uv-receipt.json" + powershell.exe -NoProfile -Command \ + "(Get-Item 'registry::HKEY_CURRENT_USER\Environment').GetValue('Path', '', 'DoNotExpandEnvironmentNames')" \ + | tr -d '\r' > "$RUNNER_TEMP/before/userpath.txt" + fi + echo "RECEIPT=$receipt" >> "$GITHUB_ENV" + if [ -e "$receipt" ]; then echo "RECEIPT_EXISTED=1" >> "$GITHUB_ENV"; fi + echo "SHELL=$SHELL" + + - name: Run the installer (git and compilers hidden) + env: + NDI_INSTALL_SOURCE: ${{ github.workspace }} + NDI_PYRAVIEW_CONSTRAINTS: ${{ github.workspace }}/install/pyraview-constraints.txt + NDI_INSTALL_PYTHON: ${{ matrix.python }} + run: | + tee=$(command -v tee) + log="$RUNNER_TEMP/install-out.log" + # shellcheck source=/dev/null + ( source "$HIDE_TOOLS" + if [ "$RUNNER_OS" = Windows ]; then + # Windows PowerShell 5.1, the oldest the script supports. + powershell.exe -NoProfile -ExecutionPolicy Bypass -File install/install-ndi.ps1 -Yes + else + sh install/install-ndi.sh --yes + fi ) | "$tee" "$log" + last=$(tail -n 1 "$log" | tr -d '\r') + if [ "$RUNNER_OS" = Windows ]; then + expected="NDI installed: $USERPROFILE\\.ndi\\bin" + else + expected="NDI installed: $HOME/.ndi/bin" + fi + if [ "$last" != "$expected" ]; then + echo "::error::last line was [$last], expected [$expected]" + exit 1 + fi + + - name: The commands MATLAB calls are in ~/.ndi/bin and run + run: | + bin="$HOME/.ndi/bin" + ls -l "$bin" + if [ "$RUNNER_OS" = Windows ]; then x=.exe; else x=; fi + for c in uv ndi napariViewGEF napariViewLightsheet; do + [ -f "$bin/$c$x" ] || { echo "::error::$bin/$c$x is missing"; exit 1; } + done + for c in ndi napariViewGEF napariViewLightsheet; do + echo "--- $c --help" + "$bin/$c$x" --help + done + "$bin/uv$x" --version + + - name: The tool environment is this platform's, and pyraview a release wheel + working-directory: ${{ runner.temp }} + run: | + if [ "$RUNNER_OS" = Windows ]; then + py="$HOME/.ndi/venvs/ndi/Scripts/python.exe" + else + py="$HOME/.ndi/venvs/ndi/bin/python" + fi + "$py" -c "import platform, sys; print(sys.version, platform.machine()); sys.exit(0 if platform.machine() == '${{ matrix.machine }}' else 'wrong machine')" + # A uv-managed interpreter in ~/.ndi/python, not a system one. + "$py" -c "import sys, pathlib; b = pathlib.Path(sys.base_prefix).resolve(); print(b); sys.exit(0 if (pathlib.Path.home() / '.ndi' / 'python').resolve() in b.parents else 'not the managed python')" + tag_args=() + for t in ${{ matrix.tags }}; do tag_args+=(--expect-tag "$t"); done + "$py" "$GITHUB_WORKSPACE/tools/check_pyraview_wheel.py" \ + --expect-machine "${{ matrix.machine }}" "${tag_args[@]}" + + - name: Started as MATLAB would, the viewer leaves MATLAB's paths behind + run: | + # NDI_LAUNCH_DEBUG makes the final process report its environment + # (src/_ndi_launch.py). On Linux and macOS that must be a re-exec'd + # process with every loader variable gone; on Windows, an in-place + # clean with MATLAB's bin dir off PATH. + err="$RUNNER_TEMP/launch.err" + if [ "$RUNNER_OS" = Windows ]; then + NDI_FROM_MATLAB=1 NDI_LAUNCH_DEBUG=1 PYTHONPATH='C:\junk' \ + PATH="/c/junk/MATLAB/R2026a/bin/win64:$PATH" \ + "$HOME/.ndi/bin/napariViewGEF.exe" --help > /dev/null 2> "$err" + want='"cleaned": "in-place"' + else + NDI_FROM_MATLAB=1 NDI_LAUNCH_DEBUG=1 \ + LD_LIBRARY_PATH=/opt/junk/MATLAB/R2026a/sys/os/glnxa64:/opt/junk/MATLAB/R2026a/sys/opengl/lib/glnxa64 \ + LD_PRELOAD=/opt/junk/libjunk.so \ + DYLD_LIBRARY_PATH=/opt/junk/MATLAB/bin/maca64 \ + DYLD_FALLBACK_LIBRARY_PATH=/opt/junk/MATLAB/sys/os/maca64 \ + PYTHONPATH=/opt/junk/python \ + "$HOME/.ndi/bin/napariViewGEF" --help > /dev/null 2> "$err" + want='"cleaned": "reexec"' + fi + cat "$err" + report=$(grep '^ndi-launch: ' "$err" | tr -d '\r') + [ "$(printf '%s\n' "$report" | wc -l)" -eq 1 ] || { echo "::error::expected one report line"; exit 1; } + case "$report" in *"$want"*) ;; *) echo "::error::report lacks $want"; exit 1 ;; esac + for v in LD_LIBRARY_PATH LD_PRELOAD DYLD_LIBRARY_PATH DYLD_FALLBACK_LIBRARY_PATH PYTHONPATH PYTHONHOME; do + case "$report" in *"\"$v\": null"*) ;; *) echo "::error::$v survived"; exit 1 ;; esac + done + # And launched normally, nothing is touched. + NDI_LAUNCH_DEBUG=1 "$HOME/.ndi/bin/napariViewGEF" --help > /dev/null 2> "$err" + grep -q '"cleaned": null' "$err" || { cat "$err"; echo "::error::an ordinary launch was cleaned"; exit 1; } + + - name: The PATH was updated once + run: | + if [ "$RUNNER_OS" = Windows ]; then + userpath=$(powershell.exe -NoProfile -Command \ + "(Get-Item 'registry::HKEY_CURRENT_USER\Environment').GetValue('Path', '', 'DoNotExpandEnvironmentNames')" | tr -d '\r') + echo "$userpath" + n=$(printf '%s' "$userpath" | tr ';' '\n' | grep -ixcF "$USERPROFILE\\.ndi\\bin" || true) + else + n=0 + for f in .zshrc .zprofile .bashrc .profile .bash_profile; do + if [ -f "$HOME/$f" ]; then + c=$(grep -c '^# >>> NDI: added by install-ndi.sh' "$HOME/$f" || true) + echo "$f: $c block(s)" + [ "$c" -le 1 ] || { echo "::error::$f has $c NDI blocks"; exit 1; } + n=$((n + c)) + fi + done + # A new login shell of the user's kind finds the commands. + "$SHELL" -lc 'command -v napariViewGEF' + fi + [ "$n" -ge 1 ] || { echo "::error::the PATH was not updated"; exit 1; } + + - name: No uv update receipt was written + run: | + if [ -z "${RECEIPT_EXISTED:-}" ] && [ -e "$RECEIPT" ]; then + cat "$RECEIPT" + echo "::error::the installer wrote $RECEIPT" + exit 1 + fi + + - name: Uninstall removes only what was installed + run: | + if [ "$RUNNER_OS" = Windows ]; then + powershell.exe -NoProfile -ExecutionPolicy Bypass -File install/install-ndi.ps1 -Uninstall + else + sh install/install-ndi.sh --uninstall + fi + for d in bin python venvs; do + if [ -e "$HOME/.ndi/$d" ]; then echo "::error::~/.ndi/$d is still there"; exit 1; fi + done + # Everything else in ~/.ndi is exactly as it was. + diff -r "$RUNNER_TEMP/before/ndi" "$HOME/.ndi" + if [ "$RUNNER_OS" = Windows ]; then + powershell.exe -NoProfile -Command \ + "(Get-Item 'registry::HKEY_CURRENT_USER\Environment').GetValue('Path', '', 'DoNotExpandEnvironmentNames')" \ + | tr -d '\r' > "$RUNNER_TEMP/after-userpath.txt" + diff "$RUNNER_TEMP/before/userpath.txt" "$RUNNER_TEMP/after-userpath.txt" + else + for f in .zshrc .zprofile .bashrc .profile .bash_profile; do + if [ -f "$RUNNER_TEMP/before/$f" ]; then + diff "$RUNNER_TEMP/before/$f" "$HOME/$f" + elif [ -f "$HOME/$f" ]; then + echo "::error::$f was created and not removed"; exit 1 + fi + done + fi + echo "uninstall left everything else as it was" diff --git a/CHANGELOG.md b/CHANGELOG.md index 09a89594..9cc5d23a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,29 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added +- **A double-click installer for end users** in `install/`: `Install NDI.command` + (macOS), `Install NDI.cmd` (Windows), and the scripts behind them, + `install-ndi.sh` (Linux and macOS) and `install-ndi.ps1` (Windows), which + NDI-matlab also runs. A pinned uv (0.11.32) installs Python 3.12 and + `ndi[gui,napari,pyraview]` as a uv tool, entirely under `~/.ndi/bin`, + `~/.ndi/python` and `~/.ndi/venvs`, with no git, compiler or admin rights; + `--uninstall` removes exactly those and the PATH lines. The + `.github/workflows/installer.yml` workflow runs the real installer, with git + hidden, on Linux x86_64/aarch64, Windows, macOS arm64, macOS Intel and + x86_64-under-Rosetta. + +### Changed + +- **`napariViewGEF` and `napariViewLightsheet` are safe to start from MATLAB + by themselves.** They now go through `_ndi_launch`, a stdlib-only module + beside the package: started with `NDI_FROM_MATLAB=1` (or with MATLAB's + directories on the loader path), it drops `LD_LIBRARY_PATH`, `LD_PRELOAD`, + `DYLD_*`, `PYTHONHOME` and `PYTHONPATH` and re-execs before `import ndi` + loads numpy and friends (on Windows: cleans `PATH` in place). The shell + wrappers `tools/napariViewGEF` and `scripts/napariViewLightsheet` that did + this are removed, and the `GUI.GEFManager.ViewerLauncher` preference now + defaults to `""`, meaning "find napariViewGEF automatically". + - **Pyraview from pre-built wheels, never from source.** New `pyraview` extra (`pyraview==0.4.2`). Pyraview is not on PyPI, so the wheels on its GitHub release are named in a committed flat index, diff --git a/README.md b/README.md index 0e934c72..a7e5a9f9 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,17 @@ NDI provides a unified interface for working with multi-modal neuroscience data ## Installation +**Just want to use NDI** (the `ndi` command and the napari viewers, also from +NDI-matlab)? Use the double-click installer for macOS, Windows and Linux in +[`install/`](install/README.md): no Python, git or compiler needed, and +everything goes in `~/.ndi`. One line from a terminal: + +```bash +curl -LsSf https://raw.githubusercontent.com/Waltham-Data-Science/NDI-python/main/install/install-ndi.sh | sh +``` + +**Developing NDI**, or using it from your own Python environment: + ```bash git clone https://github.com/Waltham-Data-Science/NDI-python.git cd NDI-python diff --git a/docs/getting-started.md b/docs/getting-started.md index f2f4ba7a..835701ca 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -2,6 +2,12 @@ ## Installation +To *use* NDI (the `ndi` command and the napari viewers, also from NDI-matlab) +without setting up Python yourself, use the double-click installer in +[`install/`](https://github.com/Waltham-Data-Science/NDI-python/tree/main/install): +it needs no Python, git or compiler and puts everything in `~/.ndi`. The rest +of this page is for working with NDI from your own Python environment. + ### Prerequisites - Python 3.10 or later diff --git a/install/Install NDI.cmd b/install/Install NDI.cmd new file mode 100644 index 00000000..7f2d5ad2 --- /dev/null +++ b/install/Install NDI.cmd @@ -0,0 +1,23 @@ +@echo off +rem Double-click this file to install NDI (Windows). +rem +rem It runs install-ndi.ps1 from the same folder, or downloads it if this file +rem was saved on its own, and waits at the end so you can read what happened. +rem -ExecutionPolicy Bypass applies to this one run only; it changes no setting. +rem If Windows SmartScreen warns about the file, choose "More info", then "Run anyway". + +setlocal +set "SCRIPT=%~dp0install-ndi.ps1" +set "URL=https://raw.githubusercontent.com/Waltham-Data-Science/NDI-python/main/install/install-ndi.ps1" + +if exist "%SCRIPT%" ( + powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%SCRIPT%" %* +) else ( + powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12; irm '%URL%' | iex" +) +set "STATUS=%ERRORLEVEL%" + +echo. +if "%STATUS%"=="0" (echo Done.) else (echo NDI was not installed; the reason is above.) +pause +exit /b %STATUS% diff --git a/install/Install NDI.command b/install/Install NDI.command new file mode 100755 index 00000000..dafe0e93 --- /dev/null +++ b/install/Install NDI.command @@ -0,0 +1,36 @@ +#!/bin/sh +# Double-click this file in the Finder to install NDI (macOS). +# +# It runs install-ndi.sh from the same folder, or downloads it if this file +# was saved on its own, and keeps the Terminal window open at the end so you +# can read what happened. The first time, macOS may refuse to open it +# because it was downloaded: right-click (or Control-click) it, choose Open, +# then Open again. See README.md in this folder. + +URL="https://raw.githubusercontent.com/Waltham-Data-Science/NDI-python/main/install/install-ndi.sh" + +cd "$(dirname "$0")" || exit 1 +if [ -f ./install-ndi.sh ]; then + sh ./install-ndi.sh "$@" + status=$? +else + tmp=$(mktemp) || exit 1 + if curl --proto '=https' --tlsv1.2 -LsSf "$URL" -o "$tmp"; then + sh "$tmp" "$@" + status=$? + else + echo "NDI install failed: could not download install-ndi.sh (check your connection)" >&2 + status=1 + fi + rm -f "$tmp" +fi + +echo +if [ "$status" -eq 0 ]; then + echo "Done." +else + echo "NDI was not installed; the reason is above." +fi +printf 'Press Return to close this window. ' +read -r _ +exit "$status" diff --git a/install/README.md b/install/README.md new file mode 100644 index 00000000..076de456 --- /dev/null +++ b/install/README.md @@ -0,0 +1,98 @@ +# Installing NDI + +This folder installs NDI-python on **macOS** (Apple Silicon and Intel), +**Windows** and **Linux**, for people who want to *use* NDI: the `ndi` +command and the two napari viewers, `napariViewGEF` and +`napariViewLightsheet`, which NDI-matlab also opens. You do not need Python, +git, a compiler or administrator rights. (Developers working on NDI itself +should use `pip install -e .` in a clone instead.) + +## Install + +**macOS** — download [`Install NDI.command`](Install%20NDI.command) and +double-click it. The first time, macOS may say it cannot be opened because it +was downloaded from the internet: **right-click** (or Control-click) the file, +choose **Open**, then **Open** again. A Terminal window shows the progress and +waits for you to press Return at the end. + +**Windows** — download [`Install NDI.cmd`](Install%20NDI.cmd) and +double-click it. If SmartScreen warns about it, choose **More info**, then +**Run anyway**. A window shows the progress and waits for a key press at the end. + +**Linux, or from any terminal** — one line: + +```sh +curl -LsSf https://raw.githubusercontent.com/Waltham-Data-Science/NDI-python/main/install/install-ndi.sh | sh +``` + +On Windows, the PowerShell equivalent: + +```powershell +powershell -ExecutionPolicy Bypass -NoProfile -c "irm https://raw.githubusercontent.com/Waltham-Data-Science/NDI-python/main/install/install-ndi.ps1 | iex" +``` + +It takes a few minutes and about 3 GB of disk. When it finishes, **open a new +terminal window** and try `ndi check`. + +## What it puts where + +Everything goes in a hidden `.ndi` folder in your home folder +(`~/.ndi` on macOS and Linux, `%USERPROFILE%\.ndi` on Windows), which +NDI-matlab uses too. The installer owns only three folders there: + +| folder | what is in it | +|---|---| +| `.ndi/bin` | the `ndi`, `napariViewGEF` and `napariViewLightsheet` commands, and [uv](https://docs.astral.sh/uv/), the tool that installs them | +| `.ndi/python` | a private copy of Python 3.12 | +| `.ndi/venvs` | NDI and the packages it needs | + +It never touches anything else in `.ndi`, such as your +`NDI_Preferences.json`. Outside `.ndi` it only adds `.ndi/bin` to the end of +your `PATH` (in `~/.zshrc` / `~/.zprofile` or `~/.bashrc` / `~/.profile`, inside +a block marked `NDI`; on Windows, your user `Path` setting). uv also keeps a +download cache in its usual place (`~/.cache/uv`, or `%LOCALAPPDATA%\uv\cache` +on Windows), which makes updates faster; `~/.ndi/bin/uv cache clean` empties it. + +## Update + +Run the installer again, the same way. It replaces NDI with the latest version. + +## Uninstall + +```sh +curl -LsSf https://raw.githubusercontent.com/Waltham-Data-Science/NDI-python/main/install/install-ndi.sh | sh -s -- --uninstall +``` + +```powershell +powershell -ExecutionPolicy Bypass -NoProfile -c "& ([scriptblock]::Create((irm https://raw.githubusercontent.com/Waltham-Data-Science/NDI-python/main/install/install-ndi.ps1))) -Uninstall" +``` + +(or run `install-ndi.sh --uninstall` / `install-ndi.ps1 -Uninstall` from this +folder). This removes `.ndi/bin`, `.ndi/python`, `.ndi/venvs` and the `PATH` +lines, and leaves the rest of `.ndi` (your settings) alone. + +## Options + +| `install-ndi.sh` | `install-ndi.ps1` | | +|---|---|---| +| `--yes` | `-Yes` | do not ask before installing | +| `--no-path` | `-NoPath` | do not change `PATH` | +| `--uninstall` | `-Uninstall` | remove what the installer installed | + +## For MATLAB and for maintainers + +NDI-matlab downloads and runs these same scripts with `--yes` / `-Yes`. The +script exits with status 0 only if the final check passed (the three +commands exist, both viewers answer `--help`, and `ndi check` passes); on +success the last line it prints is `NDI installed: `. +MATLAB starts the viewers with `NDI_FROM_MATLAB=1`, and they then drop +MATLAB's library paths before loading anything (see `src/_ndi_launch.py`). + +The uv version is pinned at the top of each script (`UV_VERSION` / +`$UvVersion`). Pyraview comes only from its release wheels, through +[`pyraview-wheels.html`](pyraview-wheels.html) and +[`pyraview-constraints.txt`](pyraview-constraints.txt) here, so nothing is +compiled. `NDI_INSTALL_SOURCE` (a local checkout or a URL) installs something +other than the `main` branch; CI uses it, together with +`NDI_PYRAVIEW_FIND_LINKS`, `NDI_PYRAVIEW_CONSTRAINTS` and `NDI_INSTALL_PYTHON`, +to install the code under test (`.github/workflows/installer.yml`). diff --git a/install/install-ndi.ps1 b/install/install-ndi.ps1 new file mode 100644 index 00000000..daace928 --- /dev/null +++ b/install/install-ndi.ps1 @@ -0,0 +1,255 @@ +<# +.SYNOPSIS + Install NDI-python for end users on Windows. + +.DESCRIPTION + powershell -ExecutionPolicy Bypass -NoProfile -c "irm https://raw.githubusercontent.com/Waltham-Data-Science/NDI-python/main/install/install-ndi.ps1 | iex" + + or double-click "Install NDI.cmd", or run this file. Run it again to update. + Developers want `pip install -e .` instead; this is for people who just want + the ndi, napariViewGEF and napariViewLightsheet commands. + + WHAT IT TOUCHES. Everything lives under %USERPROFILE%\.ndi, which NDI-matlab + shares, and this script owns ONLY three folders there: + .ndi\bin uv itself, plus ndi.exe / napariViewGEF.exe / napariViewLightsheet.exe + .ndi\python the Python interpreter uv downloads (UV_PYTHON_INSTALL_DIR) + .ndi\venvs the environment NDI is installed into (UV_TOOL_DIR) + Everything else in .ndi (NDI_Preferences.json, tools, caches, ...) is never + touched, by install or by -Uninstall. Outside .ndi it changes only your + user-level PATH (unless -NoPath); and uv keeps its download cache in its + usual place (%LOCALAPPDATA%\uv\cache), shared with any other uv you use. + No administrator rights, ever. + + FOR TESTING (CI uses these; end users never need them): + NDI_INSTALL_SOURCE what to install instead of the main branch: + a local checkout directory, or a URL + NDI_PYRAVIEW_FIND_LINKS URL of the Pyraview wheel index (install/pyraview-wheels.html) + NDI_PYRAVIEW_CONSTRAINTS URL or path of install/pyraview-constraints.txt + NDI_INSTALL_PYTHON the uv Python request instead of 3.12 + + This is the contract NDI-matlab codes against: the folders above, the + switches, exit status 0 only when the post-install check passed, and, on + success, a last line of output reading `NDI installed: `. + Keep install/install-ndi.sh in step with it. + +.PARAMETER Yes + Do not ask before installing (MATLAB always passes this). +.PARAMETER NoPath + Do not add .ndi\bin to the user PATH. +.PARAMETER Uninstall + Remove what this script installed, and nothing else. +#> +[CmdletBinding()] +param( + [switch]$Yes, + [switch]$NoPath, + [switch]$Uninstall +) + +# Works in Windows PowerShell 5.1 and PowerShell 7: no ?? or ?: operators. +Set-StrictMode -Version 2 +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest's bar is very slow on 5.1 + +# ---- pinned versions: the one place to change them --------------------------- +$UvVersion = '0.11.32' # https://github.com/astral-sh/uv/releases +$PythonVersion = '3.12' +# ------------------------------------------------------------------------------ + +$RepoRaw = 'https://raw.githubusercontent.com/Waltham-Data-Science/NDI-python/main' +$DefaultSource = 'https://github.com/Waltham-Data-Science/NDI-python/archive/refs/heads/main.zip' +# gui: the PySide6 apps; napari: the two viewers; pyraview: signal pyramids. +$Extras = 'gui,napari,pyraview' + +$NdiHome = Join-Path $env:USERPROFILE '.ndi' +$Bin = Join-Path $NdiHome 'bin' +$PyDir = Join-Path $NdiHome 'python' +$Venvs = Join-Path $NdiHome 'venvs' + +function Fail([string]$Reason) { + # One line saying why, and a nonzero exit. [Console]::Error so it is + # plain text on 5.1 rather than a red ErrorRecord block. + [Console]::Error.WriteLine("NDI install failed: $Reason") + exit 1 +} + +# PowerShell 5.1 may not offer TLS 1.2 by default; GitHub and astral.sh need it. +[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + +# Started from MATLAB: nothing here needs Python's own variables, and they can +# point the Python that builds and checks NDI at the wrong library. (MATLAB's +# effect on Windows is unmeasured; see src/_ndi_launch.py.) +Remove-Item Env:PYTHONHOME, Env:PYTHONPATH -ErrorAction SilentlyContinue + +# ---- the user PATH ----------------------------------------------------------- +# Read and written as the raw registry value so %VARIABLES% in it stay +# unexpanded (REG_EXPAND_SZ): the stock entry %USERPROFILE%\AppData\Local\ +# Microsoft\WindowsApps would otherwise be flattened. The same approach uv's +# own installer takes. A dummy SetEnvironmentVariable broadcasts the change so +# new windows see it. + +$EnvKey = 'registry::HKEY_CURRENT_USER\Environment' + +function Get-UserPathEntries { + $raw = (Get-Item -LiteralPath $EnvKey).GetValue('Path', '', 'DoNotExpandEnvironmentNames') + return @($raw -split ';' | Where-Object { $_ -ne '' }) +} + +function Test-IsNdiBin([string]$Entry) { + return $Entry.TrimEnd('\') -ieq $Bin.TrimEnd('\') +} + +function Set-UserPathEntries([string[]]$Entries) { + Set-ItemProperty -Type ExpandString -LiteralPath $EnvKey -Name Path -Value ($Entries -join ';') + $dummy = 'NDI-installer-' + [guid]::NewGuid().ToString() + [Environment]::SetEnvironmentVariable($dummy, '1', 'User') + [Environment]::SetEnvironmentVariable($dummy, [NullString]::Value, 'User') +} + +function Add-NdiToPath { + $entries = Get-UserPathEntries + if (@($entries | Where-Object { Test-IsNdiBin $_ }).Count -gt 0) { + Write-Host " $Bin is already on your PATH" + return + } + # Appended, not prepended, so the uv in .ndi\bin never shadows one of yours. + Set-UserPathEntries (@($entries) + $Bin) + Write-Host " added $Bin to your PATH" +} + +function Remove-NdiFromPath { + $entries = Get-UserPathEntries + $kept = @($entries | Where-Object { -not (Test-IsNdiBin $_) }) + if ($kept.Count -ne $entries.Count) { + Set-UserPathEntries $kept + Write-Host " removed $Bin from your PATH" + } +} + +# ---- uninstall --------------------------------------------------------------- + +if ($Uninstall) { + Write-Host "Removing NDI from $NdiHome (bin, python, venvs) ..." + foreach ($d in @($Bin, $PyDir, $Venvs)) { + if (Test-Path -LiteralPath $d) { + try { Remove-Item -LiteralPath $d -Recurse -Force } + catch { Fail "could not remove $d (is an NDI program still running?)" } + } + } + Remove-NdiFromPath + Write-Host "Your NDI settings and anything else in $NdiHome were left in place." + Write-Output "NDI uninstalled: $Bin" + exit 0 +} + +# ---- install ----------------------------------------------------------------- + +Write-Host 'NDI installer' +Write-Host " installs into: $NdiHome (bin, python, venvs)" +Write-Host ' needs about 3 GB of disk and a few minutes on a good connection' +if (-not $Yes -and [Environment]::UserInteractive -and -not [Console]::IsInputRedirected) { + $answer = Read-Host 'Continue? [Y/n]' + if ($answer -and $answer -notmatch '^(y|yes)$') { Fail 'cancelled' } +} + +try { New-Item -ItemType Directory -Force -Path $Bin, $PyDir, $Venvs | Out-Null } +catch { Fail "could not create folders in $NdiHome" } + +# 1. uv, pinned. Astral's versioned installer, told to put uv in .ndi\bin. +# UV_UNMANAGED_INSTALL also stops it writing an update receipt (which would +# redirect a `uv self update` of a uv you already have to this copy) and +# stops it editing your PATH itself. It runs in a child PowerShell so its +# variables and settings cannot leak into this script. +$Uv = Join-Path $Bin 'uv.exe' +function Get-UvVersion { + if (-not (Test-Path -LiteralPath $Uv)) { return '' } + try { return ((& $Uv --version) -split ' ')[1] } catch { return '' } +} +if ((Get-UvVersion) -eq $UvVersion) { + Write-Host "uv $UvVersion is already in $Bin" +} else { + Write-Host "Downloading uv $UvVersion ..." + $env:UV_INSTALL_DIR = $Bin + $env:UV_UNMANAGED_INSTALL = $Bin + $env:UV_NO_MODIFY_PATH = '1' + $shell = (Get-Process -Id $PID).Path + $cmd = "[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12; " + + "irm https://astral.sh/uv/$UvVersion/install.ps1 | iex" + & $shell -NoProfile -ExecutionPolicy Bypass -Command $cmd + $uvExit = $LASTEXITCODE + Remove-Item Env:UV_INSTALL_DIR, Env:UV_UNMANAGED_INSTALL -ErrorAction SilentlyContinue + if ($uvExit -ne 0) { Fail 'could not download or run the uv installer from astral.sh' } + if ((Get-UvVersion) -ne $UvVersion) { Fail "uv $UvVersion did not end up in $Bin" } +} + +# 2. NDI, as a uv "tool": its own environment, its own managed Python, and its +# commands placed in .ndi\bin. Each setting keeps uv inside the three folders +# above and away from anything else on this machine: +$env:UV_PYTHON_INSTALL_DIR = $PyDir # the interpreter +$env:UV_TOOL_DIR = $Venvs # the environment +$env:UV_TOOL_BIN_DIR = $Bin # the commands +$env:UV_PYTHON_PREFERENCE = 'only-managed' # never a system/Store/conda Python that may vanish +$env:UV_NO_CONFIG = '1' # ignore any uv.toml of yours +$env:UV_NO_MODIFY_PATH = '1' + +$Source = [Environment]::GetEnvironmentVariable('NDI_INSTALL_SOURCE') +if (-not $Source) { $Source = $DefaultSource } +if (Test-Path -LiteralPath $Source) { + # A local checkout (CI): a file:/// URL to its absolute path. + $Source = ([System.Uri](Resolve-Path -LiteralPath $Source).ProviderPath).AbsoluteUri +} +$FindLinks = [Environment]::GetEnvironmentVariable('NDI_PYRAVIEW_FIND_LINKS') +if (-not $FindLinks) { $FindLinks = "$RepoRaw/install/pyraview-wheels.html" } +$Constraints = [Environment]::GetEnvironmentVariable('NDI_PYRAVIEW_CONSTRAINTS') +if (-not $Constraints) { $Constraints = "$RepoRaw/install/pyraview-constraints.txt" } +$PythonRequest = [Environment]::GetEnvironmentVariable('NDI_INSTALL_PYTHON') +if (-not $PythonRequest) { $PythonRequest = $PythonVersion } + +Write-Host "Installing NDI from $Source ..." +# --reinstall (which implies --refresh) so that running this again really +# updates: NDI and its GitHub-archive dependencies are re-fetched. +# Pyraview is not on PyPI: its release wheels are found through FindLinks, +# Constraints pins the version, and --no-build-package makes a missing wheel +# an error rather than a C++ compile. See pyproject.toml's pyraview extra. +# PATH includes .ndi\bin from here on (this process only), so uv does not warn that the +# folder is not on the PATH; step 4 deals with that. +$env:PATH = "$Bin;$env:PATH" +& $Uv tool install ` + --python $PythonRequest ` + --reinstall ` + --find-links $FindLinks ` + --constraints $Constraints ` + --no-build-package pyraview ` + "ndi[$Extras] @ $Source" +if ($LASTEXITCODE -ne 0) { Fail 'uv could not install NDI (see the messages above)' } + +# 3. Check the result: the commands MATLAB will call, and `ndi check`. +Write-Host 'Checking the installation ...' +foreach ($c in @('ndi', 'napariViewGEF', 'napariViewLightsheet')) { + if (-not (Test-Path -LiteralPath (Join-Path $Bin "$c.exe"))) { Fail "$Bin\$c.exe is missing" } +} +foreach ($c in @('napariViewGEF', 'napariViewLightsheet')) { + & (Join-Path $Bin "$c.exe") --help | Out-Null + if ($LASTEXITCODE -ne 0) { Fail "$c --help did not run" } +} +# 'Continue' while capturing: under 'Stop', Windows PowerShell 5.1 turns any +# line the program writes to stderr into a terminating error. +$ErrorActionPreference = 'Continue' +$checkOutput = & (Join-Path $Bin 'ndi.exe') check 2>&1 +$checkExit = $LASTEXITCODE +$ErrorActionPreference = 'Stop' +if ($checkExit -ne 0) { + $checkOutput | ForEach-Object { Write-Host $_ } + Fail 'ndi check reported problems (see above)' +} +Write-Host ' ndi check: all checks passed' + +# 4. PATH. +if (-not $NoPath) { + Add-NdiToPath + Write-Host 'Open a new terminal window to use the ndi, napariViewGEF and napariViewLightsheet commands.' +} else { + Write-Host "PATH left alone (-NoPath); the commands are in $Bin" +} + +Write-Output "NDI installed: $Bin" diff --git a/install/install-ndi.sh b/install/install-ndi.sh new file mode 100755 index 00000000..27516aba --- /dev/null +++ b/install/install-ndi.sh @@ -0,0 +1,291 @@ +#!/bin/sh +# install-ndi.sh -- install NDI-python for end users on Linux and macOS. +# +# curl -LsSf https://raw.githubusercontent.com/Waltham-Data-Science/NDI-python/main/install/install-ndi.sh | sh +# +# or double-click "Install NDI.command" (macOS), or run this file. Run it again +# to update. Developers want `pip install -e .` instead; this is for people +# who just want the `ndi`, `napariViewGEF` and `napariViewLightsheet` commands. +# +# Options: +# --yes do not ask before installing (MATLAB always passes this) +# --no-path do not add ~/.ndi/bin to the PATH in your shell start-up files +# --uninstall remove what this script installed, and nothing else +# +# WHAT IT TOUCHES. Everything lives under ~/.ndi, which NDI-matlab shares, +# and this script owns ONLY three folders there: +# ~/.ndi/bin uv itself, plus the ndi / napariViewGEF / napariViewLightsheet commands +# ~/.ndi/python the Python interpreter uv downloads (UV_PYTHON_INSTALL_DIR) +# ~/.ndi/venvs the environment NDI is installed into (UV_TOOL_DIR) +# Everything else in ~/.ndi (NDI_Preferences.json, tools/, caches, ...) is +# never touched, by install or by --uninstall. Outside ~/.ndi it edits only +# your shell start-up files, inside a marked block (see add_to_path), unless +# --no-path; and uv keeps its download cache in its usual place +# (~/.cache/uv), which it shares with any other uv you use. No sudo, ever. +# +# FOR TESTING (CI uses these; end users never need them): +# NDI_INSTALL_SOURCE what to install instead of the main branch: +# a local checkout directory, or a URL +# NDI_PYRAVIEW_FIND_LINKS URL of the Pyraview wheel index (install/pyraview-wheels.html) +# NDI_PYRAVIEW_CONSTRAINTS URL or path of install/pyraview-constraints.txt +# NDI_INSTALL_PYTHON the uv Python request instead of 3.12, e.g. +# cpython-3.12-macos-x86_64 to get an Intel Mac's +# install on Apple Silicon (Rosetta 2) +# +# This is the contract NDI-matlab codes against: the folders above, the flags, +# exit status 0 only when the post-install check passed, and, on success, a +# last line of output reading `NDI installed: `. +# Keep install/install-ndi.ps1 in step with it. + +set -eu + +# ---- pinned versions: the one place to change them --------------------------- +UV_VERSION="0.11.32" # https://github.com/astral-sh/uv/releases +PYTHON_VERSION="3.12" +# ------------------------------------------------------------------------------ + +REPO_RAW="https://raw.githubusercontent.com/Waltham-Data-Science/NDI-python/main" +DEFAULT_SOURCE="https://github.com/Waltham-Data-Science/NDI-python/archive/refs/heads/main.zip" +# gui: the PySide6 apps; napari: the two viewers; pyraview: signal pyramids. +EXTRAS="gui,napari,pyraview" + +NDI_HOME="$HOME/.ndi" +BIN="$NDI_HOME/bin" +PYDIR="$NDI_HOME/python" +VENVS="$NDI_HOME/venvs" + +PATH_BEGIN="# >>> NDI: added by install-ndi.sh; remove with install-ndi.sh --uninstall >>>" +PATH_END="# <<< NDI <<<" + +say() { printf '%s\n' "$*"; } +fail() { + # One line, on stderr, saying why. The caller sees nonzero. + printf 'NDI install failed: %s\n' "$*" >&2 + exit 1 +} + +usage() { + say "Usage: install-ndi.sh [--yes] [--no-path] [--uninstall]" + say " --yes do not ask before installing" + say " --no-path do not add ~/.ndi/bin to your PATH" + say " --uninstall remove ~/.ndi/bin, ~/.ndi/python, ~/.ndi/venvs and the PATH lines" +} + +YES=0 +NO_PATH=0 +UNINSTALL=0 +for arg in "$@"; do + case "$arg" in + --yes | -y) YES=1 ;; + --no-path) NO_PATH=1 ;; + --uninstall) UNINSTALL=1 ;; + --help | -h) usage; exit 0 ;; + *) usage >&2; fail "unknown option: $arg" ;; + esac +done + +[ -n "${HOME:-}" ] || fail "HOME is not set" + +# MATLAB runs this with its own libraries first on LD_LIBRARY_PATH (Linux), +# which can break the system curl (NDI-matlab's systemCurlEnvPrefix exists for +# exactly that) and the Python that builds and checks NDI below. Nothing this +# script runs needs any of these. The same list, and the reasons, are in +# src/_ndi_launch.py, which does this for the viewers themselves. +unset LD_LIBRARY_PATH LD_PRELOAD DYLD_LIBRARY_PATH DYLD_FALLBACK_LIBRARY_PATH \ + DYLD_FRAMEWORK_PATH DYLD_INSERT_LIBRARIES PYTHONHOME PYTHONPATH + +# ---- shell start-up files ---------------------------------------------------- + +# The files a new terminal reads, for the shells we know. zsh is the macOS +# default and bash the usual Linux one; a file that already exists is used +# whatever the login shell, so switching shells later still finds ~/.ndi/bin. +profile_files() { + login_shell=$(basename "${SHELL:-sh}") + for f in .zshrc .zprofile; do + if [ "$login_shell" = zsh ] || [ -f "$HOME/$f" ]; then say "$HOME/$f"; fi + done + for f in .bashrc .profile; do + if [ "$login_shell" = bash ] || [ -f "$HOME/$f" ]; then say "$HOME/$f"; fi + done + # A login bash reads .bash_profile INSTEAD of .profile when it exists + # (macOS Terminal starts login shells), so it needs the block too. + if [ -f "$HOME/.bash_profile" ]; then say "$HOME/.bash_profile"; fi + # Any other shell: .profile at least. + case "$login_shell" in zsh | bash) ;; *) say "$HOME/.profile" ;; esac +} + +# Delete our marked block from one file, in place (keeping the file itself, +# so its permissions, owner and any symlink survive). +remove_block() { + file=$1 + [ -f "$file" ] || return 0 + grep -qF "$PATH_BEGIN" "$file" || return 0 + tmp=$(mktemp) + awk -v b="$PATH_BEGIN" -v e="$PATH_END" ' + $0 == b { skip = 1; next } + skip && $0 == e { skip = 0; next } + !skip { print } + ' "$file" > "$tmp" + cat "$tmp" > "$file" + rm -f "$tmp" +} + +# Idempotent: any earlier block is replaced, never duplicated. APPENDED to the +# PATH, not prepended, so the uv in ~/.ndi/bin never shadows a uv of your own. +add_to_path() { + profile_files | sort -u | while IFS= read -r file; do + remove_block "$file" + # A last line with no newline would run into our first one. + if [ -s "$file" ] && [ -n "$(tail -c 1 "$file")" ]; then say >> "$file"; fi + { + say "$PATH_BEGIN" + # shellcheck disable=SC2016 # expanded when the shell starts, not now + say 'case ":$PATH:" in *":$HOME/.ndi/bin:"*) ;; *) export PATH="$PATH:$HOME/.ndi/bin" ;; esac' + say "$PATH_END" + } >> "$file" + say " added ~/.ndi/bin to the PATH in $file" + done +} + +remove_from_path() { + for file in "$HOME/.zshrc" "$HOME/.zprofile" "$HOME/.bashrc" "$HOME/.profile" "$HOME/.bash_profile"; do + if [ -f "$file" ] && grep -qF "$PATH_BEGIN" "$file"; then + remove_block "$file" + say " removed the NDI PATH lines from $file" + # Nothing left: it was a file we created. Remove it again. + if [ ! -s "$file" ]; then rm -f "$file"; fi + fi + done +} + +# ---- uninstall --------------------------------------------------------------- + +if [ "$UNINSTALL" = 1 ]; then + say "Removing NDI from $NDI_HOME (bin, python, venvs) ..." + for d in "$BIN" "$PYDIR" "$VENVS"; do + # uv marks some of its files read-only; make them removable first. + if [ -d "$d" ]; then chmod -R u+w "$d" 2>/dev/null || true; fi + rm -rf "$d" || fail "could not remove $d" + done + remove_from_path + say "Your NDI settings and anything else in $NDI_HOME were left in place." + say "NDI uninstalled: $BIN" + exit 0 +fi + +# ---- install ----------------------------------------------------------------- + +case "$(uname -s)" in + Linux | Darwin) ;; + *) fail "this installer is for Linux and macOS; on Windows use install-ndi.ps1" ;; +esac + +download() { # url dest + if command -v curl >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -LsSf "$1" -o "$2" + elif command -v wget >/dev/null 2>&1; then + wget -q --https-only "$1" -O "$2" + else + return 1 + fi +} + +say "NDI installer" +say " installs into: $NDI_HOME (bin, python, venvs)" +say " needs about 3 GB of disk and a few minutes on a good connection" +if [ "$YES" = 0 ] && (: < /dev/tty) 2>/dev/null; then + printf 'Continue? [Y/n] ' + read -r answer < /dev/tty || answer="" + case "$answer" in + "" | [Yy] | [Yy][Ee][Ss]) ;; + *) fail "cancelled" ;; + esac +fi + +mkdir -p "$BIN" "$PYDIR" "$VENVS" || fail "could not create folders in $NDI_HOME" + +# 1. uv, pinned. Astral's versioned installer, told to put uv in ~/.ndi/bin. +# UV_UNMANAGED_INSTALL also stops it writing an update receipt in +# ~/.config/uv, which would otherwise redirect a `uv self update` of a uv you +# already have to this copy, and stops it editing your PATH itself. +uv_version() { "$BIN/uv" --version 2>/dev/null | awk '{print $2}'; } +if [ -x "$BIN/uv" ] && [ "$(uv_version)" = "$UV_VERSION" ]; then + say "uv $UV_VERSION is already in $BIN" +else + say "Downloading uv $UV_VERSION ..." + tmp=$(mktemp) + download "https://astral.sh/uv/$UV_VERSION/install.sh" "$tmp" \ + || { rm -f "$tmp"; fail "could not download the uv installer from astral.sh (need curl or wget, and a connection)"; } + if ! UV_INSTALL_DIR="$BIN" UV_UNMANAGED_INSTALL="$BIN" UV_NO_MODIFY_PATH=1 \ + sh "$tmp" --quiet; then + rm -f "$tmp" + fail "the uv installer did not succeed" + fi + rm -f "$tmp" + [ "$(uv_version)" = "$UV_VERSION" ] || fail "uv $UV_VERSION did not end up in $BIN" +fi + +# 2. NDI, as a uv "tool": its own environment, its own managed Python, and +# its commands linked into ~/.ndi/bin. Each setting keeps uv inside the three +# folders above and away from anything else on this machine: +export UV_PYTHON_INSTALL_DIR="$PYDIR" # the interpreter +export UV_TOOL_DIR="$VENVS" # the environment +export UV_TOOL_BIN_DIR="$BIN" # the commands +export UV_PYTHON_PREFERENCE=only-managed # never a system/Homebrew/conda Python that may vanish +export UV_NO_CONFIG=1 # ignore any uv.toml of yours +export UV_NO_MODIFY_PATH=1 + +SOURCE="${NDI_INSTALL_SOURCE:-$DEFAULT_SOURCE}" +if [ -d "$SOURCE" ]; then + # A local checkout (CI): a file URL to its absolute path. + SOURCE="file://$(cd "$SOURCE" && pwd -P | sed 's/ /%20/g')" +elif [ -f "$SOURCE" ]; then + # A local wheel or source archive. + SOURCE="file://$(cd "$(dirname "$SOURCE")" && pwd -P | sed 's/ /%20/g')/$(basename "$SOURCE" | sed 's/ /%20/g')" +fi +FIND_LINKS="${NDI_PYRAVIEW_FIND_LINKS:-$REPO_RAW/install/pyraview-wheels.html}" +CONSTRAINTS="${NDI_PYRAVIEW_CONSTRAINTS:-$REPO_RAW/install/pyraview-constraints.txt}" + +say "Installing NDI from $SOURCE ..." +# --reinstall (which implies --refresh) so that running this again really +# updates: NDI and its GitHub-archive dependencies are re-fetched. +# Pyraview is not on PyPI: its release wheels are found through FIND_LINKS, +# CONSTRAINTS pins the version, and --no-build-package makes a missing wheel +# an error rather than a C++ compile. See pyproject.toml's pyraview extra. +# PATH includes ~/.ndi/bin for this one command only, so uv does not warn +# that the folder is not on the PATH; step 4 deals with that. +PATH="$BIN:$PATH" "$BIN/uv" tool install \ + --python "${NDI_INSTALL_PYTHON:-$PYTHON_VERSION}" \ + --reinstall \ + --find-links "$FIND_LINKS" \ + --constraints "$CONSTRAINTS" \ + --no-build-package pyraview \ + "ndi[$EXTRAS] @ $SOURCE" \ + || fail "uv could not install NDI (see the messages above)" + +# 3. Check the result: the commands MATLAB will call, and `ndi check`. +say "Checking the installation ..." +for cmd in ndi napariViewGEF napariViewLightsheet; do + [ -x "$BIN/$cmd" ] || fail "$BIN/$cmd is missing" +done +for cmd in napariViewGEF napariViewLightsheet; do + "$BIN/$cmd" --help > /dev/null || fail "$cmd --help did not run" +done +log=$(mktemp) +if ! "$BIN/ndi" check > "$log" 2>&1; then + cat "$log" + rm -f "$log" + fail "ndi check reported problems (see above)" +fi +rm -f "$log" +say " ndi check: all checks passed" + +# 4. PATH. +if [ "$NO_PATH" = 0 ]; then + add_to_path + say "Open a new terminal window to use the ndi, napariViewGEF and napariViewLightsheet commands." +else + say "PATH left alone (--no-path); the commands are in $BIN" +fi + +say "NDI installed: $BIN" diff --git a/pyproject.toml b/pyproject.toml index e3e3ea6a..a807de8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -171,13 +171,13 @@ all = [ [project.scripts] ndi = "ndi.__main__:main" -# pip generates this with the venv's absolute interpreter in its shebang, so -# a symlink from /usr/local/bin works with no activation and no PYTHONPATH. -napariViewGEF = "ndi.gui.app.genepyramid.cli:main" -# Sibling of napariViewGEF for lightsheet OME-Zarr pyramids. Same shell-wrapper -# story: the MATLAB LightsheetZarrManager execs /usr/local/bin/napariViewLightsheet, -# which scrubs MATLAB's library paths and then runs this console script. -napariViewLightsheet = "ndi.gui.app.lightsheetZarr.cli:main" +# The two napari viewers, which MATLAB starts (with NDI_FROM_MATLAB=1) from +# ~/.ndi/bin, where install/install-ndi.sh puts them. They go through the +# stdlib-only _ndi_launch module rather than straight to ndi.gui.app.*.cli:main +# so MATLAB's library paths are dropped -- by re-exec -- BEFORE `import ndi` +# loads numpy and friends. That replaced the shell wrappers that used to do it. +napariViewGEF = "_ndi_launch:napariViewGEF" +napariViewLightsheet = "_ndi_launch:napariViewLightsheet" [project.urls] Homepage = "https://github.com/Waltham-Data-Science/NDI-python" @@ -189,7 +189,10 @@ Issues = "https://github.com/Waltham-Data-Science/NDI-python/issues" allow-direct-references = true [tool.hatch.build.targets.wheel] -packages = ["src/ndi"] +# The package, plus the console scripts' launcher: one top-level module beside +# it, so running a viewer does not import `ndi` first (see src/_ndi_launch.py). +only-include = ["src/ndi", "src/_ndi_launch.py"] +sources = ["src"] [tool.hatch.build.targets.sdist] include = ["/src", "/tests", "/examples", "LICENSE", "README.md", "CHANGELOG.md", "MATLAB_MAPPING.md"] diff --git a/scripts/napariViewLightsheet b/scripts/napariViewLightsheet deleted file mode 100644 index ecf6a52c..00000000 --- a/scripts/napariViewLightsheet +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# napariViewLightsheet - shell wrapper around the ndi.gui.app.lightsheetZarr.cli -# console script. -# -# The MATLAB ndi.gui.app.LightsheetZarrManager shells out to this file at -# /usr/local/bin/napariViewLightsheet. It exists so MATLAB's library -# paths (LD_LIBRARY_PATH, DYLD_LIBRARY_PATH, DYLD_FALLBACK_LIBRARY_PATH) -# do not leak into the Python process it starts -- MATLAB's copies of -# libtiff, libcurl, etc. crash or subtly misbehave when loaded into a -# non-MATLAB Python. -# -# Install (once, per box): -# sudo install -m 0755 scripts/napariViewLightsheet \ -# /usr/local/bin/napariViewLightsheet -# -# Point NAPARI_VIEW_LIGHTSHEET_PYTHON at the interpreter (or venv python) -# that has `ndi` and `ndi[napari]` installed; defaults to `python3` on -# PATH. - -set -euo pipefail - -unset LD_LIBRARY_PATH DYLD_LIBRARY_PATH DYLD_FALLBACK_LIBRARY_PATH -unset LD_PRELOAD DYLD_INSERT_LIBRARIES - -PYTHON_BIN="${NAPARI_VIEW_LIGHTSHEET_PYTHON:-python3}" -exec "$PYTHON_BIN" -m ndi.gui.app.lightsheetZarr.cli "$@" diff --git a/src/_ndi_launch.py b/src/_ndi_launch.py new file mode 100644 index 00000000..5aa0367e --- /dev/null +++ b/src/_ndi_launch.py @@ -0,0 +1,335 @@ +"""Start NDI's command-line viewers safely when MATLAB is the parent process. + +The ``napariViewGEF`` and ``napariViewLightsheet`` console scripts point HERE +rather than at their ``cli.main`` functions, and that is the whole reason this +module exists outside the ``ndi`` package. Importing anything under ``ndi`` +runs ``ndi/__init__.py``, which loads numpy, pydantic_core, DID and roughly a +thousand other modules -- so by the time a ``main()`` inside ``ndi`` runs, the +dynamic loader has already resolved their shared libraries against whatever +LD_LIBRARY_PATH the process inherited. Cleaning the environment there would be +too late for those libraries. This module imports only the standard library, +so the console script can clean up and re-exec before any of that happens. +(The two ``main()`` functions call :func:`escapeMatlabEnvironment` too, which +covers ``python -m ndi napari`` and is a no-op after the re-exec.) + +WHY. MATLAB exports library search paths of its own, and a process it starts +with ``system()`` inherits them. MEASURED, not assumed (these notes came from +the shell wrappers this module replaced, tools/napariViewGEF and +scripts/napariViewLightsheet): + + Linux -- MATLAB prepends nine of its own directories to LD_LIBRARY_PATH, + among them sys/os/glnxa64 (its libstdc++) and + sys/opengl/lib/glnxa64 (OpenGL, which napari needs through vispy). + numpy and PyQt5 imported fine anyway under R2026a on Ubuntu 24.04, + so this is insurance rather than a fix for a known break, except + for OpenGL, which can only be tested on a machine with a GPU. + macOS -- MATLAB sets no MW_* except MW_DESKTOP_IGNORE_STDIN and leaves LD_* + and DYLD_* empty. Nothing to protect against today; the DYLD_* + variables are cleared anyway, at no cost. + Windows -- UNMEASURED. See :func:`cleanedEnvironment`. + +MW_ORIG_LD_LIBRARY_PATH was unset on both measured platforms, so it can +neither detect MATLAB nor restore the caller's original value. Detection is +therefore the NDI_FROM_MATLAB variable the MATLAB-side launcher sets (one call +site, no dependence on MATLAB internals), plus a fallback for callers that +forget it: a library path naming one of MATLAB's architecture directories. + +WHY RE-EXEC. On Linux and macOS the dynamic loader reads LD_LIBRARY_PATH and +DYLD_* once, when the process starts; changing ``os.environ`` afterwards does +not change where later ``dlopen`` calls look. So the cleaned environment has to +be handed to a NEW process image: ``os.execve`` of the same interpreter with +the same command line. A marker variable stops the new image from doing it +again. + +Python-only; no MATLAB counterpart. The MATLAB side solves the mirror-image +problem for curl with ``ndi.common.systemCurlEnvPrefix`` (recorded +``matlab_only`` in src/ndi/common/ndi_matlab_python_bridge.yaml). Standard +library only, deliberately -- which is also why it does not use the +``@pydantic.validate_call`` that NDI's public API carries: importing pydantic +loads a compiled extension, which is exactly what must not happen first. +""" + +from __future__ import annotations + +import json +import os +import sys +from collections.abc import Callable, Mapping, Sequence + +__all__ = [ + "LIBRARY_VARS", + "MARKER_VAR", + "cleanedEnvironment", + "escapeMatlabEnvironment", + "isFromMatlab", + "napariViewGEF", + "napariViewLightsheet", +] + +# Removed from the environment of a viewer started from MATLAB, POSIX only. +# Each one is something a Python GUI process must not inherit from MATLAB: +LIBRARY_VARS = ( + # Linux: MATLAB prepends its own lib dirs (libstdc++, OpenGL, ...) and the + # loader searches them BEFORE a wheel's own RUNPATH, so numpy, Qt or + # napari's OpenGL could bind MATLAB's copies. Measured; see above. + "LD_LIBRARY_PATH", + # Linux: forces a library into every process. Nothing in a viewer should + # inherit one from MATLAB. (This also drops a VirtualGL `vglrun` preload, + # should MATLAB itself have been started that way; run the viewer under + # vglrun directly in that case.) + "LD_PRELOAD", + # macOS: the dyld counterparts of the two above, with the same failure + # modes -- a library or framework from MATLAB's tree bound in place of the + # one a wheel ships, or injected outright. Measured empty under MATLAB + # today; cleared so that staying true does not depend on MATLAB. + "DYLD_LIBRARY_PATH", + "DYLD_FALLBACK_LIBRARY_PATH", + "DYLD_FRAMEWORK_PATH", + "DYLD_INSERT_LIBRARIES", + # Both platforms: PYTHONHOME replaces the interpreter's prefix outright, so + # the venv's standard library and site-packages would not be the ones + # used. PYTHONPATH puts directories AHEAD of the venv's site-packages, so + # a numpy or Qt built for another Python could shadow the installed one. + # The viewers are installed as self-contained tools and need neither. + "PYTHONHOME", + "PYTHONPATH", +) + +# Set in the cleaned environment. Its presence means "already done": the +# re-exec'd process must not re-exec again, whatever else it finds. Its value +# says how it was done: "reexec", or "in-place" (Windows, or no faithful +# restart possible). +MARKER_VAR = "NDI_MATLAB_ENV_CLEANED" + +# Set by MATLAB's launcher (any non-empty value). +FROM_MATLAB_VAR = "NDI_FROM_MATLAB" + +# Hidden: when non-empty, the final process prints one JSON line describing +# its environment to stderr. Used by CI to prove the re-exec happened. +DEBUG_VAR = "NDI_LAUNCH_DEBUG" + +# MathWorks' own architecture names. No other software names a directory +# this way, so a library path holding one is MATLAB's. +_MATLAB_ARCH_DIRS = frozenset({"glnxa64", "maci64", "maca64"}) + +# The variables whose contents can betray MATLAB without NDI_FROM_MATLAB. +_DETECTION_VARS = ( + "LD_LIBRARY_PATH", + "LD_PRELOAD", + "DYLD_LIBRARY_PATH", + "DYLD_FALLBACK_LIBRARY_PATH", + "DYLD_FRAMEWORK_PATH", + "DYLD_INSERT_LIBRARIES", +) + +# MATLAB's binary directories on Windows, relative to matlabroot. +_WINDOWS_MATLAB_TAILS = ( + ("bin", "win64"), + ("runtime", "win64"), + ("sys", "os", "win64"), + ("extern", "bin", "win64"), +) + + +def _parts(path: str) -> list[str]: + """Split a path on both separators, dropping empties; lower-cased.""" + return [p for p in path.replace("\\", "/").lower().split("/") if p] + + +def _isMatlabLibraryEntry(entry: str) -> bool: + return any(p in _MATLAB_ARCH_DIRS for p in _parts(entry)) + + +def _isMatlabWindowsPathEntry(entry: str) -> bool: + parts = _parts(entry) + return any(len(parts) > len(t) and tuple(parts[-len(t) :]) == t for t in _WINDOWS_MATLAB_TAILS) + + +def isFromMatlab(environ: Mapping[str, str]) -> bool: + """True when ``environ`` looks like that of a process MATLAB started. + + Either MATLAB's launcher said so (NDI_FROM_MATLAB is non-empty), or a + dynamic-loader variable names one of MATLAB's architecture directories + (``glnxa64``, ``maci64``, ``maca64``), which nothing but MATLAB uses. + """ + if environ.get(FROM_MATLAB_VAR, ""): + return True + return any( + _isMatlabLibraryEntry(entry) + for var in _DETECTION_VARS + for entry in environ.get(var, "").split(":") + if entry + ) + + +def cleanedEnvironment(environ: Mapping[str, str], platform: str) -> dict[str, str] | None: + """Return the environment a viewer started from MATLAB should run in. + + ``None`` means "leave it alone": the environment was already cleaned (the + marker is set), it does not come from MATLAB, or there is nothing in it to + remove. Otherwise a new dict: the input minus :data:`LIBRARY_VARS` (minus + MATLAB's directories on PATH, on Windows), plus the marker. + + WINDOWS IS UNMEASURED. Nobody has recorded what MATLAB exports to a child + there, so this does the minimum that is certainly right: drop MATLAB's + own binary directories (``\\bin\\win64`` and kin) from PATH, + and PYTHONHOME / PYTHONPATH. Since Python 3.8 extension modules load + their DLLs without consulting PATH, so this matters mainly for programs + the viewer starts. On Windows only NDI_FROM_MATLAB triggers it. + """ + if environ.get(MARKER_VAR, ""): + return None + windows = platform.startswith("win") + if windows: + if not environ.get(FROM_MATLAB_VAR, ""): + return None + elif not isFromMatlab(environ): + return None + + env = dict(environ) + changed = False + removable = ("PYTHONHOME", "PYTHONPATH") if windows else LIBRARY_VARS + for var in removable: + if var in env: + del env[var] + changed = True + if windows: + # Windows variable names are case-insensitive; find PATH however spelled. + for key in [k for k in env if k.upper() == "PATH"]: + entries = env[key].split(";") + kept = [e for e in entries if not _isMatlabWindowsPathEntry(e)] + if len(kept) != len(entries): + env[key] = ";".join(kept) + changed = True + if not changed: + return None + env[MARKER_VAR] = "in-place" if windows else "reexec" + return env + + +def _isStandaloneInterpreter() -> bool: + """True when this process is a Python interpreter running a program. + + False when Python is embedded in a host -- MATLAB's in-process ``pyenv``, + say -- where exec would replace the HOST, whatever ``sys.orig_argv`` + holds. Two signs: an embedded interpreter has no ``__main__`` file, and + (Linux) the running binary is not a Python. + """ + main = sys.modules.get("__main__") + if not getattr(main, "__file__", None): + return False + try: + exe = os.path.basename(os.path.realpath("/proc/self/exe")) + except OSError: + return True + return not os.path.exists("/proc/self/exe") or exe.lower().startswith("python") + + +_reported = False + + +def _debugReport(environ: Mapping[str, str]) -> None: + # Once per process: the console script and main() both call in. + global _reported + if _reported or not environ.get(DEBUG_VAR, ""): + return + _reported = True + report = { + "cleaned": environ.get(MARKER_VAR) or None, + "fromMatlab": bool(environ.get(FROM_MATLAB_VAR, "")), + "env": {var: environ.get(var) for var in LIBRARY_VARS}, + } + print("ndi-launch: " + json.dumps(report, sort_keys=True), file=sys.stderr, flush=True) + + +def escapeMatlabEnvironment( + *, + environ: Mapping[str, str] | None = None, + platform: str | None = None, + executable: str | None = None, + orig_argv: Sequence[str] | None = None, + execve: Callable[..., object] | None = None, + standalone: bool | None = None, +) -> bool: + """Leave MATLAB's library paths behind before NDI's viewers load anything. + + Call it FIRST in a viewer's entry point. When :func:`cleanedEnvironment` + finds something to remove, on Linux and macOS this replaces the current + process with a fresh copy of itself -- same interpreter, same command + line (``sys.orig_argv``), cleaned environment -- and does not return. On + Windows it cleans ``os.environ`` in place and returns True. Otherwise it + returns False and changes nothing. + + It never re-execs a process it cannot faithfully restart: one with no + interpreter path or no recorded command line, or one where Python is + embedded in another program (MATLAB's in-process ``pyenv`` among them), + where an exec would replace the HOST. Those fall back to cleaning + ``os.environ`` in place, which protects child processes but not this one. + + The keyword arguments exist for tests; each defaults to the real thing + (``os.environ``, ``sys.platform``, ``sys.executable``, ``sys.orig_argv``, + ``os.execve``, :func:`_isStandaloneInterpreter`). + """ + live = environ is None + if live: + environ = os.environ + platform = sys.platform if platform is None else platform + executable = sys.executable if executable is None else executable + if orig_argv is None: + orig_argv = getattr(sys, "orig_argv", []) + execve = os.execve if execve is None else execve + + env = cleanedEnvironment(environ, platform) + if env is None: + _debugReport(environ) + return False + + restartable = ( + not platform.startswith("win") + and bool(executable) + and len(orig_argv) >= 2 + and not getattr(sys, "frozen", False) + and (_isStandaloneInterpreter() if standalone is None else standalone) + ) + if restartable: + argv = [executable, *orig_argv[1:]] + try: + sys.stdout.flush() + sys.stderr.flush() + execve(executable, argv, env) + # A real execve does not return; an injected one does. + return True + except OSError as err: + print( + f"ndi: could not restart without MATLAB's library paths ({err}); " + "continuing with them removed for child processes only.", + file=sys.stderr, + ) + + # Windows, or no faithful restart possible: clean in place. + env[MARKER_VAR] = "in-place" + if live: + for var in [k for k in environ if k not in env]: + del environ[var] + environ.update(env) + _debugReport(env) + return True + + +# --- console-script entry points (pyproject.toml [project.scripts]) -------- + + +def napariViewGEF() -> int: + """Console script ``napariViewGEF``: escape, then run the real CLI.""" + escapeMatlabEnvironment() + from ndi.gui.app.genepyramid.cli import main + + return main() + + +def napariViewLightsheet() -> int: + """Console script ``napariViewLightsheet``: escape, then run the real CLI.""" + escapeMatlabEnvironment() + from ndi.gui.app.lightsheetZarr.cli import main + + return main() diff --git a/src/ndi/fun/doc_lightsheet/ndi_matlab_python_bridge.yaml b/src/ndi/fun/doc_lightsheet/ndi_matlab_python_bridge.yaml index 6fae12ca..4ec7bfb7 100644 --- a/src/ndi/fun/doc_lightsheet/ndi_matlab_python_bridge.yaml +++ b/src/ndi/fun/doc_lightsheet/ndi_matlab_python_bridge.yaml @@ -213,6 +213,13 @@ functions: doc.gene package, but the shape is the same); the lightsheet package puts it under +doc/+lightsheet because the manager stays thin. + 2026-10-16: the shell wrapper the MATLAB side names here, + scripts/napariViewLightsheet installed at + /usr/local/bin/napariViewLightsheet, is gone from NDI-python. The + napariViewLightsheet console script now drops MATLAB's library paths + itself (src/_ndi_launch.py), and install/install-ndi.sh puts it in + ~/.ndi/bin; the MATLAB-side default moves in NDI-matlab. Nothing + here to port; the hash is unchanged. - name: view matlab_path: "+ndi/+fun/+doc/+lightsheet/view.m" matlab_last_sync_hash: "2f9b0eaf" @@ -234,6 +241,13 @@ functions: view gains 'debug' and 'logFile' options; --list mode is unchanged. Nothing on the Python side to follow: the Python side is the launched process, and it already honours NDI_LIGHTSHEET_DEBUG. + 2026-10-16: the shell wrapper the MATLAB side names here, + scripts/napariViewLightsheet installed at + /usr/local/bin/napariViewLightsheet, is gone from NDI-python. The + napariViewLightsheet console script now drops MATLAB's library paths + itself (src/_ndi_launch.py), and install/install-ndi.sh puts it in + ~/.ndi/bin; the MATLAB-side default moves in NDI-matlab. Nothing + here to port; the hash is unchanged. - name: launchViewer matlab_path: "+ndi/+fun/+doc/+lightsheet/launchViewer.m" diff --git a/src/ndi/gui/app/genepyramid/cli.py b/src/ndi/gui/app/genepyramid/cli.py index 0f395c49..320d5a1a 100644 --- a/src/ndi/gui/app/genepyramid/cli.py +++ b/src/ndi/gui/app/genepyramid/cli.py @@ -1,11 +1,14 @@ """Command line for viewing a spatial gene expression pyramid. -Reachable three ways, all the same function: +Reachable two ways, both the same function: napariViewGEF ... the installed console script python -m ndi napari ... the ndi subcommand - /usr/local/bin/napariViewGEF ... the shell wrapper, which scrubs - MATLAB's environment first + +Both are safe to start from MATLAB (which sets NDI_FROM_MATLAB=1): the +console script goes through the stdlib-only ``_ndi_launch`` module, which +drops MATLAB's library paths and re-execs before ``ndi`` is even imported, +and main() calls the same escape first for the ``python -m`` route. --report and --list do their work WITHOUT a display. That is deliberate: it gives the entry point a path that CI can exercise, so the argument @@ -355,6 +358,13 @@ def _resolve_gene_layers(spec: str): def main(argv=None) -> int: + if argv is None: + # The real command line, so possibly started by MATLAB: leave its + # library paths behind before Qt and napari load. A no-op when the + # console script already did it (see src/_ndi_launch.py). + from _ndi_launch import escapeMatlabEnvironment + + escapeMatlabEnvironment() args = build_parser().parse_args(argv) from .progress import stage diff --git a/src/ndi/gui/app/genepyramid/ndi_matlab_python_bridge.yaml b/src/ndi/gui/app/genepyramid/ndi_matlab_python_bridge.yaml index 034995b0..94cef23a 100644 --- a/src/ndi/gui/app/genepyramid/ndi_matlab_python_bridge.yaml +++ b/src/ndi/gui/app/genepyramid/ndi_matlab_python_bridge.yaml @@ -112,11 +112,15 @@ functions: - name: status type_python: "int" decision_log: > - Python-only. The napariViewGEF command line, reachable three ways - that are all this function: the installed console script, "python -m - ndi napari", and the shell wrapper that scrubs MATLAB's environment - before exec. Takes argv so a test can drive it; returns a status - rather than calling sys.exit, for the same reason. + Python-only. The napariViewGEF command line, reachable two ways + that are both this function: the installed console script (through + the stdlib-only _ndi_launch module, which drops MATLAB's library + paths and re-execs before `ndi` is imported -- it replaced the shell + wrapper tools/napariViewGEF), and "python -m ndi napari". When argv + is None it calls _ndi_launch.escapeMatlabEnvironment first, which + covers the second route and is a no-op after the first. Takes argv + so a test can drive it; returns a status rather than calling + sys.exit, for the same reason. --report and --list do their work without a display deliberately, so the argument handling, pyramid lookup and gene resolution have a path CI can exercise; only the viewing itself needs napari. diff --git a/src/ndi/gui/app/lightsheetZarr/README-lightsheet-zarr.md b/src/ndi/gui/app/lightsheetZarr/README-lightsheet-zarr.md index 6f9e4519..74345749 100644 --- a/src/ndi/gui/app/lightsheetZarr/README-lightsheet-zarr.md +++ b/src/ndi/gui/app/lightsheetZarr/README-lightsheet-zarr.md @@ -2,8 +2,9 @@ Sibling of `ndi.gui.app.genepyramid` for lightsheet OME-Zarr pyramids. Provides the `napariViewLightsheet` console script that the MATLAB -`ndi.gui.app.LightsheetZarrManager` shells out to (via -`/usr/local/bin/napariViewLightsheet`) and the pure-Python +`ndi.gui.app.LightsheetZarrManager` shells out to (installed in +`~/.ndi/bin` by [`install/install-ndi.sh`](../../../../../install/README.md)) +and the pure-Python multiscale/viewer surface underneath. ## Layout @@ -45,20 +46,26 @@ Scaffold, with the specific hook named at the raise site: Add to `[project.scripts]`: ```toml -napariViewLightsheet = "ndi.gui.app.lightsheetZarr.cli:main" +napariViewLightsheet = "_ndi_launch:napariViewLightsheet" ``` +(`_ndi_launch.napariViewLightsheet` runs `ndi.gui.app.lightsheetZarr.cli:main` +after leaving MATLAB's library paths behind; see below.) + Existing `[project.optional-dependencies].napari` (napari + dask) already covers this app. `require_napari()` in `viewer.py` names the extra when it is missing. -## Install the shell wrapper +## Starting it from MATLAB -Same pattern as `napariViewGEF`: install `scripts/napariViewLightsheet` -at `/usr/local/bin/napariViewLightsheet`. The wrapper scrubs MATLAB's -`LD_LIBRARY_PATH` / `DYLD_*` before exec-ing the console script, so a -Python started from MATLAB does not pick up MATLAB's copies of libtiff / -libcurl / etc. +No shell wrapper is needed any more (`scripts/napariViewLightsheet` is +gone). The console script itself goes through `src/_ndi_launch.py`: when +`NDI_FROM_MATLAB` is set, or the loader paths name MATLAB's own +directories, it drops `LD_LIBRARY_PATH` / `LD_PRELOAD` / `DYLD_*` / +`PYTHONHOME` / `PYTHONPATH` and re-execs itself before `ndi` (and numpy, +Qt, napari) is imported, so a Python started from MATLAB does not pick up +MATLAB's copies of libstdc++ / OpenGL / etc. End users get the script in +`~/.ndi/bin` from the installer in `install/`. ## Design notes diff --git a/src/ndi/gui/app/lightsheetZarr/cli.py b/src/ndi/gui/app/lightsheetZarr/cli.py index 11795667..57a43617 100644 --- a/src/ndi/gui/app/lightsheetZarr/cli.py +++ b/src/ndi/gui/app/lightsheetZarr/cli.py @@ -1,9 +1,11 @@ """napariViewLightsheet - open a lightsheetZarrPyramid in napari. Installed by ``pyproject.toml`` `[project.scripts]` as -``napariViewLightsheet``; the MATLAB ``ndi.gui.app.LightsheetZarrManager`` -shell-execs it through ``/usr/local/bin/napariViewLightsheet``, a small -wrapper that scrubs MATLAB's library paths before starting Python. The +``napariViewLightsheet``, which the MATLAB +``ndi.gui.app.LightsheetZarrManager`` shell-execs. The console script goes +through the stdlib-only ``_ndi_launch`` module, which drops MATLAB's +library paths and re-execs before ``ndi`` is imported; main() calls the +same escape first when run from a real command line. The argument set here MUST stay in sync with ``ndi.fun.doc.lightsheet.viewCommand`` in NDI-matlab. @@ -275,6 +277,13 @@ def _configure_logging() -> None: def main(argv: Sequence[str] | None = None) -> int: + if argv is None: + # The real command line, so possibly started by MATLAB: leave its + # library paths behind before Qt and napari load. A no-op when the + # console script already did it (see src/_ndi_launch.py). + from _ndi_launch import escapeMatlabEnvironment + + escapeMatlabEnvironment() args = build_parser().parse_args(argv) _configure_logging() diff --git a/src/ndi/gui/app/lightsheetZarr/ndi_matlab_python_bridge.yaml b/src/ndi/gui/app/lightsheetZarr/ndi_matlab_python_bridge.yaml index 0e6b3a5e..195faab1 100644 --- a/src/ndi/gui/app/lightsheetZarr/ndi_matlab_python_bridge.yaml +++ b/src/ndi/gui/app/lightsheetZarr/ndi_matlab_python_bridge.yaml @@ -23,15 +23,15 @@ project_metadata: # The MATLAB GUI class ndi.gui.app.LightsheetZarrManager is recorded here # rather than in src/ndi/gui/app/ndi_matlab_python_bridge.yaml so that # every part of the lightsheetZarr feature lives in one file. It's not -# ported (napari is Python-only), but the shell wrapper it targets is, +# ported (napari is Python-only), but the console script it targets is, # and that's the piece the CLI is documented against. # ========================================================================= functions: # ----------------------------------------------------------------------- # MATLAB GUI class — not ported (napari has no MATLAB counterpart). - # The manager shell-execs /usr/local/bin/napariViewLightsheet, which is - # a bash wrapper around this package's cli.main. + # The manager shell-execs the napariViewLightsheet console script, which + # runs this package's cli.main (through _ndi_launch; see src/_ndi_launch.py). # ----------------------------------------------------------------------- - name: LightsheetZarrManager matlab_path: "+ndi/+gui/+app/LightsheetZarrManager.m" @@ -40,9 +40,14 @@ functions: decision_log: >- matlab_only: napari is Python-only, so the manager class stays MATLAB-only. It is the caller side of the CLI in this package - - shell-execs /usr/local/bin/napariViewLightsheet (a wrapper that - scrubs MATLAB's library paths) which then runs - ndi.gui.app.lightsheetZarr.cli.main. Mirrors the GEFManager / + shell-execs the napariViewLightsheet console script, which runs + ndi.gui.app.lightsheetZarr.cli.main. (Until 2026-10 it ran a bash + wrapper, scripts/napariViewLightsheet, at + /usr/local/bin/napariViewLightsheet to scrub MATLAB's library paths; + the console script now does that itself, through the stdlib-only + _ndi_launch module, and install/install-ndi.sh puts it in + ~/.ndi/bin. The MATLAB-side launch path changes in step in + NDI-matlab.) Mirrors the GEFManager / napariViewGEF split; if the day ever comes that a Python-hosted MATLAB-app equivalent is built, this entry is where it will get declared. @@ -95,9 +100,13 @@ functions: - name: exit_code type_python: int decision_log: >- - Python-only. The napariViewLightsheet command line, reachable two - ways that are the same function: the installed console script, and - the shell wrapper that scrubs MATLAB's environment before exec. + Python-only. The napariViewLightsheet command line, reached through + the installed console script, which goes through the stdlib-only + _ndi_launch module first: that drops MATLAB's library paths and + re-execs before `ndi` is imported (it replaced the shell wrapper + scripts/napariViewLightsheet). When argv is None main calls + _ndi_launch.escapeMatlabEnvironment itself as well, a no-op after + the console script's. Handles --list and --report without importing napari or dask so those subcommands stay cheap on a headless box; napari is loaded only when a viewer is actually opened. Returns an integer exit diff --git a/src/ndi/ndi_matlab_python_bridge.yaml b/src/ndi/ndi_matlab_python_bridge.yaml index 1a01385b..b724ec50 100644 --- a/src/ndi/ndi_matlab_python_bridge.yaml +++ b/src/ndi/ndi_matlab_python_bridge.yaml @@ -1177,6 +1177,13 @@ classes: Corrected, and MATLAB's public Items property (SetAccess = private leaves GetAccess public) is now exposed read-only beside the private _items list. + 2026-10-16: GUI/GEFManager/ViewerLauncher defaults to "" (find + napariViewGEF automatically: ~/.ndi/bin, where install/install-ndi.sh + puts it, then the PATH) instead of /usr/local/bin/napariViewGEF, the + shell wrapper this repo no longer ships -- the console script now drops + MATLAB's library paths itself (src/_ndi_launch.py). NDI-matlab makes + the same change to ndi.preferences in a parallel PR; the hash moves + when that lands and is examined. properties: - name: Items diff --git a/src/ndi/preferences.py b/src/ndi/preferences.py index 5ece3191..1904c92d 100644 --- a/src/ndi/preferences.py +++ b/src/ndi/preferences.py @@ -230,14 +230,15 @@ def _register_defaults(self) -> None: "GUI", "GEFManager", "ViewerLauncher", - "/usr/local/bin/napariViewGEF", + "", "str", "Program the GEF Manager's View button runs to open a pyramid in " - "napari. Normally a small shell wrapper rather than the " - "napariViewGEF console script itself: MATLAB exports library " - "paths of its own, and a Python process started from MATLAB loads " - "MATLAB's copies of libraries it must not use, so the wrapper " - "scrubs the environment before running the real entry point.", + "napari. Empty (the default) means find it automatically: the " + "napariViewGEF console script the NDI installer puts in " + "~/.ndi/bin, else the one on the PATH. Set a full path only to " + "use a different copy. No shell wrapper is needed: started with " + "NDI_FROM_MATLAB=1, the console script itself drops MATLAB's " + "library paths before loading anything.", ) def _add_item( diff --git a/tests/test_ndi_launch.py b/tests/test_ndi_launch.py new file mode 100644 index 00000000..99d1b45c --- /dev/null +++ b/tests/test_ndi_launch.py @@ -0,0 +1,298 @@ +"""Tests for _ndi_launch: leaving MATLAB's library paths behind. + +Nothing here really execs: every test injects the environment, platform, +interpreter, command line and execve, and inspects what would have run. +""" + +from __future__ import annotations + +import io +import json +import subprocess +import sys + +import pytest + +import _ndi_launch as L + +# What MATLAB R2026a on Linux leaves in LD_LIBRARY_PATH (abridged; measured). +MATLAB_LD = ( + "/usr/local/MATLAB/R2026a/sys/opengl/lib/glnxa64:" + "/usr/local/MATLAB/R2026a/bin/glnxa64:" + "/usr/local/MATLAB/R2026a/sys/os/glnxa64" +) + + +class FakeExec: + def __init__(self, raise_with: OSError | None = None): + self.calls = [] + self.raise_with = raise_with + + def __call__(self, path, argv, env): + self.calls.append((path, list(argv), dict(env))) + if self.raise_with: + raise self.raise_with + + +@pytest.fixture(autouse=True) +def _fresh_report_flag(monkeypatch): + monkeypatch.setattr(L, "_reported", False) + + +def run( + environ, + platform="linux", + orig_argv=("/venv/bin/python", "/bin/napariViewGEF", "x"), + standalone=True, +): + fake = FakeExec() + result = L.escapeMatlabEnvironment( + environ=environ, + platform=platform, + executable="/venv/bin/python", + orig_argv=list(orig_argv), + execve=fake, + standalone=standalone, + ) + return result, fake.calls + + +# --- detection -------------------------------------------------------------- + + +class TestDetection: + def test_the_flag_is_enough(self): + assert L.isFromMatlab({"NDI_FROM_MATLAB": "1"}) + + def test_an_empty_flag_is_not(self): + assert not L.isFromMatlab({"NDI_FROM_MATLAB": ""}) + + def test_matlab_library_dirs_are_recognised_without_the_flag(self): + assert L.isFromMatlab({"LD_LIBRARY_PATH": MATLAB_LD}) + assert L.isFromMatlab({"DYLD_LIBRARY_PATH": "/Applications/MATLAB_R2026a.app/bin/maca64"}) + assert L.isFromMatlab({"DYLD_FALLBACK_LIBRARY_PATH": "/x/sys/os/maci64:/usr/lib"}) + + @pytest.mark.parametrize( + "value", + [ + "/usr/local/cuda/lib64:/opt/intel/lib", + "/home/me/glnxa64stuff/lib", # a component must BE the arch name + "", + ], + ) + def test_ordinary_library_paths_are_left_alone(self, value): + assert not L.isFromMatlab({"LD_LIBRARY_PATH": value}) + + +# --- the cleaned environment ------------------------------------------------ + + +class TestCleanedEnvironment: + def test_every_library_variable_goes_and_the_rest_stays(self): + environ = dict.fromkeys(L.LIBRARY_VARS, "/something") + environ.update(NDI_FROM_MATLAB="1", HOME="/home/me", PATH="/usr/bin", DISPLAY=":0") + env = L.cleanedEnvironment(environ, "linux") + assert env is not None + for var in L.LIBRARY_VARS: + assert var not in env + assert env["HOME"] == "/home/me" + assert env["PATH"] == "/usr/bin" + assert env["DISPLAY"] == ":0" + assert env["NDI_FROM_MATLAB"] == "1" + assert env[L.MARKER_VAR] == "reexec" + + def test_the_input_is_not_modified(self): + environ = {"NDI_FROM_MATLAB": "1", "LD_LIBRARY_PATH": MATLAB_LD} + L.cleanedEnvironment(environ, "linux") + assert environ == {"NDI_FROM_MATLAB": "1", "LD_LIBRARY_PATH": MATLAB_LD} + + def test_already_cleaned_means_leave_it(self): + environ = {"NDI_FROM_MATLAB": "1", "LD_LIBRARY_PATH": MATLAB_LD, L.MARKER_VAR: "1"} + assert L.cleanedEnvironment(environ, "linux") is None + + def test_not_from_matlab_means_leave_it(self): + assert L.cleanedEnvironment({"LD_LIBRARY_PATH": "/usr/local/cuda/lib64"}, "linux") is None + + def test_from_matlab_with_nothing_to_remove_means_leave_it(self): + # macOS today: MATLAB sets no LD/DYLD at all, so no restart is needed. + assert L.cleanedEnvironment({"NDI_FROM_MATLAB": "1", "HOME": "/h"}, "darwin") is None + + def test_detection_without_the_flag_cleans_too(self): + env = L.cleanedEnvironment({"LD_LIBRARY_PATH": MATLAB_LD}, "linux") + assert env == {L.MARKER_VAR: "reexec"} + + +class TestWindows: + PATH = ( + r"C:\Program Files\MATLAB\R2026a\bin\win64;" + r"C:\Program Files\MATLAB\R2026a\runtime\win64;" + r"C:\Windows\system32;" + r"C:\Program Files\MATLAB\R2026a\bin;" + r"C:\Users\me\.ndi\bin" + ) + + def test_matlab_binary_dirs_leave_path_and_the_rest_stays(self): + env = L.cleanedEnvironment( + {"NDI_FROM_MATLAB": "1", "Path": self.PATH, "PYTHONPATH": r"C:\x"}, "win32" + ) + assert env["Path"] == ( + r"C:\Windows\system32;C:\Program Files\MATLAB\R2026a\bin;C:\Users\me\.ndi\bin" + ) + assert "PYTHONPATH" not in env + assert env[L.MARKER_VAR] == "in-place" + + def test_only_the_flag_triggers_it(self): + # Unmeasured platform: no guessing from PATH alone. + assert L.cleanedEnvironment({"Path": self.PATH}, "win32") is None + + def test_posix_loader_variables_are_not_touched(self): + env = L.cleanedEnvironment( + {"NDI_FROM_MATLAB": "1", "PYTHONHOME": "x", "LD_LIBRARY_PATH": "y"}, "win32" + ) + assert env["LD_LIBRARY_PATH"] == "y" + + def test_windows_never_execs(self): + environ = {"NDI_FROM_MATLAB": "1", "PATH": self.PATH} + result, calls = run(environ, platform="win32") + assert result is True + assert calls == [] + + +# --- the restart ------------------------------------------------------------ + + +class TestRestart: + def test_execs_the_same_command_line_with_the_cleaned_environment(self): + environ = {"NDI_FROM_MATLAB": "1", "LD_LIBRARY_PATH": MATLAB_LD, "HOME": "/h"} + result, calls = run(environ, orig_argv=["python3", "/b/napariViewGEF", "/s", "--list"]) + assert result is True + path, argv, env = calls[0] + assert path == "/venv/bin/python" + assert argv == ["/venv/bin/python", "/b/napariViewGEF", "/s", "--list"] + assert "LD_LIBRARY_PATH" not in env + assert env[L.MARKER_VAR] == "reexec" + assert env["HOME"] == "/h" + + def test_python_dash_m_is_restarted_as_python_dash_m(self): + environ = {"NDI_FROM_MATLAB": "1", "PYTHONPATH": "/x"} + _, calls = run(environ, orig_argv=["python", "-I", "-m", "ndi", "napari", "/s"]) + assert calls[0][1] == ["/venv/bin/python", "-I", "-m", "ndi", "napari", "/s"] + + def test_a_cleaned_process_does_not_exec_again(self): + environ = {"NDI_FROM_MATLAB": "1", "LD_LIBRARY_PATH": "/junk", L.MARKER_VAR: "1"} + result, calls = run(environ) + assert result is False + assert calls == [] + + def test_an_ordinary_launch_does_not_exec(self): + result, calls = run({"HOME": "/h", "LD_LIBRARY_PATH": "/usr/local/cuda/lib64"}) + assert result is False + assert calls == [] + + @pytest.mark.parametrize("orig_argv", [[], [""]]) + def test_an_embedded_interpreter_is_never_replaced(self, orig_argv): + # No recorded command line: Python is embedded in a host (MATLAB's + # in-process pyenv, say). exec would replace the HOST. + environ = {"NDI_FROM_MATLAB": "1", "LD_LIBRARY_PATH": MATLAB_LD} + result, calls = run(environ, orig_argv=orig_argv) + assert result is True + assert calls == [] + # Injected environ: reported, never written back. + assert environ == {"NDI_FROM_MATLAB": "1", "LD_LIBRARY_PATH": MATLAB_LD} + + def test_a_host_with_a_command_line_of_its_own_is_never_replaced(self): + # Embedded, but the host passed its own argv through: still no exec. + environ = {"NDI_FROM_MATLAB": "1", "LD_LIBRARY_PATH": MATLAB_LD} + result, calls = run(environ, orig_argv=["matlab", "-nodesktop"], standalone=False) + assert result is True + assert calls == [] + + def test_pytest_itself_counts_as_standalone(self): + assert L._isStandaloneInterpreter() + + def test_a_failed_exec_falls_back_to_cleaning_in_place(self, monkeypatch): + environ = {"NDI_FROM_MATLAB": "1", "LD_LIBRARY_PATH": MATLAB_LD} + monkeypatch.setattr(L.os, "environ", environ) + fake = FakeExec(raise_with=OSError("nope")) + result = L.escapeMatlabEnvironment( + platform="linux", executable="/p", orig_argv=["p", "s"], execve=fake, standalone=True + ) + assert result is True + assert len(fake.calls) == 1 + assert "LD_LIBRARY_PATH" not in environ + assert environ[L.MARKER_VAR] == "in-place" + + +class TestDebugReport: + def test_reports_once_and_only_when_asked(self, monkeypatch): + err = io.StringIO() + monkeypatch.setattr(L.sys, "stderr", err) + environ = {L.MARKER_VAR: "reexec", "NDI_FROM_MATLAB": "1", L.DEBUG_VAR: "1"} + run(environ) + run(environ) + lines = err.getvalue().splitlines() + assert len(lines) == 1 + report = json.loads(lines[0].removeprefix("ndi-launch: ")) + assert report["cleaned"] == "reexec" + assert report["fromMatlab"] is True + assert set(report["env"]) == set(L.LIBRARY_VARS) + assert all(v is None for v in report["env"].values()) + + def test_silent_by_default(self, monkeypatch): + err = io.StringIO() + monkeypatch.setattr(L.sys, "stderr", err) + run({"HOME": "/h"}) + assert err.getvalue() == "" + + +# --- wiring ----------------------------------------------------------------- + + +def test_the_console_scripts_go_through_the_launcher(): + from importlib.metadata import entry_points + + scripts = {ep.name: ep.value for ep in entry_points(group="console_scripts")} + assert scripts["napariViewGEF"] == "_ndi_launch:napariViewGEF" + assert scripts["napariViewLightsheet"] == "_ndi_launch:napariViewLightsheet" + + +def test_the_launcher_imports_nothing_heavy(): + # The point of the module: importing it must not import ndi or numpy. + code = ( + "import sys, _ndi_launch; " + "bad = [m for m in ('ndi', 'numpy', 'pydantic', 'PySide6') if m in sys.modules]; " + "sys.exit(f'imported {bad}' if bad else 0)" + ) + subprocess.run([sys.executable, "-c", code], check=True) + + +CHILD = ( + "import _ndi_launch, os, sys; _ndi_launch.escapeMatlabEnvironment(); " + "print(os.environ.get('LD_LIBRARY_PATH'), os.environ.get('NDI_MATLAB_ENV_CLEANED'))" +) +CHILD_ENV = { + "PATH": "/usr/bin:/bin", + "NDI_FROM_MATLAB": "1", + "LD_LIBRARY_PATH": "/junk/R2026a/sys/os/glnxa64", +} + + +@pytest.mark.skipif(sys.platform.startswith("win"), reason="no exec on Windows") +def test_a_real_restart_drops_the_variables(tmp_path): + # One true exec, end to end, in a child process so pytest is not replaced. + script = tmp_path / "viewer.py" + script.write_text(CHILD) + out = subprocess.run( + [sys.executable, str(script)], env=CHILD_ENV, capture_output=True, text=True, check=True + ) + assert out.stdout.split() == ["None", "reexec"] + + +@pytest.mark.skipif(sys.platform.startswith("win"), reason="no exec on Windows") +def test_no_main_file_means_no_exec(): + # `python -c` has no __main__ file, like an embedded interpreter: cleaned + # in place rather than restarted. + out = subprocess.run( + [sys.executable, "-c", CHILD], env=CHILD_ENV, capture_output=True, text=True, check=True + ) + assert out.stdout.split() == ["None", "in-place"] diff --git a/tools/napariViewGEF b/tools/napariViewGEF deleted file mode 100755 index c580cc6b..00000000 --- a/tools/napariViewGEF +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -# Launcher for the NDI gene pyramid viewer, from a shell or from MATLAB. -# -# Install by symlinking the console script pip already generated, which -# carries the venv's absolute interpreter in its shebang: -# -# ln -s /usr/local/ndi/python/bin/napariViewGEF /usr/local/bin/ -# -# Use THIS wrapper instead when the caller may be MATLAB, which is the -# only reason it exists. -# -# MATLAB prepends its bundled libraries to LD_LIBRARY_PATH on Linux and -# system() children inherit it, so Qt and numpy can load MATLAB's -# libstdc++ instead of the OS copies. Same problem -# ndi.common.systemCurlEnvPrefix solves for curl. -# -# MEASURED on both platforms rather than assumed: -# macOS -- MATLAB sets no MW_* except MW_DESKTOP_IGNORE_STDIN and leaves -# LD/DYLD empty. Nothing to protect against. -# Linux -- MATLAB prepends nine of its own directories, including -# sys/os/glnxa64 (libstdc++) and sys/opengl/lib/glnxa64. -# Contamination is real; numpy and PyQt5 imported fine anyway -# under R2026a on Ubuntu 24.04, so this is insurance rather -# than a fix for a known break. NOT covered: OpenGL, which -# napari needs through vispy and which is what that first path -# entry shadows. Test that by running the viewer on the -# machine, not in CI, where GL is llvmpipe under Xvfb. -# -# MW_ORIG_LD_LIBRARY_PATH is unset on BOTH platforms, so it cannot serve as -# a MATLAB detector and the restore below falls back to blanking -- which -# is what systemCurlEnvPrefix ends up doing too, despite its docstring -# presenting restore as the primary path. Hence the gate is -# NDI_FROM_MATLAB, set by the MATLAB-side launcher: one call site, and no -# dependence on MATLAB internals that vary by platform. -# -# system('NDI_FROM_MATLAB=1 /usr/local/bin/napariViewGEF &'); -if [ -n "${NDI_FROM_MATLAB:-}" ]; then - LD_LIBRARY_PATH="${MW_ORIG_LD_LIBRARY_PATH-}"; export LD_LIBRARY_PATH - DYLD_LIBRARY_PATH="${MW_ORIG_DYLD_LIBRARY_PATH-}"; export DYLD_LIBRARY_PATH - unset DYLD_FRAMEWORK_PATH DYLD_INSERT_LIBRARIES -fi - -# Never legitimate here: it overrides the venv's prefix outright. -unset PYTHONHOME - -exec "${NDI_PYTHON:-/usr/local/ndi/python/bin/python}" -m ndi napari "$@" From 4bbfa6408e89d7000c93aaabd64144ecb2a41971 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 00:25:23 +0000 Subject: [PATCH 2/3] install-ndi.sh: leave profile files byte-identical after uninstall The macOS runners' ~/.bash_profile ends without a newline. Adding the PATH block added one, and --uninstall left it behind. The block's first line now records when a newline was added (and when the file was created), and --uninstall takes exactly that away again. A pre-existing empty profile file is no longer deleted; only one this script created. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015KyLF4xAq4Sm4aT5gDyMTS --- install/install-ndi.sh | 41 ++++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/install/install-ndi.sh b/install/install-ndi.sh index 27516aba..7cca2a4b 100755 --- a/install/install-ndi.sh +++ b/install/install-ndi.sh @@ -56,6 +56,11 @@ VENVS="$NDI_HOME/venvs" PATH_BEGIN="# >>> NDI: added by install-ndi.sh; remove with install-ndi.sh --uninstall >>>" PATH_END="# <<< NDI <<<" +# Marks a block added after a last line that had no newline, so --uninstall +# can take that newline away again and leave the file exactly as it was. +JOINED=" (newline added)" +# Marks a block in a file this script created, which --uninstall deletes. +CREATED=" (file created)" say() { printf '%s\n' "$*"; } fail() { @@ -115,16 +120,24 @@ profile_files() { } # Delete our marked block from one file, in place (keeping the file itself, -# so its permissions, owner and any symlink survive). +# so its permissions, owner and any symlink survive). If the block says it +# added a newline to the line before it, and that line is still the last, +# the newline goes too. remove_block() { file=$1 [ -f "$file" ] || return 0 grep -qF "$PATH_BEGIN" "$file" || return 0 tmp=$(mktemp) - awk -v b="$PATH_BEGIN" -v e="$PATH_END" ' - $0 == b { skip = 1; next } + awk -v b="$PATH_BEGIN" -v j="$PATH_BEGIN$JOINED" -v c="$PATH_BEGIN$CREATED" -v e="$PATH_END" ' + $0 == b || $0 == j || $0 == c { if ($0 == j) joined = n; skip = 1; next } skip && $0 == e { skip = 0; next } - !skip { print } + !skip { line[++n] = $0 } + END { + for (i = 1; i <= n; i++) { + printf "%s", line[i] + if (!(joined && i == joined && i == n)) printf "\n" + } + } ' "$file" > "$tmp" cat "$tmp" > "$file" rm -f "$tmp" @@ -134,11 +147,19 @@ remove_block() { # PATH, not prepended, so the uv in ~/.ndi/bin never shadows a uv of your own. add_to_path() { profile_files | sort -u | while IFS= read -r file; do + begin=$PATH_BEGIN + if [ ! -e "$file" ] || grep -qxF "$PATH_BEGIN$CREATED" "$file"; then + begin="$PATH_BEGIN$CREATED" + fi remove_block "$file" - # A last line with no newline would run into our first one. - if [ -s "$file" ] && [ -n "$(tail -c 1 "$file")" ]; then say >> "$file"; fi + # A last line with no newline would run into our first one: add the + # newline, and say so in the marker so --uninstall can remove it. + if [ -s "$file" ] && [ -n "$(tail -c 1 "$file")" ]; then + say >> "$file" + begin="$PATH_BEGIN$JOINED" + fi { - say "$PATH_BEGIN" + say "$begin" # shellcheck disable=SC2016 # expanded when the shell starts, not now say 'case ":$PATH:" in *":$HOME/.ndi/bin:"*) ;; *) export PATH="$PATH:$HOME/.ndi/bin" ;; esac' say "$PATH_END" @@ -150,10 +171,12 @@ add_to_path() { remove_from_path() { for file in "$HOME/.zshrc" "$HOME/.zprofile" "$HOME/.bashrc" "$HOME/.profile" "$HOME/.bash_profile"; do if [ -f "$file" ] && grep -qF "$PATH_BEGIN" "$file"; then + created=0 + if grep -qxF "$PATH_BEGIN$CREATED" "$file"; then created=1; fi remove_block "$file" say " removed the NDI PATH lines from $file" - # Nothing left: it was a file we created. Remove it again. - if [ ! -s "$file" ]; then rm -f "$file"; fi + # A file this script created, with nothing else in it: delete it. + if [ "$created" = 1 ] && [ ! -s "$file" ]; then rm -f "$file"; fi fi done } From 52b58cc065e492080dc1b6a1d1196ae2c2de3d1b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 00:26:14 +0000 Subject: [PATCH 3/3] install-ndi.ps1: keep the user PATH string exact across install and uninstall The Windows runner's user Path ends in ';'. Dropping empty entries on the way in meant uninstall wrote back the string without it. Entries are now kept as they are, ours goes before a trailing empty one, and removal takes only ours, so add-then-remove returns the identical string. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015KyLF4xAq4Sm4aT5gDyMTS --- install/install-ndi.ps1 | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/install/install-ndi.ps1 b/install/install-ndi.ps1 index daace928..012b416b 100644 --- a/install/install-ndi.ps1 +++ b/install/install-ndi.ps1 @@ -90,9 +90,11 @@ Remove-Item Env:PYTHONHOME, Env:PYTHONPATH -ErrorAction SilentlyContinue $EnvKey = 'registry::HKEY_CURRENT_USER\Environment' +# Entries are kept exactly, empty ones included (a trailing ';' is common), +# so that adding and then removing ours gives back the identical string. function Get-UserPathEntries { - $raw = (Get-Item -LiteralPath $EnvKey).GetValue('Path', '', 'DoNotExpandEnvironmentNames') - return @($raw -split ';' | Where-Object { $_ -ne '' }) + $raw = [string](Get-Item -LiteralPath $EnvKey).GetValue('Path', '', 'DoNotExpandEnvironmentNames') + return , @($raw -split ';') } function Test-IsNdiBin([string]$Entry) { @@ -112,8 +114,15 @@ function Add-NdiToPath { Write-Host " $Bin is already on your PATH" return } - # Appended, not prepended, so the uv in .ndi\bin never shadows one of yours. - Set-UserPathEntries (@($entries) + $Bin) + # Appended, not prepended, so the uv in .ndi\bin never shadows one of yours; + # before a trailing empty entry, so "a;b;" becomes "a;b;;". + $n = $entries.Count + if ($entries[$n - 1] -eq '') { + if ($n -gt 1) { $new = @($entries[0..($n - 2)]) + $Bin + '' } else { $new = @($Bin, '') } + } else { + $new = @($entries) + $Bin + } + Set-UserPathEntries $new Write-Host " added $Bin to your PATH" }