From 9b2893fcc7f1559d73a2d024f3ccf8ab200e161e Mon Sep 17 00:00:00 2001 From: David Ackerman Date: Thu, 22 Oct 2026 00:44:47 -0400 Subject: [PATCH 1/2] Cut 0.3.0 The first release since 0.2.3 (2026-03-19): the cleanup (#103), the browser app, finetuning in both modes, serving on chunkmirage, the worldarray reader, the consolidation (#150) and the paint logic written once (#152), as the changelog's Unreleased section listed them; it is dated here. The version moves in the three places that carry it (the package, pixi.toml, Fileglancer's runnables.yaml). The commands from before 0.3.0 keep working as aliases this release and go in the next, with the shims, once salamander, skoots-mito and cellmap-analyze have moved off them. The pyproject comment on painting names the neuroglancer fork commit pixi pins, not the moving branch. Claude-Session: https://claude.ai/code/session_01EzWMS4pG3tm1ZtvKULfSfW --- CHANGELOG.md | 2 ++ cellmap_flow/__init__.py | 2 +- pixi.toml | 2 +- pyproject.toml | 7 +++++-- runnables.yaml | 2 +- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cc49252..ed1cfe25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ All notable changes to cellmap-flow. The format follows [Keep a Changelog](https ## Unreleased +## 0.3.0 — 2026-10-22 + ### Added - **The paint logic written once, in Python, and run by the browser too.** `cellmap_flow/portable/` holds the Python both modes run, importable with the standard library, numpy, scipy, pydantic and Pillow alone (`tests/portable/test_boundary.py` holds it to that): the rank score (`portable/rank.py`, moved whole from `finetune/rank.py`, which keeps the import path) and the maths of *Use this model*'s paint carry (`portable/carry.py`, out of `carry_paint` in `finetune/session/volume.py`, which keeps the zarr reads and writes around it). The browser app bundles those files as text and runs them in chunkmirage's Pyodide worker (`Inference.python`: chunkmirage's `Engine.call` by module path, the one hook added to chunkmirage for it, pending there as PR *consumer-python*); Pyodide and numpy load in the background when a dataset opens, about 17 MB once per browser. The TypeScript ports `web/src/finetune/rank.ts` and `carry.ts` are gone with their unit tests, and the rank routes are tested on the same Python in node's Pyodide (`web/test/unit/pyodide.ts`; skipped without the network). The pilot of *One implementation per behaviour* in `web/ARCHITECTURE.md`. - **The dashboard's contract, written once and generated for the browser.** `cellmap_flow/schema.py` exports a JSON Schema of every request model (`dashboard/requests.py`), the table of every route with where it is answered (both modes, a browser stub, the server only, the browser only) and the constants both modes share (the rank score's, Undo's depth, the AI annotation's caps, the Finetune form's defaults); `npm run gen` in web/ writes `web/src/generated/cellmap-flow.{schema.json,ts}` and `docs/api.md`, and a test fails when they are stale, when a Flask route is added without saying where it is answered, or when the browser's router registers a route the table does not give it or misses one it does. The browser takes the shared numbers from the generated file. The rule it serves, *One implementation per behaviour*, is in `web/ARCHITECTURE.md`. diff --git a/cellmap_flow/__init__.py b/cellmap_flow/__init__.py index b9cd04ef..0e39e121 100644 --- a/cellmap_flow/__init__.py +++ b/cellmap_flow/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.2.3" +__version__ = "0.3.0" __version_info__ = tuple(int(i) for i in __version__.split(".")) # Plugins (~/.cellmap_flow/plugins/*.py) are not loaded here: the commands diff --git a/pixi.toml b/pixi.toml index 2d93bf29..5b851cef 100644 --- a/pixi.toml +++ b/pixi.toml @@ -15,7 +15,7 @@ # serve and finetune on Apple's GPU). CUDA is Linux-only. [workspace] name = "cellmap-flow" -version = "0.2.3" +version = "0.3.0" description = "Realtime prediction using neuroglancer" channels = ["nvidia", "conda-forge", "pytorch"] platforms = ["linux-64", "osx-arm64"] diff --git a/pyproject.toml b/pyproject.toml index 657de837..a21c4546 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,8 +86,11 @@ finetune = [ "tensorboard", # Training curves; lora_trainer silently skips logging without it # MinIO server and client are required for annotations but not available on PyPI. # Install via conda-forge: mamba install minio-server minio-client -c conda-forge - # Neuroglancer with voxel annotation support is required but not yet on PyPI. - # Install via: pip install git+https://github.com/briossant/neuroglancer@feature/voxel-annotation + # Painting needs the neuroglancer fork with the voxel-annotation tools + # (google/neuroglancer#858, not on PyPI; a URL here would make this + # package unpublishable). pixi installs it; with pip, install the commit + # pixi.toml pins: pip install + # git+https://github.com/briossant/neuroglancer@fae54db8c667176a602a34f0d9d1028956009836 ] # AI-assisted annotation in the finetune tab (docs/ai_annotate.md): the diff --git a/runnables.yaml b/runnables.yaml index 3365bb1e..349c8d0b 100644 --- a/runnables.yaml +++ b/runnables.yaml @@ -1,6 +1,6 @@ name: CellMap Flow description: Real-time neural network inference on EM microscopy data. -version: "0.2.3" +version: "0.3.0" requirements: # pixi.lock is lockfile version 7, which pixi reads from 0.68.0 on. From bd57ca3c6e3ce8f915cb402c561b9cbcf368014a Mon Sep 17 00:00:00 2001 From: David Ackerman Date: Thu, 22 Oct 2026 09:36:43 -0400 Subject: [PATCH 2/2] Say at the dashboard's start when painting is unavailable, and pin the doctor's fork install to pixi's commit The PyPI release depends on upstream neuroglancer, whose client has none of the paint fork's tools; the fork is installed by pixi. A pip install now hears why at start (cli.doctor.painting_unavailable, the doctor's check), with the install line, and that line names the commit pixi pins instead of the fork's branch; tests/web/test_pins.py keeps the two the same. Claude-Session: https://claude.ai/code/session_01EzWMS4pG3tm1ZtvKULfSfW --- CHANGELOG.md | 1 + cellmap_flow/cli/doctor.py | 21 ++++++++++++++++++++- cellmap_flow/dashboard/app.py | 7 +++++++ tests/web/test_pins.py | 19 +++++++++++++++++-- 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed1cfe25..c1d6d5a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,6 +116,7 @@ All notable changes to cellmap-flow. The format follows [Keep a Changelog](https - **The Flask inference server** (`cellmap_flow/server.py`, `CellMapFlowServer`) and `CELLMAP_FLOW_ENGINE`: chunkmirage serves every model. A model's own environment (`env:`) must have chunkmirage installed, as every environment in `pixi.toml` has; a server started where it is missing fails saying so, where it fell back to the Flask server with a warning. The dashboard draws only chunkmirage layers (`zarr:////zarr`, channels first), so this dashboard no longer draws a server from an older cellmap-flow, nor an older dashboard this release's servers. With the server go `cellmap_flow serve --debug` (Flask's dev-server flag, which the chunkmirage server ignored), `CELLMAP_FLOW_RAW_READ_CONCURRENCY` (`CELLMAP_FLOW_RAW_CACHE_BYTES` stays), the request-cancellation plumbing in `inference.runner` (chunkmirage drops the chunks nobody waits for itself), `ImageDataInterface`'s `concurrency_limit` and `cache_bytes` arguments (the server was their only user; `io.source.open_array` keeps them) and the `flask-cors` dependency. Flask itself stays: the dashboard is a Flask app. ### Fixed +- **The dashboard says when painting is unavailable.** A pip install has upstream neuroglancer, whose client lacks the paint fork's brush, flood fill and MinIO writes (the fork is not on PyPI; pixi installs it). `cellmap_flow view` and the other dashboard commands now log one warning at start, with the install line, instead of a Draw tab that comes up without a brush; `cellmap_flow doctor` says the same. The doctor's install line names the commit pixi pins, not the fork's branch, and a test keeps the two together. - **A `cellmap` model is served at the shape its export runs at.** cellmap_models traces the TorchScript at the metadata's `inference_input_shape`, and a traced UNet runs only at that shape; the model type read the training `input_shape` instead, so `jrc_mus-livers_16nm_to_8nm_mito` and `_peroxisome` (216³ in training, 288³ exported) failed every chunk with "padded input size per channel: (1 x 1 x 1)". The export is traced at whichever shape the exporter passed, and the metadata does not say which (`mito_aff_unet_setup_16` is the other way round: traced at its training 178³, it answers 56³ to the inference 378³), so the model type now asks the model: a forward at the training shape that answers the training output means the training shapes, anything else the inference shapes. - **daisy is pinned below 2.** daisy 2.0.0 (released 2026-10-21) is a rewrite without `daisy.tcp`, `daisy.Server` and `daisy.Client`, which blockwise uses; the pin keeps installs on the 1.2 line until blockwise is ported. - **A model's `scale` is honoured wherever it is started.** A config with `scale` launched from the Models tab (ticked again after a YAML started it, or run by *Add a model* or *Use this model*) was served at the session's data path, the scale ignored; only `cellmap_flow infer`, `yaml` and blockwise applied it. Every launcher now reads the level the scale names (`serving.launch.launch`). diff --git a/cellmap_flow/cli/doctor.py b/cellmap_flow/cli/doctor.py index bf1e6d1d..49013352 100644 --- a/cellmap_flow/cli/doctor.py +++ b/cellmap_flow/cli/doctor.py @@ -28,8 +28,13 @@ # would not work: the fork keeps upstream's version string. NEUROGLANCER_FORK_MARKER = "Paint Value" +# The fork's commit pixi installs (pixi.toml, the finetune feature's +# pypi-dependencies); tests/web/test_pins.py keeps the two the same, so a pip +# user who follows the doctor gets the client that deploys. The install +# builds the client with npm and a C++ extension with the compiler at hand. +NEUROGLANCER_FORK_COMMIT = "fae54db8c667176a602a34f0d9d1028956009836" NEUROGLANCER_FORK_INSTALL = ( - "pip install git+https://github.com/briossant/neuroglancer@feature/voxel-annotation" + f"pip install git+https://github.com/briossant/neuroglancer@{NEUROGLANCER_FORK_COMMIT}" ) # MinIO stopped publishing prebuilt community server binaries: dl.min.io @@ -130,6 +135,20 @@ def _check_neuroglancer_fork(): ) +def painting_unavailable(): + """One line for the dashboard's start when the installed neuroglancer + cannot paint (_check_neuroglancer_fork), with the install line; None when + it can. dashboard.app.create_and_run_app logs it, so a pip install, which + has upstream neuroglancer, hears at start why the Draw tab has no brush.""" + result = _check_neuroglancer_fork() + if result.status == "ok": + return None + return ( + f"Painting is unavailable: {result.detail}. To paint, install the fork: " + f"{result.fix} (cellmap_flow doctor checks the rest of the environment)." + ) + + def _check_binary(name, fix): path = shutil.which(name) if path is None: diff --git a/cellmap_flow/dashboard/app.py b/cellmap_flow/dashboard/app.py index cacf2666..7d3dd52e 100644 --- a/cellmap_flow/dashboard/app.py +++ b/cellmap_flow/dashboard/app.py @@ -115,11 +115,18 @@ def create_and_run_app(neuroglancer_url=None, server=None): the plugins, whose normalizers, postprocessors and model types its pages offer, if nothing has yet. """ + from cellmap_flow.cli.doctor import painting_unavailable from cellmap_flow.plugins import load_plugins if not logging.getLogger().handlers: configure_logging() load_plugins() + # A pip install has upstream neuroglancer, whose client lacks the paint + # fork's brush, flood fill and MinIO writes: say so here, with the + # install line, rather than let the Draw tab come up without a brush. + unavailable = painting_unavailable() + if unavailable: + logger.warning(unavailable) get_session().neuroglancer_url = neuroglancer_url if server is None: server = bind_dashboard() diff --git a/tests/web/test_pins.py b/tests/web/test_pins.py index dc7f1b3d..e1f773ae 100644 --- a/tests/web/test_pins.py +++ b/tests/web/test_pins.py @@ -1,5 +1,6 @@ -"""chunkmirage is pinned twice, and the two pins move together. -pyproject.toml's is a version range, the release the Python inference server +"""Pins that move together: chunkmirage's two, and the neuroglancer fork's two. + +chunkmirage is pinned twice. pyproject.toml's is a version range, the release the Python inference server is installed from: PyPI refuses an upload whose dependencies name a URL, so a git pin there would keep cellmap-flow from being released. web/chunkmirage.lock's is a commit, the one the browser engine is cloned @@ -31,3 +32,17 @@ def test_the_server_takes_a_chunkmirage_release_and_the_browser_a_commit(): ) commits = LOCK_PIN.findall((ROOT / "web" / "chunkmirage.lock").read_text()) assert len(commits) == 1, f"web/chunkmirage.lock should pin one full commit, found {commits}" + + +def test_the_doctors_neuroglancer_fork_install_names_the_commit_pixi_installs(): + """The paint fork is installed by pixi (pixi.toml's finetune feature) and, + for a pip install, by the line `cellmap_flow doctor` prints: the two name + one commit, or a pip user who follows the doctor gets a different client + from the one that deploys. pyproject.toml's comment names it too.""" + from cellmap_flow.cli.doctor import NEUROGLANCER_FORK_COMMIT, NEUROGLANCER_FORK_INSTALL + + pixi = tomllib.loads((ROOT / "pixi.toml").read_text()) + pinned = pixi["feature"]["finetune"]["pypi-dependencies"]["neuroglancer"]["rev"] + assert NEUROGLANCER_FORK_COMMIT == pinned + assert NEUROGLANCER_FORK_INSTALL.endswith(f"@{pinned}") + assert pinned in (ROOT / "pyproject.toml").read_text()