From c90286883d07ba2247ddc016681bf3d7a07ee322 Mon Sep 17 00:00:00 2001 From: Christian Tabedzki <35670232+tabedzki@users.noreply.github.com> Date: Fri, 18 Sep 2026 11:18:06 -0400 Subject: [PATCH] fix: own the on-disk annotation layer: zarr decode, atomic fetch, right level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit brainglobe's `annotation` property materialises the volume with `dask.compute()`: every chunk is decompressed into its own buffer and only then concatenated, so the process briefly holds two copies. Measured on whs_sd_rat_39um (1.07 GB array): 2,305 MB peak RSS. On a 3 GB container that peak, on top of the Bokeh baseline and whatever atlas is already resident, is what OOM-kills the server when a user computes an overlay with a second large atlas. `load_annotation` reads the same chunks straight through zarr indexing into one preallocated array. Peak drops to 1,263 MB (the array plus baseline), the decode is ~2x faster, and the result is byte-identical. The pyramid level is chosen from the standard OME `multiscales` metadata by matching the atlas resolution. Nothing in this module touches `atlas.annotation` any more. Owning that step exposes two more problems in the same layer: * A killed download passed for a finished one. `fs.get` writes chunks one at a time, and the `c` directory it creates is the only "already downloaded" signal both this loader and brainglobe use; an OOM kill mid-transfer (which a large atlas is exactly the thing to cause) left a partial `c` that was trusted forever and read back with the missing chunks as zeros. Counting chunks can't detect this -- a complete download omits all-zero chunks (the rat atlas ships 217 of 512) -- so the fetch now lands in a staging directory and is renamed into place. * `is_downloaded` accepted *any* cached pyramid level. One annotation set serves a whole atlas family: allen_mouse_25um is level s1, allen_mouse_10um is s0. With 25 µm baked into the image, the 10 µm atlas (4.8 GB, nothing local) reported as downloaded, so the GUI took its "this is free" path and the download ran inline on the Bokeh event loop. The check now resolves the atlas's own level. `ensure_downloaded` now loads via `canonical_annotation` so the volume decoded on the worker thread is the one the overlay compute reads, not a throwaway copy. Assisted-by: Claude-Code:Opus-5 --- pixelmap/anatomy/atlas.py | 131 +++++++++++++++++++++--- tests/test_anatomy_atlas.py | 192 +++++++++++++++++++++++++++++++++--- 2 files changed, 297 insertions(+), 26 deletions(-) diff --git a/pixelmap/anatomy/atlas.py b/pixelmap/anatomy/atlas.py index 7fc1090..aacab31 100644 --- a/pixelmap/anatomy/atlas.py +++ b/pixelmap/anatomy/atlas.py @@ -25,6 +25,12 @@ * Atlases are small on disk: allen_mouse_25um and whs_sd_rat_39um together are ~9 MB of compressed chunks, against ~1.3 GB of tiffs under v2. That is what makes baking them into the Docker image cheap and reliable. +* brainglobe decodes ``annotation`` through dask (``.data.compute()``), which + holds every decompressed chunk *and* the assembled array at once: the + 1.07 GB rat volume peaks at +2.1 GB RSS, and a 4.8 GB 10 µm volume would + need ~10 GB. :func:`load_annotation` reads the same chunks straight through + zarr into one preallocated array instead (peak ≈ the array itself, and about + twice as fast), so nothing here goes through ``atlas.annotation``. Why the network never runs inline --------------------------------- @@ -40,13 +46,20 @@ from __future__ import annotations import functools +import os +import shutil import threading from dataclasses import dataclass from pathlib import Path import numpy as np +import zarr from brainglobe_atlasapi import BrainGlobeAtlas -from brainglobe_atlasapi.descriptors import V3_ANNOTATION_NAME, V3_ATLAS_ROOTDIR +from brainglobe_atlasapi.descriptors import ( + V3_ANNOTATION_NAME, + V3_ATLAS_ROOTDIR, + remote_url_s3, +) from brainglobe_atlasapi.list_atlases import ( get_all_atlases_lastversions, get_downloaded_atlases, @@ -96,8 +109,99 @@ def ensure_downloaded(name: str = _DEFAULT_ATLAS) -> None: a single place so callers can push it onto a worker thread — which the GUI does, because on the server the calling thread also serves every other session. A no-op once the atlas is local. + + Loads into the same cache the overlay compute reads from, so the volume + decoded here is the one used afterwards rather than a throwaway copy. + """ + canonical_annotation(name) + + +def load_annotation(atlas) -> np.ndarray: + """Decode ``atlas``'s annotation volume into memory, at ~1x its size. + + The drop-in for ``atlas.annotation``. brainglobe's property builds the + array with ``dask.compute()``, which decompresses every chunk into its own + buffer and only then concatenates them -- so the process briefly holds two + copies of the volume (measured: 2.3 GB peak for the 1.07 GB rat atlas). + Reading the level through zarr's own indexing decodes chunk by chunk into + one preallocated array; the peak is the array plus a chunk. + + Fetches the level from S3 first if its chunks aren't local -- the same + ``fs.get`` brainglobe's property makes, without the decode that follows + it, and staged so a killed download can't pass for a finished one (see + :func:`_download_level`). This is the one blocking call in the module; + see :func:`ensure_downloaded`. + + Test doubles that carry ``annotation`` as a plain attribute have no zarr + on disk and are read as-is. """ - _ = get_atlas(name).annotation + if not hasattr(atlas, "root_dir") or not hasattr(atlas, "metadata"): + return np.asarray(atlas.annotation) + location, root = _annotation_root(atlas) + dataset = _pyramid_dataset(root, atlas.resolution) + level = root / dataset + if not (level / "c").is_dir(): + _download_level( + atlas.fs, + remote_url_s3.format(f"{location}/{V3_ANNOTATION_NAME}/{dataset}/"), + level, + ) + return zarr.open_array(level, mode="r")[:] + + +def _annotation_root(atlas) -> tuple[str, Path]: + """``(location, path)`` of the atlas's OME-Zarr annotation pyramid on disk. + + ``location`` is the S3-relative key brainglobe stores (leading ``/`` + stripped); the same annotation set is shared by every resolution of an + atlas family, with one pyramid level per resolution. + """ + location = atlas.metadata["annotation_set"]["location"][1:] + return location, Path(atlas.root_dir) / location / V3_ANNOTATION_NAME + + +def _download_level(fs, remote: str, level: Path) -> None: + """Fetch a pyramid level so that ``level/c`` exists only once it is complete. + + ``fs.get`` writes chunk files one at a time. A container OOM-killed + mid-transfer -- which a large atlas is exactly the thing to cause -- + leaves a ``c`` directory holding some of the chunks, and that directory + is the *only* signal both this loader and brainglobe use for "already + downloaded". Nothing re-fetches, and zarr silently reads every missing + chunk as fill value (0, "outside the brain"): a truncated atlas that looks + fine. A chunk count can't catch it either, because a complete download + legitimately omits all-zero chunks (the rat atlas ships 217 of 512). + + So fetch into a sibling staging directory and move the finished tree + into place with a rename, which is atomic on the same filesystem: ``c`` + is either whole or absent. A stale staging dir from an earlier kill is + just discarded. + """ + staging = level.with_name(level.name + ".partial") + shutil.rmtree(staging, ignore_errors=True) + fs.get(remote, str(staging), recursive=True) + level.mkdir(parents=True, exist_ok=True) + for item in staging.iterdir(): + target = level / item.name + if item.name == "c" or not target.exists(): + os.replace(item, target) + shutil.rmtree(staging, ignore_errors=True) + + +def _pyramid_dataset(root: Path, resolution_um) -> str: + """Path (within the OME-Zarr pyramid at ``root``) of the level at ``resolution_um``. + + Matched the way brainglobe picks its level: the dataset whose ``scale`` + transform (in mm) equals the atlas resolution. Reads only ``zarr.json``. + """ + attrs = zarr.open_group(root, mode="r").attrs + multiscales = (attrs.get("ome") or attrs)["multiscales"][0] + want = np.asarray(resolution_um, dtype=float) / 1000.0 + for dataset in multiscales["datasets"]: + for transform in dataset.get("coordinateTransformations", ()): + if transform.get("type") == "scale" and np.allclose(transform["scale"][-3:], want): + return str(dataset["path"]) + raise ValueError(f"no pyramid level at {tuple(resolution_um)} µm under {root}") @functools.lru_cache(maxsize=1) @@ -212,20 +316,25 @@ def is_downloaded(name: str = _DEFAULT_ATLAS) -> bool: def _annotation_is_cached(atlas) -> bool: - """True if the lazily-fetched annotation chunks are already local. + """True if the chunks for *this atlas's* pyramid level are already local. Mirrors the check ``core.Atlas.annotation`` makes before hitting S3: the OME-Zarr pyramid holds one directory per scale level, and a level's voxels - live under ``/c``. We accept *any* cached level rather than - reaching into brainglobe's private ``_annotation_pyramid_level`` — the - atlas name pins the resolution, so the only level this app ever pulls is - the one it would read back. + live under ``/c``. It has to be the level this atlas reads, not + any level: one annotation set serves a whole family, so with + allen_mouse_25um cached (level ``s1``) the pyramid directory already + exists and ``allen_mouse_10um`` -- ``s0``, a 4.8 GB volume with nothing + local -- would otherwise report as downloaded. The GUI then took its + "free" path, and the download ran inline on the event loop. """ - location = atlas.metadata["annotation_set"]["location"][1:] - root = Path(atlas.root_dir) / location / V3_ANNOTATION_NAME + _, root = _annotation_root(atlas) if not root.is_dir(): return False - return any(level.joinpath("c").is_dir() for level in root.iterdir() if level.is_dir()) + try: + dataset = _pyramid_dataset(root, atlas.resolution) + except (ValueError, KeyError, FileNotFoundError): + return False + return (root / dataset / "c").is_dir() def origin_corner(name: str = _DEFAULT_ATLAS) -> str: @@ -448,7 +557,7 @@ def canonical_annotation(atlas_name: str): atlas = get_atlas(atlas_name) axes = anatomical_axes(atlas) order = (axes["AP"].array_axis, axes["DV"].array_axis, axes["ML"].array_axis) - arr = np.transpose(atlas.annotation, order) + arr = np.transpose(load_annotation(atlas), order) flip_axes = tuple(i for i, kind in enumerate(("AP", "DV", "ML")) if axes[kind].flip) if flip_axes: arr = np.flip(arr, axis=flip_axes) diff --git a/tests/test_anatomy_atlas.py b/tests/test_anatomy_atlas.py index 504763d..fd9df3e 100644 --- a/tests/test_anatomy_atlas.py +++ b/tests/test_anatomy_atlas.py @@ -9,6 +9,7 @@ import gc import weakref +from pathlib import Path import numpy as np import pytest @@ -268,6 +269,18 @@ def test_cached_chunks_are_downloaded(self, monkeypatch, listed, tmp_path): ) assert atlas_module.is_downloaded("x") is True + def test_another_resolution_of_the_family_does_not_count( + self, monkeypatch, listed, tmp_path + ): + """allen_mouse_25um and allen_mouse_10um share one annotation set: + 25 µm is level s1, 10 µm is s0. With only s1 local, the 10 µm atlas + is *not* downloaded -- its level is a 4.8 GB fetch away.""" + monkeypatch.setattr( + atlas_module, "get_atlas", + lambda name: _v3_atlas(tmp_path, chunks=True, resolution=(10.0,) * 3), + ) + assert atlas_module.is_downloaded("x") is False + def test_unreadable_metadata_reports_not_downloaded(self, monkeypatch, listed): """A half-written manifest must send callers down the "this will cost you" path rather than crashing the GUI.""" @@ -279,22 +292,22 @@ def _boom(name): assert atlas_module.is_downloaded("x") is False -def _v3_atlas(root, *, chunks: bool): - """A stand-in for a v3 atlas whose OME-Zarr chunks may or may not be local.""" - annotation_dir = root / "annotation-sets" / "some-annotation" / "1_0" - # brainglobe names pyramid levels s0, s1, ...; the level a given atlas - # pulls depends on its resolution, so the check accepts any of them. - scale = annotation_dir / atlas_module.V3_ANNOTATION_NAME / "s1" - (scale / "c" if chunks else scale).mkdir(parents=True) +def _v3_atlas(root, *, chunks: bool, resolution=(25.0, 25.0, 25.0)): + """A stand-in for a v3 atlas over a two-level pyramid (s0 @ 10 µm, s1 @ + 25 µm) whose 25 µm chunks may or may not be local.""" + import shutil - class _A: - root_dir = root - metadata = { - # brainglobe stores locations with a leading "/" that it strips. - "annotation_set": {"location": "/annotation-sets/some-annotation/1_0"} - } - - return _A() + atlas = _DiskAtlas(root, resolution=resolution) + _write_pyramid(atlas.pyramid_root, { + "s0": ((10.0,) * 3, np.zeros((1, 1, 1), np.uint32)), + "s1": ((25.0,) * 3, np.ones((2, 2, 2), np.uint32)), + }) + # s0's chunks are never local (all-zero chunks aren't even written); drop + # s1's unless the test wants them. + shutil.rmtree(atlas.pyramid_root / "s0" / "c", ignore_errors=True) + if not chunks: + shutil.rmtree(atlas.pyramid_root / "s1" / "c") + return atlas class TestRegistryCachePath: @@ -377,6 +390,10 @@ def test_materialises_the_annotation(self, monkeypatch): class _A: def __init__(self, name, **_kwargs): self.name = name + self.orientation = "asr" + self.resolution = (25.0, 25.0, 25.0) + self.shape = (2, 2, 2) + self.structures = {} @property def annotation(self): @@ -386,6 +403,151 @@ def annotation(self): monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _A) atlas_module.ensure_downloaded("some_atlas") assert reads == ["some_atlas"] + # ...and the decoded volume is the one the overlay will read: no + # second decode when the compute follows. + atlas_module.canonical_annotation("some_atlas") + assert reads == ["some_atlas"] + + +def _write_pyramid(root, levels, chunks=(2, 2, 2)): + """Build a tiny OME-Zarr pyramid like brainglobe v3 ships: one group with + ``ome.multiscales`` and a uint32 array per level. ``levels`` maps a dataset + path to ``(scale_um, array)``.""" + import zarr + + group = zarr.open_group(root, mode="w") + datasets = [] + for path, (scale_um, arr) in levels.items(): + z = group.create_array(path, shape=arr.shape, chunks=chunks, dtype="uint32") + z[:] = arr + datasets.append({ + "path": path, + "coordinateTransformations": [ + {"type": "scale", "scale": [s / 1000.0 for s in scale_um]}, + {"type": "translation", "translation": [0.0, 0.0, 0.0]}, + ], + }) + group.attrs["ome"] = {"multiscales": [{"datasets": datasets}]} + return root + + +class _DiskAtlas: + """A brainglobe-v3-shaped atlas over a pyramid written by ``_write_pyramid``. + + ``annotation`` raises so the tests prove the loader never takes dask's + path; ``fs`` records what a download would fetch.""" + + def __init__(self, tmp_path, resolution=(25.0, 25.0, 25.0)): + self.root_dir = tmp_path + self.metadata = {"annotation_set": {"location": "/annotation-sets/x"}} + self.resolution = resolution + self.orientation = "asr" + self.structures = {} + self.fetched = [] + self.fs = self + + def get(self, remote, local, recursive=False): # fsspec-shaped + self.fetched.append((remote, local)) + + @property + def annotation(self): + raise AssertionError("must decode through zarr, not brainglobe's dask path") + + @property + def pyramid_root(self): + return self.root_dir / "annotation-sets" / "x" / atlas_module.V3_ANNOTATION_NAME + + +class TestLoadAnnotation: + """``load_annotation`` decodes straight from the zarr chunks. + + brainglobe's ``annotation`` property computes a dask graph that holds every + decompressed chunk alongside the assembled array (2x the volume at peak: + 2.3 GB for the 1.07 GB rat atlas). Reading the level through zarr + indexing keeps the peak at the array itself. + """ + + def test_reads_the_level_matching_the_atlas_resolution(self, tmp_path): + atlas = _DiskAtlas(tmp_path, resolution=(50.0, 50.0, 50.0)) + fine = np.arange(64, dtype=np.uint32).reshape(4, 4, 4) + coarse = np.full((2, 2, 2), 7, dtype=np.uint32) + _write_pyramid(atlas.pyramid_root, {"s0": ((25.0,) * 3, fine), "s1": ((50.0,) * 3, coarse)}) + out = atlas_module.load_annotation(atlas) + assert out.dtype == np.uint32 and out.shape == (2, 2, 2) + assert (out == 7).all() + assert atlas.fetched == [], "chunks were local: no download" + + def test_returns_a_plain_array_not_a_lazy_view(self, tmp_path): + atlas = _DiskAtlas(tmp_path) + _write_pyramid(atlas.pyramid_root, {"s0": ((25.0,) * 3, np.ones((4, 4, 4), np.uint32))}) + out = atlas_module.load_annotation(atlas) + assert isinstance(out, np.ndarray) and out.flags.owndata + + def test_downloads_the_level_when_chunks_are_missing(self, tmp_path): + import shutil + + atlas = _DiskAtlas(tmp_path) + arr = np.arange(64, dtype=np.uint32).reshape(4, 4, 4) + _write_pyramid(atlas.pyramid_root, {"s0": ((25.0,) * 3, arr)}) + level = atlas.pyramid_root / "s0" + # Stash the chunks so a "download" can restore them. + stash = tmp_path / "remote" + shutil.move(level / "c", stash) + + def _fetch(remote, local, recursive=False): + atlas.fetched.append((remote, local)) + shutil.copytree(stash, Path(local) / "c") + shutil.copy(level / "zarr.json", Path(local) / "zarr.json") + + atlas.get = _fetch + out = atlas_module.load_annotation(atlas) + name = atlas_module.V3_ANNOTATION_NAME + staging = level.with_name("s0.partial") + assert atlas.fetched == [ + (f"s3://brainglobe/atlas/annotation-sets/x/{name}/s0/", str(staging)) + ] + np.testing.assert_array_equal(out, arr) + assert (level / "c").is_dir() and not staging.exists() + + def test_a_killed_download_leaves_no_chunk_dir_behind(self, tmp_path): + """The `c` directory is the only "downloaded" signal; a partial one + would be trusted forever and read back as a truncated atlas.""" + import shutil + + atlas = _DiskAtlas(tmp_path) + arr = np.arange(64, dtype=np.uint32).reshape(4, 4, 4) + _write_pyramid(atlas.pyramid_root, {"s0": ((25.0,) * 3, arr)}) + level = atlas.pyramid_root / "s0" + stash = tmp_path / "remote" + shutil.move(level / "c", stash) + attempts = [] + + def _fetch(remote, local, recursive=False): + attempts.append(local) + if len(attempts) == 1: + # Half the chunks land, then the process dies. + (Path(local) / "c" / "0" / "0").mkdir(parents=True) + (Path(local) / "c" / "0" / "0" / "0").write_bytes(b"partial") + raise KeyboardInterrupt("OOM-killed mid-transfer") + shutil.copytree(stash, Path(local) / "c") + + atlas.get = _fetch + with pytest.raises(KeyboardInterrupt): + atlas_module.load_annotation(atlas) + assert not (level / "c").exists(), "partial download must not look complete" + # Next attempt discards the stale staging dir and fetches again. + np.testing.assert_array_equal(atlas_module.load_annotation(atlas), arr) + assert len(attempts) == 2 + + def test_unknown_resolution_is_an_error_not_a_silent_level(self, tmp_path): + atlas = _DiskAtlas(tmp_path, resolution=(10.0, 10.0, 10.0)) + _write_pyramid(atlas.pyramid_root, {"s0": ((25.0,) * 3, np.ones((2, 2, 2), np.uint32))}) + with pytest.raises(ValueError, match="10.0"): + atlas_module.load_annotation(atlas) + + def test_doubles_without_a_zarr_are_read_as_is(self): + out = atlas_module.load_annotation(_FakeAtlas("x")) + assert out.shape == (4, 4, 4) class TestAtlasesAreReleased: