diff --git a/pixelmap/anatomy/atlas.py b/pixelmap/anatomy/atlas.py index 7fc1090..aacab31 100644 --- a/pixelmap/anatomy/atlas.py +++ b/pixelmap/anatomy/atlas.py @@ -25,6 +25,12 @@ * Atlases are small on disk: allen_mouse_25um and whs_sd_rat_39um together are ~9 MB of compressed chunks, against ~1.3 GB of tiffs under v2. That is what makes baking them into the Docker image cheap and reliable. +* brainglobe decodes ``annotation`` through dask (``.data.compute()``), which + holds every decompressed chunk *and* the assembled array at once: the + 1.07 GB rat volume peaks at +2.1 GB RSS, and a 4.8 GB 10 µm volume would + need ~10 GB. :func:`load_annotation` reads the same chunks straight through + zarr into one preallocated array instead (peak ≈ the array itself, and about + twice as fast), so nothing here goes through ``atlas.annotation``. Why the network never runs inline --------------------------------- @@ -40,13 +46,20 @@ from __future__ import annotations import functools +import os +import shutil import threading from dataclasses import dataclass from pathlib import Path import numpy as np +import zarr from brainglobe_atlasapi import BrainGlobeAtlas -from brainglobe_atlasapi.descriptors import V3_ANNOTATION_NAME, V3_ATLAS_ROOTDIR +from brainglobe_atlasapi.descriptors import ( + V3_ANNOTATION_NAME, + V3_ATLAS_ROOTDIR, + remote_url_s3, +) from brainglobe_atlasapi.list_atlases import ( get_all_atlases_lastversions, get_downloaded_atlases, @@ -96,8 +109,99 @@ def ensure_downloaded(name: str = _DEFAULT_ATLAS) -> None: a single place so callers can push it onto a worker thread — which the GUI does, because on the server the calling thread also serves every other session. A no-op once the atlas is local. + + Loads into the same cache the overlay compute reads from, so the volume + decoded here is the one used afterwards rather than a throwaway copy. + """ + canonical_annotation(name) + + +def load_annotation(atlas) -> np.ndarray: + """Decode ``atlas``'s annotation volume into memory, at ~1x its size. + + The drop-in for ``atlas.annotation``. brainglobe's property builds the + array with ``dask.compute()``, which decompresses every chunk into its own + buffer and only then concatenates them -- so the process briefly holds two + copies of the volume (measured: 2.3 GB peak for the 1.07 GB rat atlas). + Reading the level through zarr's own indexing decodes chunk by chunk into + one preallocated array; the peak is the array plus a chunk. + + Fetches the level from S3 first if its chunks aren't local -- the same + ``fs.get`` brainglobe's property makes, without the decode that follows + it, and staged so a killed download can't pass for a finished one (see + :func:`_download_level`). This is the one blocking call in the module; + see :func:`ensure_downloaded`. + + Test doubles that carry ``annotation`` as a plain attribute have no zarr + on disk and are read as-is. """ - _ = get_atlas(name).annotation + if not hasattr(atlas, "root_dir") or not hasattr(atlas, "metadata"): + return np.asarray(atlas.annotation) + location, root = _annotation_root(atlas) + dataset = _pyramid_dataset(root, atlas.resolution) + level = root / dataset + if not (level / "c").is_dir(): + _download_level( + atlas.fs, + remote_url_s3.format(f"{location}/{V3_ANNOTATION_NAME}/{dataset}/"), + level, + ) + return zarr.open_array(level, mode="r")[:] + + +def _annotation_root(atlas) -> tuple[str, Path]: + """``(location, path)`` of the atlas's OME-Zarr annotation pyramid on disk. + + ``location`` is the S3-relative key brainglobe stores (leading ``/`` + stripped); the same annotation set is shared by every resolution of an + atlas family, with one pyramid level per resolution. + """ + location = atlas.metadata["annotation_set"]["location"][1:] + return location, Path(atlas.root_dir) / location / V3_ANNOTATION_NAME + + +def _download_level(fs, remote: str, level: Path) -> None: + """Fetch a pyramid level so that ``level/c`` exists only once it is complete. + + ``fs.get`` writes chunk files one at a time. A container OOM-killed + mid-transfer -- which a large atlas is exactly the thing to cause -- + leaves a ``c`` directory holding some of the chunks, and that directory + is the *only* signal both this loader and brainglobe use for "already + downloaded". Nothing re-fetches, and zarr silently reads every missing + chunk as fill value (0, "outside the brain"): a truncated atlas that looks + fine. A chunk count can't catch it either, because a complete download + legitimately omits all-zero chunks (the rat atlas ships 217 of 512). + + So fetch into a sibling staging directory and move the finished tree + into place with a rename, which is atomic on the same filesystem: ``c`` + is either whole or absent. A stale staging dir from an earlier kill is + just discarded. + """ + staging = level.with_name(level.name + ".partial") + shutil.rmtree(staging, ignore_errors=True) + fs.get(remote, str(staging), recursive=True) + level.mkdir(parents=True, exist_ok=True) + for item in staging.iterdir(): + target = level / item.name + if item.name == "c" or not target.exists(): + os.replace(item, target) + shutil.rmtree(staging, ignore_errors=True) + + +def _pyramid_dataset(root: Path, resolution_um) -> str: + """Path (within the OME-Zarr pyramid at ``root``) of the level at ``resolution_um``. + + Matched the way brainglobe picks its level: the dataset whose ``scale`` + transform (in mm) equals the atlas resolution. Reads only ``zarr.json``. + """ + attrs = zarr.open_group(root, mode="r").attrs + multiscales = (attrs.get("ome") or attrs)["multiscales"][0] + want = np.asarray(resolution_um, dtype=float) / 1000.0 + for dataset in multiscales["datasets"]: + for transform in dataset.get("coordinateTransformations", ()): + if transform.get("type") == "scale" and np.allclose(transform["scale"][-3:], want): + return str(dataset["path"]) + raise ValueError(f"no pyramid level at {tuple(resolution_um)} µm under {root}") @functools.lru_cache(maxsize=1) @@ -212,20 +316,25 @@ def is_downloaded(name: str = _DEFAULT_ATLAS) -> bool: def _annotation_is_cached(atlas) -> bool: - """True if the lazily-fetched annotation chunks are already local. + """True if the chunks for *this atlas's* pyramid level are already local. Mirrors the check ``core.Atlas.annotation`` makes before hitting S3: the OME-Zarr pyramid holds one directory per scale level, and a level's voxels - live under ``/c``. We accept *any* cached level rather than - reaching into brainglobe's private ``_annotation_pyramid_level`` — the - atlas name pins the resolution, so the only level this app ever pulls is - the one it would read back. + live under ``/c``. It has to be the level this atlas reads, not + any level: one annotation set serves a whole family, so with + allen_mouse_25um cached (level ``s1``) the pyramid directory already + exists and ``allen_mouse_10um`` -- ``s0``, a 4.8 GB volume with nothing + local -- would otherwise report as downloaded. The GUI then took its + "free" path, and the download ran inline on the event loop. """ - location = atlas.metadata["annotation_set"]["location"][1:] - root = Path(atlas.root_dir) / location / V3_ANNOTATION_NAME + _, root = _annotation_root(atlas) if not root.is_dir(): return False - return any(level.joinpath("c").is_dir() for level in root.iterdir() if level.is_dir()) + try: + dataset = _pyramid_dataset(root, atlas.resolution) + except (ValueError, KeyError, FileNotFoundError): + return False + return (root / dataset / "c").is_dir() def origin_corner(name: str = _DEFAULT_ATLAS) -> str: @@ -448,7 +557,7 @@ def canonical_annotation(atlas_name: str): atlas = get_atlas(atlas_name) axes = anatomical_axes(atlas) order = (axes["AP"].array_axis, axes["DV"].array_axis, axes["ML"].array_axis) - arr = np.transpose(atlas.annotation, order) + arr = np.transpose(load_annotation(atlas), order) flip_axes = tuple(i for i, kind in enumerate(("AP", "DV", "ML")) if axes[kind].flip) if flip_axes: arr = np.flip(arr, axis=flip_axes) diff --git a/tests/test_anatomy_atlas.py b/tests/test_anatomy_atlas.py index 504763d..fd9df3e 100644 --- a/tests/test_anatomy_atlas.py +++ b/tests/test_anatomy_atlas.py @@ -9,6 +9,7 @@ import gc import weakref +from pathlib import Path import numpy as np import pytest @@ -268,6 +269,18 @@ def test_cached_chunks_are_downloaded(self, monkeypatch, listed, tmp_path): ) assert atlas_module.is_downloaded("x") is True + def test_another_resolution_of_the_family_does_not_count( + self, monkeypatch, listed, tmp_path + ): + """allen_mouse_25um and allen_mouse_10um share one annotation set: + 25 µm is level s1, 10 µm is s0. With only s1 local, the 10 µm atlas + is *not* downloaded -- its level is a 4.8 GB fetch away.""" + monkeypatch.setattr( + atlas_module, "get_atlas", + lambda name: _v3_atlas(tmp_path, chunks=True, resolution=(10.0,) * 3), + ) + assert atlas_module.is_downloaded("x") is False + def test_unreadable_metadata_reports_not_downloaded(self, monkeypatch, listed): """A half-written manifest must send callers down the "this will cost you" path rather than crashing the GUI.""" @@ -279,22 +292,22 @@ def _boom(name): assert atlas_module.is_downloaded("x") is False -def _v3_atlas(root, *, chunks: bool): - """A stand-in for a v3 atlas whose OME-Zarr chunks may or may not be local.""" - annotation_dir = root / "annotation-sets" / "some-annotation" / "1_0" - # brainglobe names pyramid levels s0, s1, ...; the level a given atlas - # pulls depends on its resolution, so the check accepts any of them. - scale = annotation_dir / atlas_module.V3_ANNOTATION_NAME / "s1" - (scale / "c" if chunks else scale).mkdir(parents=True) +def _v3_atlas(root, *, chunks: bool, resolution=(25.0, 25.0, 25.0)): + """A stand-in for a v3 atlas over a two-level pyramid (s0 @ 10 µm, s1 @ + 25 µm) whose 25 µm chunks may or may not be local.""" + import shutil - class _A: - root_dir = root - metadata = { - # brainglobe stores locations with a leading "/" that it strips. - "annotation_set": {"location": "/annotation-sets/some-annotation/1_0"} - } - - return _A() + atlas = _DiskAtlas(root, resolution=resolution) + _write_pyramid(atlas.pyramid_root, { + "s0": ((10.0,) * 3, np.zeros((1, 1, 1), np.uint32)), + "s1": ((25.0,) * 3, np.ones((2, 2, 2), np.uint32)), + }) + # s0's chunks are never local (all-zero chunks aren't even written); drop + # s1's unless the test wants them. + shutil.rmtree(atlas.pyramid_root / "s0" / "c", ignore_errors=True) + if not chunks: + shutil.rmtree(atlas.pyramid_root / "s1" / "c") + return atlas class TestRegistryCachePath: @@ -377,6 +390,10 @@ def test_materialises_the_annotation(self, monkeypatch): class _A: def __init__(self, name, **_kwargs): self.name = name + self.orientation = "asr" + self.resolution = (25.0, 25.0, 25.0) + self.shape = (2, 2, 2) + self.structures = {} @property def annotation(self): @@ -386,6 +403,151 @@ def annotation(self): monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _A) atlas_module.ensure_downloaded("some_atlas") assert reads == ["some_atlas"] + # ...and the decoded volume is the one the overlay will read: no + # second decode when the compute follows. + atlas_module.canonical_annotation("some_atlas") + assert reads == ["some_atlas"] + + +def _write_pyramid(root, levels, chunks=(2, 2, 2)): + """Build a tiny OME-Zarr pyramid like brainglobe v3 ships: one group with + ``ome.multiscales`` and a uint32 array per level. ``levels`` maps a dataset + path to ``(scale_um, array)``.""" + import zarr + + group = zarr.open_group(root, mode="w") + datasets = [] + for path, (scale_um, arr) in levels.items(): + z = group.create_array(path, shape=arr.shape, chunks=chunks, dtype="uint32") + z[:] = arr + datasets.append({ + "path": path, + "coordinateTransformations": [ + {"type": "scale", "scale": [s / 1000.0 for s in scale_um]}, + {"type": "translation", "translation": [0.0, 0.0, 0.0]}, + ], + }) + group.attrs["ome"] = {"multiscales": [{"datasets": datasets}]} + return root + + +class _DiskAtlas: + """A brainglobe-v3-shaped atlas over a pyramid written by ``_write_pyramid``. + + ``annotation`` raises so the tests prove the loader never takes dask's + path; ``fs`` records what a download would fetch.""" + + def __init__(self, tmp_path, resolution=(25.0, 25.0, 25.0)): + self.root_dir = tmp_path + self.metadata = {"annotation_set": {"location": "/annotation-sets/x"}} + self.resolution = resolution + self.orientation = "asr" + self.structures = {} + self.fetched = [] + self.fs = self + + def get(self, remote, local, recursive=False): # fsspec-shaped + self.fetched.append((remote, local)) + + @property + def annotation(self): + raise AssertionError("must decode through zarr, not brainglobe's dask path") + + @property + def pyramid_root(self): + return self.root_dir / "annotation-sets" / "x" / atlas_module.V3_ANNOTATION_NAME + + +class TestLoadAnnotation: + """``load_annotation`` decodes straight from the zarr chunks. + + brainglobe's ``annotation`` property computes a dask graph that holds every + decompressed chunk alongside the assembled array (2x the volume at peak: + 2.3 GB for the 1.07 GB rat atlas). Reading the level through zarr + indexing keeps the peak at the array itself. + """ + + def test_reads_the_level_matching_the_atlas_resolution(self, tmp_path): + atlas = _DiskAtlas(tmp_path, resolution=(50.0, 50.0, 50.0)) + fine = np.arange(64, dtype=np.uint32).reshape(4, 4, 4) + coarse = np.full((2, 2, 2), 7, dtype=np.uint32) + _write_pyramid(atlas.pyramid_root, {"s0": ((25.0,) * 3, fine), "s1": ((50.0,) * 3, coarse)}) + out = atlas_module.load_annotation(atlas) + assert out.dtype == np.uint32 and out.shape == (2, 2, 2) + assert (out == 7).all() + assert atlas.fetched == [], "chunks were local: no download" + + def test_returns_a_plain_array_not_a_lazy_view(self, tmp_path): + atlas = _DiskAtlas(tmp_path) + _write_pyramid(atlas.pyramid_root, {"s0": ((25.0,) * 3, np.ones((4, 4, 4), np.uint32))}) + out = atlas_module.load_annotation(atlas) + assert isinstance(out, np.ndarray) and out.flags.owndata + + def test_downloads_the_level_when_chunks_are_missing(self, tmp_path): + import shutil + + atlas = _DiskAtlas(tmp_path) + arr = np.arange(64, dtype=np.uint32).reshape(4, 4, 4) + _write_pyramid(atlas.pyramid_root, {"s0": ((25.0,) * 3, arr)}) + level = atlas.pyramid_root / "s0" + # Stash the chunks so a "download" can restore them. + stash = tmp_path / "remote" + shutil.move(level / "c", stash) + + def _fetch(remote, local, recursive=False): + atlas.fetched.append((remote, local)) + shutil.copytree(stash, Path(local) / "c") + shutil.copy(level / "zarr.json", Path(local) / "zarr.json") + + atlas.get = _fetch + out = atlas_module.load_annotation(atlas) + name = atlas_module.V3_ANNOTATION_NAME + staging = level.with_name("s0.partial") + assert atlas.fetched == [ + (f"s3://brainglobe/atlas/annotation-sets/x/{name}/s0/", str(staging)) + ] + np.testing.assert_array_equal(out, arr) + assert (level / "c").is_dir() and not staging.exists() + + def test_a_killed_download_leaves_no_chunk_dir_behind(self, tmp_path): + """The `c` directory is the only "downloaded" signal; a partial one + would be trusted forever and read back as a truncated atlas.""" + import shutil + + atlas = _DiskAtlas(tmp_path) + arr = np.arange(64, dtype=np.uint32).reshape(4, 4, 4) + _write_pyramid(atlas.pyramid_root, {"s0": ((25.0,) * 3, arr)}) + level = atlas.pyramid_root / "s0" + stash = tmp_path / "remote" + shutil.move(level / "c", stash) + attempts = [] + + def _fetch(remote, local, recursive=False): + attempts.append(local) + if len(attempts) == 1: + # Half the chunks land, then the process dies. + (Path(local) / "c" / "0" / "0").mkdir(parents=True) + (Path(local) / "c" / "0" / "0" / "0").write_bytes(b"partial") + raise KeyboardInterrupt("OOM-killed mid-transfer") + shutil.copytree(stash, Path(local) / "c") + + atlas.get = _fetch + with pytest.raises(KeyboardInterrupt): + atlas_module.load_annotation(atlas) + assert not (level / "c").exists(), "partial download must not look complete" + # Next attempt discards the stale staging dir and fetches again. + np.testing.assert_array_equal(atlas_module.load_annotation(atlas), arr) + assert len(attempts) == 2 + + def test_unknown_resolution_is_an_error_not_a_silent_level(self, tmp_path): + atlas = _DiskAtlas(tmp_path, resolution=(10.0, 10.0, 10.0)) + _write_pyramid(atlas.pyramid_root, {"s0": ((25.0,) * 3, np.ones((2, 2, 2), np.uint32))}) + with pytest.raises(ValueError, match="10.0"): + atlas_module.load_annotation(atlas) + + def test_doubles_without_a_zarr_are_read_as_is(self): + out = atlas_module.load_annotation(_FakeAtlas("x")) + assert out.shape == (4, 4, 4) class TestAtlasesAreReleased: