From ad106530e4531d410cbf9c1e34e17ca31ef7f6e2 Mon Sep 17 00:00:00 2001 From: Christian Tabedzki <35670232+tabedzki@users.noreply.github.com> Date: Fri, 18 Sep 2026 11:28:08 -0400 Subject: [PATCH] fix: refuse atlases whose decoded volume cannot fit the memory cap An annotation volume is one dense uint32 array: allen_mouse_25um is 308 MB, whs_sd_rat_39um 1.07 GB, and any of the 10 um mouse atlases 4.6 GB. On a container capped at 3 GB the last of those can never be decoded -- the process is OOM-killed, Docker's restart policy brings it back, and the next user to pick the same atlas kills it again. Nothing in the app distinguished "large" from "impossible". The manifest publishes `shape`, so the decoded size is known before a single chunk is fetched. `check_fits` compares it with the cap -- PIXELMAP_MAX_ATLAS_MB, or the cgroup limit minus a 512 MB process baseline, or unlimited outside a container -- and raises AtlasTooLarge. It runs in `ensure_downloaded` (the download worker, which fetches the manifest anyway) and in `canonical_annotation`, so an oversized atlas whose chunks are already on disk is refused at compute time too. The GUI turns either into an error notification that names both sizes and leaves the button usable. Test doubles without a `shape` are treated as "no opinion", never as "fits". Assisted-by: Claude-Code:Opus-5 --- env.template | 7 +++ pixelmap/anatomy/_memory.py | 69 ++++++++++++++++++++++++++++ pixelmap/anatomy/atlas.py | 87 ++++++++++++++++++++++++++++++++++- pixelmap/gui/gui.py | 15 ++++++ tests/test_anatomy_atlas.py | 92 +++++++++++++++++++++++++++++++++++++ tests/test_gui_overlays.py | 43 +++++++++++++++++ 6 files changed, 312 insertions(+), 1 deletion(-) create mode 100644 pixelmap/anatomy/_memory.py diff --git a/env.template b/env.template index fdbdadd..9f350f2 100644 --- a/env.template +++ b/env.template @@ -9,6 +9,13 @@ ADDRESS=localhost # sized for N x (base memory + atlases). NUM_PROCS=1 +# Largest atlas (decoded annotation, in MB) a user may compute with. Unset, +# it is the container's cgroup memory limit minus 512 MB for the process +# itself, and unlimited outside a container. Larger atlases are refused +# with a message instead of OOM-killing the server. allen_mouse_25um is +# 308 MB, whs_sd_rat_39um 1,074 MB, any 10um mouse atlas 4,592 MB. +# PIXELMAP_MAX_ATLAS_MB=2560 + # Visitor analytics configuration ANALYTICS_DB_PATH=/app/output/visitor_analytics.sqlite3 ANALYTICS_TRUST_PROXY_HEADERS=false diff --git a/pixelmap/anatomy/_memory.py b/pixelmap/anatomy/_memory.py new file mode 100644 index 0000000..c534f60 --- /dev/null +++ b/pixelmap/anatomy/_memory.py @@ -0,0 +1,69 @@ +"""Process-memory helpers for the atlas layer. + +Two facts about the deployed server that the atlas code has to reason +about, kept here so that every caller reasons about them the same way: + +* The container runs under a cgroup memory cap (Compose's + ``deploy.resources.limits.memory``), and that cap is enforced on RSS. + :func:`cgroup_memory_limit_bytes` reads it, so cache budgets and size + gates can be derived from the real limit instead of guessed. +* Dropping the last reference to a large array does not lower RSS. glibc + keeps freed pages in its arenas for reuse (``MALLOC_ARENA_MAX`` caps how + many arenas there are, not what they retain). Measured after clearing + every atlas cache: 1,188 MB RSS, then 211 MB after ``malloc_trim``. An + eviction that doesn't trim is one the OOM killer never sees, which is + what :func:`release_freed_memory` is for. +""" + +from __future__ import annotations + +import ctypes +import gc +from pathlib import Path + +# Where a cgroup exposes the memory cap this process runs under -- v2 first, +# then v1. +_CGROUP_LIMIT_FILES = ( + "/sys/fs/cgroup/memory.max", + "/sys/fs/cgroup/memory/memory.limit_in_bytes", +) + + +def cgroup_memory_limit_bytes(paths=None) -> int | None: + """The container's memory cap, or ``None`` when there isn't one.""" + for path in _CGROUP_LIMIT_FILES if paths is None else paths: + try: + raw = Path(path).read_text().strip() + except OSError: + continue + if raw == "max": + return None + try: + limit = int(raw) + except ValueError: + continue + # cgroup v1 reports a huge number (~2**63) for "unlimited". + if limit <= 0 or limit >= 1 << 60: + return None + return limit + return None + + +_libc = None + + +def release_freed_memory() -> None: + """Return freed heap pages to the OS. + + Collects garbage first so that reference cycles holding large arrays are + actually freed, then asks glibc to trim. A no-op on platforms without + ``malloc_trim`` (macOS, musl). + """ + global _libc + gc.collect() + try: + if _libc is None: + _libc = ctypes.CDLL("libc.so.6") + _libc.malloc_trim(0) + except (OSError, AttributeError): + pass diff --git a/pixelmap/anatomy/atlas.py b/pixelmap/anatomy/atlas.py index 7fc1090..50b3472 100644 --- a/pixelmap/anatomy/atlas.py +++ b/pixelmap/anatomy/atlas.py @@ -26,6 +26,18 @@ ~9 MB of compressed chunks, against ~1.3 GB of tiffs under v2. That is what makes baking them into the Docker image cheap and reliable. +Why some atlases are refused outright +------------------------------------- +The annotation is one dense ``uint32`` array: allen_mouse_25um is 308 MB, +whs_sd_rat_39um 1.07 GB, and any of the 10 µm mouse atlases 4.8 GB. On a +container capped at 3 GB the last of those cannot be decoded at all -- the +process is OOM-killed, Docker restarts it, and the next user who picks the +same atlas kills it again. The array's size is known from the manifest +(``shape``) before a single chunk is fetched, so :func:`check_fits` compares +it with the cap and raises :class:`AtlasTooLarge` first; the GUI turns that +into a message instead of a dead server. ``PIXELMAP_MAX_ATLAS_MB`` overrides +the derived limit; with no cgroup cap and no override nothing is refused. + Why the network never runs inline --------------------------------- The deployed app is a single-process Panel/Bokeh server: a blocking call on @@ -40,18 +52,24 @@ from __future__ import annotations import functools +import os import threading from dataclasses import dataclass from pathlib import Path import numpy as np from brainglobe_atlasapi import BrainGlobeAtlas -from brainglobe_atlasapi.descriptors import V3_ANNOTATION_NAME, V3_ATLAS_ROOTDIR +from brainglobe_atlasapi.descriptors import ( + ANNOTATION_DTYPE, + V3_ANNOTATION_NAME, + V3_ATLAS_ROOTDIR, +) from brainglobe_atlasapi.list_atlases import ( get_all_atlases_lastversions, get_downloaded_atlases, ) +from pixelmap.anatomy import _memory from pixelmap.anatomy._registry_snapshot import REGISTRY_SNAPSHOT _DEFAULT_ATLAS = "allen_mouse_25um" @@ -96,10 +114,76 @@ def ensure_downloaded(name: str = _DEFAULT_ATLAS) -> None: a single place so callers can push it onto a worker thread — which the GUI does, because on the server the calling thread also serves every other session. A no-op once the atlas is local. + + Raises :class:`AtlasTooLarge` before fetching anything bulky if the + decoded volume would not fit the process's memory cap. """ + check_fits(name) _ = get_atlas(name).annotation +class AtlasTooLarge(RuntimeError): + """The atlas's annotation volume exceeds what this process may decode.""" + + def __init__(self, name: str, nbytes: int, limit: int): + self.name, self.nbytes, self.limit = name, nbytes, limit + super().__init__( + f"{name}'s annotation is {nbytes / 2**20:,.0f} MB decoded; this " + f"server allows at most {limit / 2**20:,.0f} MB per atlas" + ) + + +def annotation_nbytes(name: str = _DEFAULT_ATLAS) -> int | None: + """Decoded size of the atlas's annotation array, from metadata alone. + + brainglobe publishes ``shape`` in the manifest and stores annotations as + ``uint32``, so the size is known without fetching a chunk. ``None`` when + the atlas carries no shape (test doubles), which callers treat as "no + opinion" rather than "fits". + """ + shape = getattr(get_atlas(name), "shape", None) + if shape is None: + return None + return int(np.prod([int(s) for s in shape])) * np.dtype(ANNOTATION_DTYPE).itemsize + + +# What the rest of the process needs regardless of atlas: the Bokeh/Panel +# baseline (~300 MB measured) plus headroom for sessions and the metadata +# caches. +_PROCESS_BASELINE_BYTES = 512 * 2**20 + + +def max_annotation_bytes() -> int | None: + """The largest annotation volume this process may decode, or ``None``. + + ``PIXELMAP_MAX_ATLAS_MB`` wins. Otherwise the cgroup memory cap minus the + process baseline: what's left is what one decoded volume may occupy. + ``None`` -- no cap, nothing refused -- outside a container. + """ + env = os.environ.get("PIXELMAP_MAX_ATLAS_MB") + if env: + return int(float(env) * 2**20) + limit = _memory.cgroup_memory_limit_bytes() + if limit is None: + return None + return max(limit - _PROCESS_BASELINE_BYTES, 0) + + +def check_fits(name: str = _DEFAULT_ATLAS) -> None: + """Raise :class:`AtlasTooLarge` if ``name`` cannot be decoded under the cap. + + Needs the manifest, which :func:`get_atlas` fetches on a cold cache, so + call it where a small network round-trip is acceptable (the worker thread + that would download the atlas anyway) or after :func:`is_downloaded`. + """ + limit = max_annotation_bytes() + if limit is None: + return + nbytes = annotation_nbytes(name) + if nbytes is not None and nbytes > limit: + raise AtlasTooLarge(name, nbytes, limit) + + @functools.lru_cache(maxsize=1) def list_atlases() -> list[str]: """Every atlas in the brainglobe registry, not just the downloaded ones. @@ -445,6 +529,7 @@ def canonical_annotation(atlas_name: str): non-Allen orientations work. For an already-``asr`` atlas this is a no-op (identity transpose, no flips), so Allen behavior is unchanged. """ + check_fits(atlas_name) atlas = get_atlas(atlas_name) axes = anatomical_axes(atlas) order = (axes["AP"].array_axis, axes["DV"].array_axis, axes["ML"].array_axis) diff --git a/pixelmap/gui/gui.py b/pixelmap/gui/gui.py index b6b3d9d..c8b3a39 100644 --- a/pixelmap/gui/gui.py +++ b/pixelmap/gui/gui.py @@ -1246,6 +1246,13 @@ async def _on_compute_anatomy_click(self, event): self.compute_anatomy_button.disabled = True try: await asyncio.to_thread(anatomy_atlas.ensure_downloaded, name) + except anatomy_atlas.AtlasTooLarge as exc: + # Known from the manifest before any chunk was fetched: refusing + # here is what keeps a 4.8 GB atlas from OOM-killing the server. + print(f"Atlas refused: {exc}") + self.compute_anatomy_button.name = "Download & compute atlas 🧠 ⏳" + _notify("error", f"{exc}. Pick a coarser resolution of this atlas.") + return except Exception as exc: print(f"Atlas download failed: {exc}") self.compute_anatomy_button.name = "Download & compute atlas 🧠 ⏳" @@ -1288,6 +1295,14 @@ def compute_anatomy_overlay(self): atlas_name=atlas_name, step_um=25.0, ) + except anatomy_atlas.AtlasTooLarge as exc: + # The chunks are on disk (downloaded by an earlier, larger deploy + # or another tool) but decoding them would exceed the memory cap. + print(f"Atlas refused: {exc}") + self.compute_anatomy_button.name = "Compute anatomical overlay 🧠" + self.compute_anatomy_button.disabled = False + _notify("error", f"{exc}. Pick a coarser resolution of this atlas.") + return except Exception as exc: print(f"Anatomy lookup failed: {exc}") self.compute_anatomy_button.name = "Compute anatomical overlay 🧠" diff --git a/tests/test_anatomy_atlas.py b/tests/test_anatomy_atlas.py index 504763d..24f1e82 100644 --- a/tests/test_anatomy_atlas.py +++ b/tests/test_anatomy_atlas.py @@ -13,6 +13,7 @@ import numpy as np import pytest +from pixelmap.anatomy import _memory from pixelmap.anatomy import atlas as atlas_module from pixelmap.anatomy import regions as regions_module from pixelmap.anatomy._registry_snapshot import REGISTRY_SNAPSHOT @@ -388,6 +389,97 @@ def annotation(self): assert reads == ["some_atlas"] +def _sized_atlas_cls(shape): + """A metadata-only fake: ``shape`` is published, ``annotation`` must not + be read -- the size gate has to decide from the manifest alone.""" + + class _A: + def __init__(self, name, **_kwargs): + self.name = name + self.orientation = "asr" + self.resolution = (25.0, 25.0, 25.0) + self.shape = shape + self.structures = {} + + @property + def annotation(self): + raise AssertionError("a refused atlas must never be decoded") + + return _A + + +class TestSizeGate: + """Atlases that cannot fit the memory cap are refused before any decode. + + allen_mouse_10um is a 1320x800x1140 uint32 array: 4.8 GB. Under a 3 GB + container cap that decode is an OOM kill, a Docker restart, and another + kill the next time anyone picks it. The manifest's ``shape`` says so + before a chunk is fetched. + """ + + ALLEN_10UM = (1320, 800, 1140) + + def test_size_comes_from_the_manifest_shape(self, monkeypatch): + monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _sized_atlas_cls(self.ALLEN_10UM)) + assert atlas_module.annotation_nbytes("x") == 1320 * 800 * 1140 * 4 + + def test_no_shape_means_no_opinion(self, monkeypatch): + monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _FakeAtlas) + assert atlas_module.annotation_nbytes("x") is None + monkeypatch.setattr(atlas_module, "max_annotation_bytes", lambda: 1) + atlas_module.check_fits("x") # no shape -> not refused + + def test_env_override_sets_the_limit(self, monkeypatch): + monkeypatch.setenv("PIXELMAP_MAX_ATLAS_MB", "1024") + assert atlas_module.max_annotation_bytes() == 1024 * 2**20 + + def test_limit_is_the_cgroup_cap_minus_the_process_baseline(self, monkeypatch, tmp_path): + monkeypatch.delenv("PIXELMAP_MAX_ATLAS_MB", raising=False) + f = tmp_path / "memory.max" + f.write_text("3221225472\n") # 3 GB `deploy` limit + monkeypatch.setattr(_memory, "_CGROUP_LIMIT_FILES", (str(f),)) + assert atlas_module.max_annotation_bytes() == 3221225472 - 512 * 2**20 + + def test_no_cap_means_nothing_is_refused(self, monkeypatch, tmp_path): + monkeypatch.delenv("PIXELMAP_MAX_ATLAS_MB", raising=False) + monkeypatch.setattr(_memory, "_CGROUP_LIMIT_FILES", (str(tmp_path / "missing"),)) + assert atlas_module.max_annotation_bytes() is None + monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _sized_atlas_cls(self.ALLEN_10UM)) + atlas_module.check_fits("x") + + def test_ensure_downloaded_refuses_before_fetching(self, monkeypatch): + monkeypatch.setenv("PIXELMAP_MAX_ATLAS_MB", "2560") + monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _sized_atlas_cls(self.ALLEN_10UM)) + with pytest.raises(atlas_module.AtlasTooLarge) as info: + atlas_module.ensure_downloaded("allen_mouse_10um") + assert info.value.name == "allen_mouse_10um" + assert "4,592 MB" in str(info.value) and "2,560 MB" in str(info.value) + + def test_every_decode_path_is_gated(self, monkeypatch): + # An atlas whose chunks are already on disk is decoded by + # canonical_annotation, not ensure_downloaded; it must refuse too. + monkeypatch.setenv("PIXELMAP_MAX_ATLAS_MB", "0.5") + monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _sized_atlas_cls((64, 64, 64))) + with pytest.raises(atlas_module.AtlasTooLarge): + atlas_module.canonical_annotation("x") + with pytest.raises(atlas_module.AtlasTooLarge): + atlas_module.lookup_regions("x", np.zeros((1, 3))) + + def test_an_atlas_that_fits_is_untouched(self, monkeypatch): + monkeypatch.setenv("PIXELMAP_MAX_ATLAS_MB", "2560") + monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _sized_atlas_cls((528, 320, 456))) + atlas_module.check_fits("allen_mouse_25um") # 308 MB: fine + + def test_cgroup_limit_parsing(self, tmp_path): + v2 = tmp_path / "memory.max"; v2.write_text("max\n") + v1 = tmp_path / "limit"; v1.write_text(str(2**63 - 4096) + "\n") + real = tmp_path / "real"; real.write_text("2147483648\n") + assert _memory.cgroup_memory_limit_bytes((str(v2),)) is None + assert _memory.cgroup_memory_limit_bytes((str(v1),)) is None + assert _memory.cgroup_memory_limit_bytes((str(tmp_path / "missing"),)) is None + assert _memory.cgroup_memory_limit_bytes((str(real),)) == 2**31 + + class TestAtlasesAreReleased: """Atlas objects must not outlive ``get_atlas``'s cache. diff --git a/tests/test_gui_overlays.py b/tests/test_gui_overlays.py index 1324ec7..16e2bbf 100644 --- a/tests/test_gui_overlays.py +++ b/tests/test_gui_overlays.py @@ -21,6 +21,7 @@ from pixelmap.anatomy import atlas as atlas_module from pixelmap.anatomy import regions as regions_module +from pixelmap.gui import gui as gui_module from pixelmap.gui.gui import ChannelmapGUI from pixelmap.types import Electrode @@ -679,6 +680,48 @@ def _boom(name): assert gui.compute_anatomy_button.disabled is False assert gui.compute_anatomy_button.name == "Download & compute atlas 🧠 ⏳" + def test_an_atlas_too_large_for_the_server_is_refused_not_downloaded( + self, served, monkeypatch + ): + """The manifest says how big the decoded volume is; a 4.8 GB atlas on + a 3 GB container is refused with a message, not OOM-killed.""" + gui, session = served + monkeypatch.setattr(atlas_module, "is_downloaded", lambda name: False) + notices = [] + monkeypatch.setattr(gui_module, "_notify", lambda level, msg, **kw: notices.append((level, msg))) + + def _too_big(name): + raise atlas_module.AtlasTooLarge(name, 4_800 * 2**20, 2_560 * 2**20) + + monkeypatch.setattr(atlas_module, "ensure_downloaded", _too_big) + + asyncio.run(session.with_document_locked(gui._on_compute_anatomy_click, None)) + + assert gui.compute_anatomy_button.disabled is False + assert gui.compute_anatomy_button.name == "Download & compute atlas 🧠 ⏳" + assert len(gui.region_band_source.data["acronym"]) == 0, "nothing was computed" + assert notices and notices[0][0] == "error" + assert "4,800 MB" in notices[0][1] and "2,560 MB" in notices[0][1] + + def test_a_too_large_atlas_already_on_disk_is_refused_at_compute( + self, served, monkeypatch + ): + gui, _session = served + monkeypatch.setattr(atlas_module, "is_downloaded", lambda name: True) + notices = [] + monkeypatch.setattr(gui_module, "_notify", lambda level, msg, **kw: notices.append((level, msg))) + monkeypatch.setattr( + atlas_module, "max_annotation_bytes", lambda: 1, # nothing fits + ) + monkeypatch.setattr(atlas_module, "annotation_nbytes", lambda name: 2) + atlas_module.canonical_annotation.cache_clear() + + gui.compute_anatomy_overlay() + + assert gui.compute_anatomy_button.disabled is False + assert len(gui.region_band_source.data["acronym"]) == 0 + assert notices and notices[0][0] == "error" + def test_no_unlocked_async_gui_callbacks(): """Every async GUI callback needs the same treatment.