diff --git a/env.template b/env.template index fdbdadd..9f350f2 100644 --- a/env.template +++ b/env.template @@ -9,6 +9,13 @@ ADDRESS=localhost # sized for N x (base memory + atlases). NUM_PROCS=1 +# Largest atlas (decoded annotation, in MB) a user may compute with. Unset, +# it is the container's cgroup memory limit minus 512 MB for the process +# itself, and unlimited outside a container. Larger atlases are refused +# with a message instead of OOM-killing the server. allen_mouse_25um is +# 308 MB, whs_sd_rat_39um 1,074 MB, any 10um mouse atlas 4,592 MB. +# PIXELMAP_MAX_ATLAS_MB=2560 + # Visitor analytics configuration ANALYTICS_DB_PATH=/app/output/visitor_analytics.sqlite3 ANALYTICS_TRUST_PROXY_HEADERS=false diff --git a/pixelmap/anatomy/_memory.py b/pixelmap/anatomy/_memory.py new file mode 100644 index 0000000..c534f60 --- /dev/null +++ b/pixelmap/anatomy/_memory.py @@ -0,0 +1,69 @@ +"""Process-memory helpers for the atlas layer. + +Two facts about the deployed server that the atlas code has to reason +about, kept here so that every caller reasons about them the same way: + +* The container runs under a cgroup memory cap (Compose's + ``deploy.resources.limits.memory``), and that cap is enforced on RSS. + :func:`cgroup_memory_limit_bytes` reads it, so cache budgets and size + gates can be derived from the real limit instead of guessed. +* Dropping the last reference to a large array does not lower RSS. glibc + keeps freed pages in its arenas for reuse (``MALLOC_ARENA_MAX`` caps how + many arenas there are, not what they retain). Measured after clearing + every atlas cache: 1,188 MB RSS, then 211 MB after ``malloc_trim``. An + eviction that doesn't trim is one the OOM killer never sees, which is + what :func:`release_freed_memory` is for. +""" + +from __future__ import annotations + +import ctypes +import gc +from pathlib import Path + +# Where a cgroup exposes the memory cap this process runs under -- v2 first, +# then v1. +_CGROUP_LIMIT_FILES = ( + "/sys/fs/cgroup/memory.max", + "/sys/fs/cgroup/memory/memory.limit_in_bytes", +) + + +def cgroup_memory_limit_bytes(paths=None) -> int | None: + """The container's memory cap, or ``None`` when there isn't one.""" + for path in _CGROUP_LIMIT_FILES if paths is None else paths: + try: + raw = Path(path).read_text().strip() + except OSError: + continue + if raw == "max": + return None + try: + limit = int(raw) + except ValueError: + continue + # cgroup v1 reports a huge number (~2**63) for "unlimited". + if limit <= 0 or limit >= 1 << 60: + return None + return limit + return None + + +_libc = None + + +def release_freed_memory() -> None: + """Return freed heap pages to the OS. + + Collects garbage first so that reference cycles holding large arrays are + actually freed, then asks glibc to trim. A no-op on platforms without + ``malloc_trim`` (macOS, musl). + """ + global _libc + gc.collect() + try: + if _libc is None: + _libc = ctypes.CDLL("libc.so.6") + _libc.malloc_trim(0) + except (OSError, AttributeError): + pass diff --git a/pixelmap/anatomy/atlas.py b/pixelmap/anatomy/atlas.py index 7fc1090..50b3472 100644 --- a/pixelmap/anatomy/atlas.py +++ b/pixelmap/anatomy/atlas.py @@ -26,6 +26,18 @@ ~9 MB of compressed chunks, against ~1.3 GB of tiffs under v2. That is what makes baking them into the Docker image cheap and reliable. +Why some atlases are refused outright +------------------------------------- +The annotation is one dense ``uint32`` array: allen_mouse_25um is 308 MB, +whs_sd_rat_39um 1.07 GB, and any of the 10 µm mouse atlases 4.8 GB. On a +container capped at 3 GB the last of those cannot be decoded at all -- the +process is OOM-killed, Docker restarts it, and the next user who picks the +same atlas kills it again. The array's size is known from the manifest +(``shape``) before a single chunk is fetched, so :func:`check_fits` compares +it with the cap and raises :class:`AtlasTooLarge` first; the GUI turns that +into a message instead of a dead server. ``PIXELMAP_MAX_ATLAS_MB`` overrides +the derived limit; with no cgroup cap and no override nothing is refused. + Why the network never runs inline --------------------------------- The deployed app is a single-process Panel/Bokeh server: a blocking call on @@ -40,18 +52,24 @@ from __future__ import annotations import functools +import os import threading from dataclasses import dataclass from pathlib import Path import numpy as np from brainglobe_atlasapi import BrainGlobeAtlas -from brainglobe_atlasapi.descriptors import V3_ANNOTATION_NAME, V3_ATLAS_ROOTDIR +from brainglobe_atlasapi.descriptors import ( + ANNOTATION_DTYPE, + V3_ANNOTATION_NAME, + V3_ATLAS_ROOTDIR, +) from brainglobe_atlasapi.list_atlases import ( get_all_atlases_lastversions, get_downloaded_atlases, ) +from pixelmap.anatomy import _memory from pixelmap.anatomy._registry_snapshot import REGISTRY_SNAPSHOT _DEFAULT_ATLAS = "allen_mouse_25um" @@ -96,10 +114,76 @@ def ensure_downloaded(name: str = _DEFAULT_ATLAS) -> None: a single place so callers can push it onto a worker thread — which the GUI does, because on the server the calling thread also serves every other session. A no-op once the atlas is local. + + Raises :class:`AtlasTooLarge` before fetching anything bulky if the + decoded volume would not fit the process's memory cap. """ + check_fits(name) _ = get_atlas(name).annotation +class AtlasTooLarge(RuntimeError): + """The atlas's annotation volume exceeds what this process may decode.""" + + def __init__(self, name: str, nbytes: int, limit: int): + self.name, self.nbytes, self.limit = name, nbytes, limit + super().__init__( + f"{name}'s annotation is {nbytes / 2**20:,.0f} MB decoded; this " + f"server allows at most {limit / 2**20:,.0f} MB per atlas" + ) + + +def annotation_nbytes(name: str = _DEFAULT_ATLAS) -> int | None: + """Decoded size of the atlas's annotation array, from metadata alone. + + brainglobe publishes ``shape`` in the manifest and stores annotations as + ``uint32``, so the size is known without fetching a chunk. ``None`` when + the atlas carries no shape (test doubles), which callers treat as "no + opinion" rather than "fits". + """ + shape = getattr(get_atlas(name), "shape", None) + if shape is None: + return None + return int(np.prod([int(s) for s in shape])) * np.dtype(ANNOTATION_DTYPE).itemsize + + +# What the rest of the process needs regardless of atlas: the Bokeh/Panel +# baseline (~300 MB measured) plus headroom for sessions and the metadata +# caches. +_PROCESS_BASELINE_BYTES = 512 * 2**20 + + +def max_annotation_bytes() -> int | None: + """The largest annotation volume this process may decode, or ``None``. + + ``PIXELMAP_MAX_ATLAS_MB`` wins. Otherwise the cgroup memory cap minus the + process baseline: what's left is what one decoded volume may occupy. + ``None`` -- no cap, nothing refused -- outside a container. + """ + env = os.environ.get("PIXELMAP_MAX_ATLAS_MB") + if env: + return int(float(env) * 2**20) + limit = _memory.cgroup_memory_limit_bytes() + if limit is None: + return None + return max(limit - _PROCESS_BASELINE_BYTES, 0) + + +def check_fits(name: str = _DEFAULT_ATLAS) -> None: + """Raise :class:`AtlasTooLarge` if ``name`` cannot be decoded under the cap. + + Needs the manifest, which :func:`get_atlas` fetches on a cold cache, so + call it where a small network round-trip is acceptable (the worker thread + that would download the atlas anyway) or after :func:`is_downloaded`. + """ + limit = max_annotation_bytes() + if limit is None: + return + nbytes = annotation_nbytes(name) + if nbytes is not None and nbytes > limit: + raise AtlasTooLarge(name, nbytes, limit) + + @functools.lru_cache(maxsize=1) def list_atlases() -> list[str]: """Every atlas in the brainglobe registry, not just the downloaded ones. @@ -445,6 +529,7 @@ def canonical_annotation(atlas_name: str): non-Allen orientations work. For an already-``asr`` atlas this is a no-op (identity transpose, no flips), so Allen behavior is unchanged. """ + check_fits(atlas_name) atlas = get_atlas(atlas_name) axes = anatomical_axes(atlas) order = (axes["AP"].array_axis, axes["DV"].array_axis, axes["ML"].array_axis) diff --git a/pixelmap/gui/gui.py b/pixelmap/gui/gui.py index b6b3d9d..c8b3a39 100644 --- a/pixelmap/gui/gui.py +++ b/pixelmap/gui/gui.py @@ -1246,6 +1246,13 @@ async def _on_compute_anatomy_click(self, event): self.compute_anatomy_button.disabled = True try: await asyncio.to_thread(anatomy_atlas.ensure_downloaded, name) + except anatomy_atlas.AtlasTooLarge as exc: + # Known from the manifest before any chunk was fetched: refusing + # here is what keeps a 4.8 GB atlas from OOM-killing the server. + print(f"Atlas refused: {exc}") + self.compute_anatomy_button.name = "Download & compute atlas 🧠 ⏳" + _notify("error", f"{exc}. Pick a coarser resolution of this atlas.") + return except Exception as exc: print(f"Atlas download failed: {exc}") self.compute_anatomy_button.name = "Download & compute atlas 🧠 ⏳" @@ -1288,6 +1295,14 @@ def compute_anatomy_overlay(self): atlas_name=atlas_name, step_um=25.0, ) + except anatomy_atlas.AtlasTooLarge as exc: + # The chunks are on disk (downloaded by an earlier, larger deploy + # or another tool) but decoding them would exceed the memory cap. + print(f"Atlas refused: {exc}") + self.compute_anatomy_button.name = "Compute anatomical overlay 🧠" + self.compute_anatomy_button.disabled = False + _notify("error", f"{exc}. Pick a coarser resolution of this atlas.") + return except Exception as exc: print(f"Anatomy lookup failed: {exc}") self.compute_anatomy_button.name = "Compute anatomical overlay 🧠" diff --git a/tests/test_anatomy_atlas.py b/tests/test_anatomy_atlas.py index 504763d..24f1e82 100644 --- a/tests/test_anatomy_atlas.py +++ b/tests/test_anatomy_atlas.py @@ -13,6 +13,7 @@ import numpy as np import pytest +from pixelmap.anatomy import _memory from pixelmap.anatomy import atlas as atlas_module from pixelmap.anatomy import regions as regions_module from pixelmap.anatomy._registry_snapshot import REGISTRY_SNAPSHOT @@ -388,6 +389,97 @@ def annotation(self): assert reads == ["some_atlas"] +def _sized_atlas_cls(shape): + """A metadata-only fake: ``shape`` is published, ``annotation`` must not + be read -- the size gate has to decide from the manifest alone.""" + + class _A: + def __init__(self, name, **_kwargs): + self.name = name + self.orientation = "asr" + self.resolution = (25.0, 25.0, 25.0) + self.shape = shape + self.structures = {} + + @property + def annotation(self): + raise AssertionError("a refused atlas must never be decoded") + + return _A + + +class TestSizeGate: + """Atlases that cannot fit the memory cap are refused before any decode. + + allen_mouse_10um is a 1320x800x1140 uint32 array: 4.8 GB. Under a 3 GB + container cap that decode is an OOM kill, a Docker restart, and another + kill the next time anyone picks it. The manifest's ``shape`` says so + before a chunk is fetched. + """ + + ALLEN_10UM = (1320, 800, 1140) + + def test_size_comes_from_the_manifest_shape(self, monkeypatch): + monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _sized_atlas_cls(self.ALLEN_10UM)) + assert atlas_module.annotation_nbytes("x") == 1320 * 800 * 1140 * 4 + + def test_no_shape_means_no_opinion(self, monkeypatch): + monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _FakeAtlas) + assert atlas_module.annotation_nbytes("x") is None + monkeypatch.setattr(atlas_module, "max_annotation_bytes", lambda: 1) + atlas_module.check_fits("x") # no shape -> not refused + + def test_env_override_sets_the_limit(self, monkeypatch): + monkeypatch.setenv("PIXELMAP_MAX_ATLAS_MB", "1024") + assert atlas_module.max_annotation_bytes() == 1024 * 2**20 + + def test_limit_is_the_cgroup_cap_minus_the_process_baseline(self, monkeypatch, tmp_path): + monkeypatch.delenv("PIXELMAP_MAX_ATLAS_MB", raising=False) + f = tmp_path / "memory.max" + f.write_text("3221225472\n") # 3 GB `deploy` limit + monkeypatch.setattr(_memory, "_CGROUP_LIMIT_FILES", (str(f),)) + assert atlas_module.max_annotation_bytes() == 3221225472 - 512 * 2**20 + + def test_no_cap_means_nothing_is_refused(self, monkeypatch, tmp_path): + monkeypatch.delenv("PIXELMAP_MAX_ATLAS_MB", raising=False) + monkeypatch.setattr(_memory, "_CGROUP_LIMIT_FILES", (str(tmp_path / "missing"),)) + assert atlas_module.max_annotation_bytes() is None + monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _sized_atlas_cls(self.ALLEN_10UM)) + atlas_module.check_fits("x") + + def test_ensure_downloaded_refuses_before_fetching(self, monkeypatch): + monkeypatch.setenv("PIXELMAP_MAX_ATLAS_MB", "2560") + monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _sized_atlas_cls(self.ALLEN_10UM)) + with pytest.raises(atlas_module.AtlasTooLarge) as info: + atlas_module.ensure_downloaded("allen_mouse_10um") + assert info.value.name == "allen_mouse_10um" + assert "4,592 MB" in str(info.value) and "2,560 MB" in str(info.value) + + def test_every_decode_path_is_gated(self, monkeypatch): + # An atlas whose chunks are already on disk is decoded by + # canonical_annotation, not ensure_downloaded; it must refuse too. + monkeypatch.setenv("PIXELMAP_MAX_ATLAS_MB", "0.5") + monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _sized_atlas_cls((64, 64, 64))) + with pytest.raises(atlas_module.AtlasTooLarge): + atlas_module.canonical_annotation("x") + with pytest.raises(atlas_module.AtlasTooLarge): + atlas_module.lookup_regions("x", np.zeros((1, 3))) + + def test_an_atlas_that_fits_is_untouched(self, monkeypatch): + monkeypatch.setenv("PIXELMAP_MAX_ATLAS_MB", "2560") + monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _sized_atlas_cls((528, 320, 456))) + atlas_module.check_fits("allen_mouse_25um") # 308 MB: fine + + def test_cgroup_limit_parsing(self, tmp_path): + v2 = tmp_path / "memory.max"; v2.write_text("max\n") + v1 = tmp_path / "limit"; v1.write_text(str(2**63 - 4096) + "\n") + real = tmp_path / "real"; real.write_text("2147483648\n") + assert _memory.cgroup_memory_limit_bytes((str(v2),)) is None + assert _memory.cgroup_memory_limit_bytes((str(v1),)) is None + assert _memory.cgroup_memory_limit_bytes((str(tmp_path / "missing"),)) is None + assert _memory.cgroup_memory_limit_bytes((str(real),)) == 2**31 + + class TestAtlasesAreReleased: """Atlas objects must not outlive ``get_atlas``'s cache. diff --git a/tests/test_gui_overlays.py b/tests/test_gui_overlays.py index 1324ec7..16e2bbf 100644 --- a/tests/test_gui_overlays.py +++ b/tests/test_gui_overlays.py @@ -21,6 +21,7 @@ from pixelmap.anatomy import atlas as atlas_module from pixelmap.anatomy import regions as regions_module +from pixelmap.gui import gui as gui_module from pixelmap.gui.gui import ChannelmapGUI from pixelmap.types import Electrode @@ -679,6 +680,48 @@ def _boom(name): assert gui.compute_anatomy_button.disabled is False assert gui.compute_anatomy_button.name == "Download & compute atlas 🧠 ⏳" + def test_an_atlas_too_large_for_the_server_is_refused_not_downloaded( + self, served, monkeypatch + ): + """The manifest says how big the decoded volume is; a 4.8 GB atlas on + a 3 GB container is refused with a message, not OOM-killed.""" + gui, session = served + monkeypatch.setattr(atlas_module, "is_downloaded", lambda name: False) + notices = [] + monkeypatch.setattr(gui_module, "_notify", lambda level, msg, **kw: notices.append((level, msg))) + + def _too_big(name): + raise atlas_module.AtlasTooLarge(name, 4_800 * 2**20, 2_560 * 2**20) + + monkeypatch.setattr(atlas_module, "ensure_downloaded", _too_big) + + asyncio.run(session.with_document_locked(gui._on_compute_anatomy_click, None)) + + assert gui.compute_anatomy_button.disabled is False + assert gui.compute_anatomy_button.name == "Download & compute atlas 🧠 ⏳" + assert len(gui.region_band_source.data["acronym"]) == 0, "nothing was computed" + assert notices and notices[0][0] == "error" + assert "4,800 MB" in notices[0][1] and "2,560 MB" in notices[0][1] + + def test_a_too_large_atlas_already_on_disk_is_refused_at_compute( + self, served, monkeypatch + ): + gui, _session = served + monkeypatch.setattr(atlas_module, "is_downloaded", lambda name: True) + notices = [] + monkeypatch.setattr(gui_module, "_notify", lambda level, msg, **kw: notices.append((level, msg))) + monkeypatch.setattr( + atlas_module, "max_annotation_bytes", lambda: 1, # nothing fits + ) + monkeypatch.setattr(atlas_module, "annotation_nbytes", lambda name: 2) + atlas_module.canonical_annotation.cache_clear() + + gui.compute_anatomy_overlay() + + assert gui.compute_anatomy_button.disabled is False + assert len(gui.region_band_source.data["acronym"]) == 0 + assert notices and notices[0][0] == "error" + def test_no_unlocked_async_gui_callbacks(): """Every async GUI callback needs the same treatment.