Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions env.template
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,13 @@ ADDRESS=localhost
# sized for N x (base memory + atlases).
NUM_PROCS=1

# Largest atlas (decoded annotation, in MB) a user may compute with. Unset,
# it is the container's cgroup memory limit minus 512 MB for the process
# itself, and unlimited outside a container. Larger atlases are refused
# with a message instead of OOM-killing the server. allen_mouse_25um is
# 308 MB, whs_sd_rat_39um 1,074 MB, any 10um mouse atlas 4,592 MB.
# PIXELMAP_MAX_ATLAS_MB=2560

# Visitor analytics configuration
ANALYTICS_DB_PATH=/app/output/visitor_analytics.sqlite3
ANALYTICS_TRUST_PROXY_HEADERS=false
Expand Down
69 changes: 69 additions & 0 deletions pixelmap/anatomy/_memory.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
"""Process-memory helpers for the atlas layer.

Two facts about the deployed server that the atlas code has to reason
about, kept here so that every caller reasons about them the same way:

* The container runs under a cgroup memory cap (Compose's
``deploy.resources.limits.memory``), and that cap is enforced on RSS.
:func:`cgroup_memory_limit_bytes` reads it, so cache budgets and size
gates can be derived from the real limit instead of guessed.
* Dropping the last reference to a large array does not lower RSS. glibc
keeps freed pages in its arenas for reuse (``MALLOC_ARENA_MAX`` caps how
many arenas there are, not what they retain). Measured after clearing
every atlas cache: 1,188 MB RSS, then 211 MB after ``malloc_trim``. An
eviction that doesn't trim is one the OOM killer never sees, which is
what :func:`release_freed_memory` is for.
"""

from __future__ import annotations

import ctypes
import gc
from pathlib import Path

# Where a cgroup exposes the memory cap this process runs under -- v2 first,
# then v1.
_CGROUP_LIMIT_FILES = (
"/sys/fs/cgroup/memory.max",
"/sys/fs/cgroup/memory/memory.limit_in_bytes",
)


def cgroup_memory_limit_bytes(paths=None) -> int | None:
"""The container's memory cap, or ``None`` when there isn't one."""
for path in _CGROUP_LIMIT_FILES if paths is None else paths:
try:
raw = Path(path).read_text().strip()
except OSError:
continue
if raw == "max":
return None
try:
limit = int(raw)
except ValueError:
continue
# cgroup v1 reports a huge number (~2**63) for "unlimited".
if limit <= 0 or limit >= 1 << 60:
return None
return limit
return None


_libc = None


def release_freed_memory() -> None:
"""Return freed heap pages to the OS.

Collects garbage first so that reference cycles holding large arrays are
actually freed, then asks glibc to trim. A no-op on platforms without
``malloc_trim`` (macOS, musl).
"""
global _libc
gc.collect()
try:
if _libc is None:
_libc = ctypes.CDLL("libc.so.6")
_libc.malloc_trim(0)
except (OSError, AttributeError):
pass
87 changes: 86 additions & 1 deletion pixelmap/anatomy/atlas.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,18 @@
~9 MB of compressed chunks, against ~1.3 GB of tiffs under v2. That is what
makes baking them into the Docker image cheap and reliable.

Why some atlases are refused outright
-------------------------------------
The annotation is one dense ``uint32`` array: allen_mouse_25um is 308 MB,
whs_sd_rat_39um 1.07 GB, and any of the 10 µm mouse atlases 4.8 GB. On a
container capped at 3 GB the last of those cannot be decoded at all -- the
process is OOM-killed, Docker restarts it, and the next user who picks the
same atlas kills it again. The array's size is known from the manifest
(``shape``) before a single chunk is fetched, so :func:`check_fits` compares
it with the cap and raises :class:`AtlasTooLarge` first; the GUI turns that
into a message instead of a dead server. ``PIXELMAP_MAX_ATLAS_MB`` overrides
the derived limit; with no cgroup cap and no override nothing is refused.

Why the network never runs inline
---------------------------------
The deployed app is a single-process Panel/Bokeh server: a blocking call on
Expand All @@ -40,18 +52,24 @@
from __future__ import annotations

import functools
import os
import threading
from dataclasses import dataclass
from pathlib import Path

import numpy as np
from brainglobe_atlasapi import BrainGlobeAtlas
from brainglobe_atlasapi.descriptors import V3_ANNOTATION_NAME, V3_ATLAS_ROOTDIR
from brainglobe_atlasapi.descriptors import (
ANNOTATION_DTYPE,
V3_ANNOTATION_NAME,
V3_ATLAS_ROOTDIR,
)
from brainglobe_atlasapi.list_atlases import (
get_all_atlases_lastversions,
get_downloaded_atlases,
)

from pixelmap.anatomy import _memory
from pixelmap.anatomy._registry_snapshot import REGISTRY_SNAPSHOT

_DEFAULT_ATLAS = "allen_mouse_25um"
Expand Down Expand Up @@ -96,10 +114,76 @@ def ensure_downloaded(name: str = _DEFAULT_ATLAS) -> None:
a single place so callers can push it onto a worker thread — which the GUI
does, because on the server the calling thread also serves every other
session. A no-op once the atlas is local.

Raises :class:`AtlasTooLarge` before fetching anything bulky if the
decoded volume would not fit the process's memory cap.
"""
check_fits(name)
_ = get_atlas(name).annotation


class AtlasTooLarge(RuntimeError):
"""The atlas's annotation volume exceeds what this process may decode."""

def __init__(self, name: str, nbytes: int, limit: int):
self.name, self.nbytes, self.limit = name, nbytes, limit
super().__init__(
f"{name}'s annotation is {nbytes / 2**20:,.0f} MB decoded; this "
f"server allows at most {limit / 2**20:,.0f} MB per atlas"
)


def annotation_nbytes(name: str = _DEFAULT_ATLAS) -> int | None:
"""Decoded size of the atlas's annotation array, from metadata alone.

brainglobe publishes ``shape`` in the manifest and stores annotations as
``uint32``, so the size is known without fetching a chunk. ``None`` when
the atlas carries no shape (test doubles), which callers treat as "no
opinion" rather than "fits".
"""
shape = getattr(get_atlas(name), "shape", None)
if shape is None:
return None
return int(np.prod([int(s) for s in shape])) * np.dtype(ANNOTATION_DTYPE).itemsize


# What the rest of the process needs regardless of atlas: the Bokeh/Panel
# baseline (~300 MB measured) plus headroom for sessions and the metadata
# caches.
_PROCESS_BASELINE_BYTES = 512 * 2**20


def max_annotation_bytes() -> int | None:
"""The largest annotation volume this process may decode, or ``None``.

``PIXELMAP_MAX_ATLAS_MB`` wins. Otherwise the cgroup memory cap minus the
process baseline: what's left is what one decoded volume may occupy.
``None`` -- no cap, nothing refused -- outside a container.
"""
env = os.environ.get("PIXELMAP_MAX_ATLAS_MB")
if env:
return int(float(env) * 2**20)
limit = _memory.cgroup_memory_limit_bytes()
if limit is None:
return None
return max(limit - _PROCESS_BASELINE_BYTES, 0)


def check_fits(name: str = _DEFAULT_ATLAS) -> None:
"""Raise :class:`AtlasTooLarge` if ``name`` cannot be decoded under the cap.

Needs the manifest, which :func:`get_atlas` fetches on a cold cache, so
call it where a small network round-trip is acceptable (the worker thread
that would download the atlas anyway) or after :func:`is_downloaded`.
"""
limit = max_annotation_bytes()
if limit is None:
return
nbytes = annotation_nbytes(name)
if nbytes is not None and nbytes > limit:
raise AtlasTooLarge(name, nbytes, limit)


@functools.lru_cache(maxsize=1)
def list_atlases() -> list[str]:
"""Every atlas in the brainglobe registry, not just the downloaded ones.
Expand Down Expand Up @@ -445,6 +529,7 @@ def canonical_annotation(atlas_name: str):
non-Allen orientations work. For an already-``asr`` atlas this is a no-op
(identity transpose, no flips), so Allen behavior is unchanged.
"""
check_fits(atlas_name)
atlas = get_atlas(atlas_name)
axes = anatomical_axes(atlas)
order = (axes["AP"].array_axis, axes["DV"].array_axis, axes["ML"].array_axis)
Expand Down
15 changes: 15 additions & 0 deletions pixelmap/gui/gui.py
Original file line number Diff line number Diff line change
Expand Up @@ -1246,6 +1246,13 @@ async def _on_compute_anatomy_click(self, event):
self.compute_anatomy_button.disabled = True
try:
await asyncio.to_thread(anatomy_atlas.ensure_downloaded, name)
except anatomy_atlas.AtlasTooLarge as exc:
# Known from the manifest before any chunk was fetched: refusing
# here is what keeps a 4.8 GB atlas from OOM-killing the server.
print(f"Atlas refused: {exc}")
self.compute_anatomy_button.name = "Download & compute atlas 🧠 ⏳"
_notify("error", f"{exc}. Pick a coarser resolution of this atlas.")
return
except Exception as exc:
print(f"Atlas download failed: {exc}")
self.compute_anatomy_button.name = "Download & compute atlas 🧠 ⏳"
Expand Down Expand Up @@ -1288,6 +1295,14 @@ def compute_anatomy_overlay(self):
atlas_name=atlas_name,
step_um=25.0,
)
except anatomy_atlas.AtlasTooLarge as exc:
# The chunks are on disk (downloaded by an earlier, larger deploy
# or another tool) but decoding them would exceed the memory cap.
print(f"Atlas refused: {exc}")
self.compute_anatomy_button.name = "Compute anatomical overlay 🧠"
self.compute_anatomy_button.disabled = False
_notify("error", f"{exc}. Pick a coarser resolution of this atlas.")
return
except Exception as exc:
print(f"Anatomy lookup failed: {exc}")
self.compute_anatomy_button.name = "Compute anatomical overlay 🧠"
Expand Down
92 changes: 92 additions & 0 deletions tests/test_anatomy_atlas.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import numpy as np
import pytest

from pixelmap.anatomy import _memory
from pixelmap.anatomy import atlas as atlas_module
from pixelmap.anatomy import regions as regions_module
from pixelmap.anatomy._registry_snapshot import REGISTRY_SNAPSHOT
Expand Down Expand Up @@ -388,6 +389,97 @@ def annotation(self):
assert reads == ["some_atlas"]


def _sized_atlas_cls(shape):
"""A metadata-only fake: ``shape`` is published, ``annotation`` must not
be read -- the size gate has to decide from the manifest alone."""

class _A:
def __init__(self, name, **_kwargs):
self.name = name
self.orientation = "asr"
self.resolution = (25.0, 25.0, 25.0)
self.shape = shape
self.structures = {}

@property
def annotation(self):
raise AssertionError("a refused atlas must never be decoded")

return _A


class TestSizeGate:
"""Atlases that cannot fit the memory cap are refused before any decode.

allen_mouse_10um is a 1320x800x1140 uint32 array: 4.8 GB. Under a 3 GB
container cap that decode is an OOM kill, a Docker restart, and another
kill the next time anyone picks it. The manifest's ``shape`` says so
before a chunk is fetched.
"""

ALLEN_10UM = (1320, 800, 1140)

def test_size_comes_from_the_manifest_shape(self, monkeypatch):
monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _sized_atlas_cls(self.ALLEN_10UM))
assert atlas_module.annotation_nbytes("x") == 1320 * 800 * 1140 * 4

def test_no_shape_means_no_opinion(self, monkeypatch):
monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _FakeAtlas)
assert atlas_module.annotation_nbytes("x") is None
monkeypatch.setattr(atlas_module, "max_annotation_bytes", lambda: 1)
atlas_module.check_fits("x") # no shape -> not refused

def test_env_override_sets_the_limit(self, monkeypatch):
monkeypatch.setenv("PIXELMAP_MAX_ATLAS_MB", "1024")
assert atlas_module.max_annotation_bytes() == 1024 * 2**20

def test_limit_is_the_cgroup_cap_minus_the_process_baseline(self, monkeypatch, tmp_path):
monkeypatch.delenv("PIXELMAP_MAX_ATLAS_MB", raising=False)
f = tmp_path / "memory.max"
f.write_text("3221225472\n") # 3 GB `deploy` limit
monkeypatch.setattr(_memory, "_CGROUP_LIMIT_FILES", (str(f),))
assert atlas_module.max_annotation_bytes() == 3221225472 - 512 * 2**20

def test_no_cap_means_nothing_is_refused(self, monkeypatch, tmp_path):
monkeypatch.delenv("PIXELMAP_MAX_ATLAS_MB", raising=False)
monkeypatch.setattr(_memory, "_CGROUP_LIMIT_FILES", (str(tmp_path / "missing"),))
assert atlas_module.max_annotation_bytes() is None
monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _sized_atlas_cls(self.ALLEN_10UM))
atlas_module.check_fits("x")

def test_ensure_downloaded_refuses_before_fetching(self, monkeypatch):
monkeypatch.setenv("PIXELMAP_MAX_ATLAS_MB", "2560")
monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _sized_atlas_cls(self.ALLEN_10UM))
with pytest.raises(atlas_module.AtlasTooLarge) as info:
atlas_module.ensure_downloaded("allen_mouse_10um")
assert info.value.name == "allen_mouse_10um"
assert "4,592 MB" in str(info.value) and "2,560 MB" in str(info.value)

def test_every_decode_path_is_gated(self, monkeypatch):
# An atlas whose chunks are already on disk is decoded by
# canonical_annotation, not ensure_downloaded; it must refuse too.
monkeypatch.setenv("PIXELMAP_MAX_ATLAS_MB", "0.5")
monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _sized_atlas_cls((64, 64, 64)))
with pytest.raises(atlas_module.AtlasTooLarge):
atlas_module.canonical_annotation("x")
with pytest.raises(atlas_module.AtlasTooLarge):
atlas_module.lookup_regions("x", np.zeros((1, 3)))

def test_an_atlas_that_fits_is_untouched(self, monkeypatch):
monkeypatch.setenv("PIXELMAP_MAX_ATLAS_MB", "2560")
monkeypatch.setattr(atlas_module, "BrainGlobeAtlas", _sized_atlas_cls((528, 320, 456)))
atlas_module.check_fits("allen_mouse_25um") # 308 MB: fine

def test_cgroup_limit_parsing(self, tmp_path):
v2 = tmp_path / "memory.max"; v2.write_text("max\n")
v1 = tmp_path / "limit"; v1.write_text(str(2**63 - 4096) + "\n")
real = tmp_path / "real"; real.write_text("2147483648\n")
assert _memory.cgroup_memory_limit_bytes((str(v2),)) is None
assert _memory.cgroup_memory_limit_bytes((str(v1),)) is None
assert _memory.cgroup_memory_limit_bytes((str(tmp_path / "missing"),)) is None
assert _memory.cgroup_memory_limit_bytes((str(real),)) == 2**31


class TestAtlasesAreReleased:
"""Atlas objects must not outlive ``get_atlas``'s cache.

Expand Down
Loading
Loading