Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,18 @@ services:
restart: unless-stopped
# Cap container memory so the *container* gets OOM-killed and restarted
# by Docker's restart policy, not the host. Size as roughly:
# ~300 MB base + <= largest atlas you allow users to download + ~6 MB
# ~300 MB base + 2 x the largest atlas you allow users to compute with
# (decoding holds every decompressed chunk *and* the assembled array at
# once: whs_sd_rat_39um is a 1.07 GB array and peaks at +2.1 GB) + ~6 MB
# per open browser tab.
#
# The app keeps decoded atlases in a cache budgeted at half this limit
# (read from the cgroup), so a larger limit also means fewer re-decodes
# when users switch atlases. PIXELMAP_ATLAS_CACHE_MB overrides the budget.
# deploy:
# resources:
# limits:
# memory: 2g
# memory: 3g

volumes:
# Persists brainglobe atlas downloads across container restarts.
Expand Down
7 changes: 5 additions & 2 deletions docs/getting-started.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,11 @@ For production deployments, use **Docker Compose** — see the included `docker-
**Memory & healthcheck**: point the healthcheck at `/liveness`, never `/app`
— `/app` builds a full Bokeh session on every probe. Set a container memory
limit (`deploy.resources.limits.memory` in Compose) so the container, not
the host, gets restarted under memory pressure. Keep `NUM_PROCS=1` unless
RAM is sized for multiple copies of every cache. Mount the
the host, gets restarted under memory pressure; size it at ~300 MB plus
twice the largest atlas users may compute with (decoding an atlas briefly
needs ~2x its array). Decoded atlases stay in a cache budgeted at half
that limit (`PIXELMAP_ATLAS_CACHE_MB` overrides it). Keep `NUM_PROCS=1`
unless RAM is sized for multiple copies of every cache. Mount the
`brainglobe_cache` volume so downloaded atlases persist across restarts.
RSS is logged to stdout every 60 s.
:::
Expand Down
5 changes: 5 additions & 0 deletions env.template
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,11 @@ ADDRESS=localhost
# sized for N x (base memory + atlases).
NUM_PROCS=1

# How many MB of decoded atlas volumes each process keeps resident. Unset,
# it is half the container's cgroup memory limit, or 2048 outside a
# container. Least-recently-used atlases are evicted past this.
# PIXELMAP_ATLAS_CACHE_MB=1536

# Visitor analytics configuration
ANALYTICS_DB_PATH=/app/output/visitor_analytics.sqlite3
ANALYTICS_TRUST_PROXY_HEADERS=false
Expand Down
69 changes: 69 additions & 0 deletions pixelmap/anatomy/_memory.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
"""Process-memory helpers for the atlas layer.

Two facts about the deployed server that the atlas code has to reason
about, kept here so that every caller reasons about them the same way:

* The container runs under a cgroup memory cap (Compose's
``deploy.resources.limits.memory``), and that cap is enforced on RSS.
:func:`cgroup_memory_limit_bytes` reads it, so cache budgets and size
gates can be derived from the real limit instead of guessed.
* Dropping the last reference to a large array does not lower RSS. glibc
keeps freed pages in its arenas for reuse (``MALLOC_ARENA_MAX`` caps how
many arenas there are, not what they retain). Measured after clearing
every atlas cache: 1,188 MB RSS, then 211 MB after ``malloc_trim``. An
eviction that doesn't trim is one the OOM killer never sees, which is
what :func:`release_freed_memory` is for.
"""

from __future__ import annotations

import ctypes
import gc
from pathlib import Path

# Where a cgroup exposes the memory cap this process runs under -- v2 first,
# then v1.
_CGROUP_LIMIT_FILES = (
"/sys/fs/cgroup/memory.max",
"/sys/fs/cgroup/memory/memory.limit_in_bytes",
)


def cgroup_memory_limit_bytes(paths=None) -> int | None:
"""The container's memory cap, or ``None`` when there isn't one."""
for path in _CGROUP_LIMIT_FILES if paths is None else paths:
try:
raw = Path(path).read_text().strip()
except OSError:
continue
if raw == "max":
return None
try:
limit = int(raw)
except ValueError:
continue
# cgroup v1 reports a huge number (~2**63) for "unlimited".
if limit <= 0 or limit >= 1 << 60:
return None
return limit
return None


_libc = None


def release_freed_memory() -> None:
"""Return freed heap pages to the OS.

Collects garbage first so that reference cycles holding large arrays are
actually freed, then asks glibc to trim. A no-op on platforms without
``malloc_trim`` (macOS, musl).
"""
global _libc
gc.collect()
try:
if _libc is None:
_libc = ctypes.CDLL("libc.so.6")
_libc.malloc_trim(0)
except (OSError, AttributeError):
pass
195 changes: 186 additions & 9 deletions pixelmap/anatomy/atlas.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@

Why a wrapper:

* Cache atlas instances per-process so repeated lookups don't re-load the
annotation volume (hundreds of MB once decompressed).
* Cache atlas instances per-process so repeated lookups don't re-read the
atlas metadata, and hold the decompressed annotation volumes (hundreds of
MB to several GB each) in one byte-bounded cache -- see "Who owns the
annotation volumes" below.
* Expose a tiny ``RegionInfo`` record so the rest of PixelMap doesn't
depend on brainglobe's object model.
* Keep every network call off the caller's thread, because on the hosted
Expand All @@ -26,6 +28,28 @@
~9 MB of compressed chunks, against ~1.3 GB of tiffs under v2. That is what
makes baking them into the Docker image cheap and reliable.

Who owns the annotation volumes
-------------------------------
Exactly one thing: :data:`_VOLUMES`, a byte-budgeted LRU keyed by atlas name,
read through :func:`canonical_annotation`. Everything else holds volumes only
transiently. This is deliberate, and worth defending:

* An ``lru_cache`` counts *entries*, not bytes. ``maxsize=4`` is four
38 MB atlases or four 4.8 GB ones (allen_mouse_10um), so it never related to
the container's memory limit. The budget here is in bytes and defaults to
half the cgroup limit (``PIXELMAP_ATLAS_CACHE_MB`` overrides it).
* Two caches with independent LRU orders leak. ``get_atlas`` and the old
``canonical_annotation`` each evicted on their own schedule; once one dropped
an atlas the other still pinned its array (a transposed *view* keeps the
base alive), and the next ``ensure_downloaded`` loaded a second copy.
brainglobe also caches the array on the atlas object itself, so
:func:`_take_annotation` detaches it and ``get_atlas`` only ever holds
cheap metadata objects.
* Dropping a reference is not freeing memory. glibc keeps freed pages in its
arenas -- after clearing every cache the process still sat at 1.2 GB RSS
until ``malloc_trim``. The container limit counts RSS, so every eviction
and every load ends with :func:`_memory.release_freed_memory`.

Why the network never runs inline
---------------------------------
The deployed app is a single-process Panel/Bokeh server: a blocking call on
Expand All @@ -40,7 +64,10 @@
from __future__ import annotations

import functools
import logging
import os
import threading
from collections import OrderedDict
from dataclasses import dataclass
from pathlib import Path

Expand All @@ -52,10 +79,13 @@
get_downloaded_atlases,
)

from pixelmap.anatomy import _memory
from pixelmap.anatomy._registry_snapshot import REGISTRY_SNAPSHOT

_DEFAULT_ATLAS = "allen_mouse_25um"

log = logging.getLogger(__name__)


@dataclass(frozen=True)
class RegionInfo:
Expand All @@ -81,10 +111,12 @@ def get_atlas(name: str = _DEFAULT_ATLAS):
neither is free on a cold cache — see :func:`ensure_downloaded` for
getting that cost off the event loop.

The cache is bounded so a server that sees many atlases doesn't hold every
annotation volume it has ever decoded. Nothing else may keep a strong
reference to an atlas object, or that bound stops meaning anything (see
:func:`_region_info_from_id`).
Objects here are metadata only -- manifest, structures table, an S3
handle -- a few MB each. The annotation volume is *not* kept on them:
:func:`_take_annotation` detaches it into :data:`_VOLUMES`, the one place
volumes live. Reading ``.annotation`` directly off one of these objects
re-materialises the array on the instance, so don't; go through
:func:`canonical_annotation`.
"""
return BrainGlobeAtlas(name, check_latest=False)

Expand All @@ -96,8 +128,13 @@ def ensure_downloaded(name: str = _DEFAULT_ATLAS) -> None:
a single place so callers can push it onto a worker thread — which the GUI
does, because on the server the calling thread also serves every other
session. A no-op once the atlas is local.

Loads into :data:`_VOLUMES` rather than merely touching ``.annotation``:
the compute that follows reads the canonical volume, so this is the copy
it will use, and touching the attribute instead would leave a second copy
cached on the atlas object.
"""
_ = get_atlas(name).annotation
canonical_annotation(name)


@functools.lru_cache(maxsize=1)
Expand Down Expand Up @@ -434,7 +471,6 @@ def anatomical_axes(atlas) -> dict[str, _AnatAxis]:
return axes


@functools.lru_cache(maxsize=4)
def canonical_annotation(atlas_name: str):
"""Return ``(annotation, resolution)`` reoriented to canonical ``(AP, DV, ML)``.

Expand All @@ -444,11 +480,20 @@ def canonical_annotation(atlas_name: str):
assumes this layout, so funnelling every atlas through here is what lets
non-Allen orientations work. For an already-``asr`` atlas this is a no-op
(identity transpose, no flips), so Allen behavior is unchanged.

Served from :data:`_VOLUMES`, the byte-bounded cache that is the only
long-lived owner of annotation arrays. Callers get the cached array
itself: treat it as read-only.
"""
return _VOLUMES.get(atlas_name)


def _load_canonical(atlas_name: str):
"""Materialise ``atlas_name``'s volume in canonical layout (uncached)."""
atlas = get_atlas(atlas_name)
axes = anatomical_axes(atlas)
order = (axes["AP"].array_axis, axes["DV"].array_axis, axes["ML"].array_axis)
arr = np.transpose(atlas.annotation, order)
arr = np.transpose(_take_annotation(atlas), order)
flip_axes = tuple(i for i, kind in enumerate(("AP", "DV", "ML")) if axes[kind].flip)
if flip_axes:
arr = np.flip(arr, axis=flip_axes)
Expand All @@ -458,6 +503,138 @@ def canonical_annotation(atlas_name: str):
return arr, res


def _take_annotation(atlas) -> np.ndarray:
"""Read the atlas's annotation and detach it from the atlas object.

brainglobe's ``annotation`` property caches the array on the instance
(``_annotation``) after the first read. Left there, ``get_atlas``'s cache
becomes a second owner of every volume, with its own eviction order -- the
duplicate-copy leak described in the module docstring. Clearing the slot
leaves the object metadata-only again; a later read would simply reload
from the local zarr chunks (no network, the chunks are cached).

Test doubles that expose ``annotation`` as a plain attribute have no slot
to clear, and the ``is`` check keeps this from touching anything else.
"""
arr = atlas.annotation
if getattr(atlas, "_annotation", None) is arr:
atlas._annotation = None
return arr


def _default_budget_bytes() -> int:
"""How many bytes of annotation volumes to keep resident.

``PIXELMAP_ATLAS_CACHE_MB`` wins. Otherwise half the cgroup limit: the
other half covers the Bokeh baseline (~300 MB) and the transient peak of
decoding the next volume (dask holds every decompressed chunk *and* the
assembled array at once, ~2x the array). Outside a container, 2 GB.
"""
env = os.environ.get("PIXELMAP_ATLAS_CACHE_MB")
if env:
return int(float(env) * 2**20)
limit = _memory.cgroup_memory_limit_bytes()
if limit is None:
return 2 * 2**30
return max(limit // 2, 256 * 2**20)


class _VolumeCache:
"""Byte-bounded LRU of canonical annotation volumes.

The sole long-lived owner of annotation arrays in the process. Evicts
least-recently-used volumes until the total fits ``budget_bytes``, then
trims the heap so the eviction actually lowers RSS. A single volume
larger than the whole budget still loads (evicting everything else) --
refusing it is a UI decision, not a cache one.

Loads run outside the lock so the Bokeh event loop can keep answering
lookups for a cached atlas while a worker thread decodes another one;
concurrent requests for the *same* atlas wait for the first load rather
than decoding a second copy.
"""

def __init__(self, budget_bytes: int | None = None):
self._budget = budget_bytes
self._entries: OrderedDict[str, tuple[np.ndarray, np.ndarray]] = OrderedDict()
self._lock = threading.Lock()
self._loading: dict[str, threading.Event] = {}

@property
def budget_bytes(self) -> int:
if self._budget is None:
self._budget = _default_budget_bytes()
return self._budget

def resident_bytes(self) -> int:
with self._lock:
return sum(arr.nbytes for arr, _ in self._entries.values())

def get(self, name: str):
while True:
with self._lock:
entry = self._entries.get(name)
if entry is not None:
self._entries.move_to_end(name)
return entry
pending = self._loading.get(name)
if pending is None:
pending = self._loading[name] = threading.Event()
break
# Someone else is decoding this atlas; wait, then re-check. If
# their load failed the entry is still absent and we try ourselves.
pending.wait()
try:
arr, res = _load_canonical(name)
except BaseException:
# Waiters must wake either way; with no entry present they retry.
with self._lock:
del self._loading[name]
pending.set()
raise
return self._insert(name, arr, res, pending)

def _insert(self, name, arr, res, pending):
evicted = []
with self._lock:
self._entries[name] = (arr, res)
self._entries.move_to_end(name)
total = sum(a.nbytes for a, _ in self._entries.values())
while total > self.budget_bytes and len(self._entries) > 1:
old, (old_arr, _) = self._entries.popitem(last=False)
total -= old_arr.nbytes
evicted.append((old, old_arr.nbytes))
del self._loading[name]
pending.set()
for old, nbytes in evicted:
log.info("atlas cache: evicted %s (%d MB) to stay under %d MB",
old, nbytes >> 20, self.budget_bytes >> 20)
# Every load leaves dask's decompressed chunks in the heap; every
# eviction leaves the array there. Either way, give it back now.
_memory.release_freed_memory()
return arr, res

def evict(self, name: str) -> None:
with self._lock:
dropped = self._entries.pop(name, None) is not None
if dropped:
_memory.release_freed_memory()

def clear(self) -> None:
with self._lock:
had = bool(self._entries)
self._entries.clear()
if had:
_memory.release_freed_memory()

# The lru_cache this replaced was cleared as ``canonical_annotation.cache_clear()``.
cache_clear = clear


_VOLUMES = _VolumeCache()
canonical_annotation.cache_clear = _VOLUMES.clear # type: ignore[attr-defined]


def volume_center_um(name: str = _DEFAULT_ATLAS) -> tuple[float, float, float]:
"""Geometric center of the atlas volume as ``(AP, ML, DV)`` µm.

Expand Down
Loading
Loading