From e45235bf02755a6fcc568651d8fdd23b2e1c87e3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 19 Sep 2026 23:53:46 +0200 Subject: [PATCH] caching: continuous cache eviction, fixes #183 The cache was only cleaned up by Store.open() and Store.close(), so reading a lot of data grew a size limited cache without bounds while the store was in use. For each cache namespace with a max_age or size limit, the Store now keeps an in-memory index (name, size, last use) and makes room before it puts an item into the cache. Items bigger than the size limit are not cached. The index is ordered by the last use by this store, so the LRU order does not depend on the atime for items used in this session (filesystems often only update the atime for the first read after a write). Shared caches: a store rescans a size limited namespace after it has put more than size / 4 bytes into it, so it also sees what other clients sharing the cache have added or evicted. Scanning a namespace the cache backend does not have yet is not an error. Co-Authored-By: Claude Fable 5.1 --- docs/store_caching.rst | 78 +++++++++--- src/borgstore/store.py | 186 ++++++++++++++++++++++----- tests/test_cache.py | 278 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 491 insertions(+), 51 deletions(-) diff --git a/docs/store_caching.rst b/docs/store_caching.rst index 8fca19b..025fd12 100644 --- a/docs/store_caching.rst +++ b/docs/store_caching.rst @@ -29,7 +29,8 @@ Each namespace configuration dictionary can have: The default is ``None`` (no age limit). - ``size``: optional maximum size in bytes. It sets a per-namespace cache size budget enforced by evicting least-recently-used items until the - namespace total size is within the configured budget. + namespace total size is within the configured budget. Items bigger than + ``size`` are not cached. Example:: @@ -58,16 +59,54 @@ Behavior - Cache keys are identical to primary backend keys (same nesting). - Soft-deleted items are cached under the same ``.del`` name as primary. - Soft delete/undelete renames cache entries as well. -- On ``Store.open()`` and ``Store.close()``, cache-enabled namespaces are scanned - to clean up the cache. Cleanup order per namespace is: +- Cache failures are non-fatal and logged as warnings. - 1. remove expired cache objects when ``max_age`` is configured, - 2. if ``size`` is configured, evict the least-recently-used remaining items - until the namespace total size is ``<= size``. +Eviction +-------- - Expired entries are always removed first, even if total size is already below - the ``size`` limit. -- Cache failures are non-fatal and logged as warnings. +For each namespace that has a ``max_age`` or ``size`` limit, the ``Store`` keeps +an in-memory index of the cached items (name, size, time of last use) while it +is opened. The index is ordered by the last use *by this store*: a cache hit, +putting an item into the cache or moving it counts as using it. + +- On ``Store.open()`` and ``Store.close()``, the namespace is scanned (listed) and + cleaned up. When scanning, items the store did not know yet are ordered by + their ``ItemInfo.atime``. +- Before an item is put into the cache (by ``store()`` or by a ``load()`` that + was a cache miss), room is made for it, so the namespace total size stays + ``<= size`` also while the store is in use. + +Cleanup order per namespace is: + +1. remove expired cache objects when ``max_age`` is configured, +2. if ``size`` is configured, evict the least-recently-used remaining items + until the namespace total size (plus the size of the item to put into the + cache) is ``<= size``. + +Expired entries are always removed first, even if total size is already below +the ``size`` limit. A cache hit never expires anything: an expired item that is +still in the cache is served (and that counts as using it). + +Shared caches +------------- + +Multiple clients (``Store`` instances, also in different processes) may use the +same cache at the same time, e.g. a cache directory shared by several +processes working with the same content-hash addressed data: + +- The posixfs backend stores items atomically, so a client never sees or + evicts an incompletely written cache item. +- If another client has evicted an item, that is a cache miss and the item + gets cached again. +- A client only knows what it has put into the cache itself and what it has + seen when it last scanned the namespace. Thus, a namespace with a ``size`` + limit is scanned again after the client has put more than ``size / 4`` bytes + into it. With N clients, the namespace total size can temporarily reach about + ``size * (1 + N / 4)``. +- Clients do not see each other's cache hits (see the ``atime`` limitation + below), so a client might evict an item another client frequently uses. +- If the clients use different limits for the same namespace, the smallest + limits win. Manual Cache Invalidation ------------------------- @@ -94,22 +133,25 @@ clients, or if cache corruption is suspected), you can use the Limitations ----------- -- Eviction by ``max_age`` or ``size`` is open-time and close-time only - (``Store.open()`` / ``Store.close()``), not continuous during - ``store()``/``load()`` operations. - No proactive cache validation/revalidation. - If an object is deleted in the primary backend by another client, the local cache will still have a stale object. -- ``max_age`` and LRU-by-``size`` depend on backend ``ItemInfo.atime`` support, - currently that is supported by ``posixfs`` and ``REST`` backends. +- For items a ``Store`` has not used itself since it was opened (items cached + in a previous session or by another client), ``max_age`` and LRU-by-``size`` + depend on backend ``ItemInfo.atime`` support, currently that is supported by + ``posixfs`` and ``REST`` backends. Filesystems often do not update the atime + for each read (e.g. ``relatime`` or ``noatime`` mounts), so it can be older + than the real last use. If ``atime`` is 0 (not implemented): - - using ``max_age`` would empty the cache on ``Store.open()`` or ``Store.close()`` - - using ``size`` would not work in LRU order, because order can't be - determined + - using ``max_age`` would remove these items from the cache when it is + scanned + - using ``size`` would not evict these items in LRU order, because their + order can't be determined - If a partial range ``load`` call for an object in a cached namespace causes a cache miss, the full object will be read from the primary backend and the - cache will be populated with the full object. + cache will be populated with the full object (if it is not bigger than + ``size``). Statistics ---------- diff --git a/src/borgstore/store.py b/src/borgstore/store.py index 1986361..6870517 100644 --- a/src/borgstore/store.py +++ b/src/borgstore/store.py @@ -11,7 +11,7 @@ """ from binascii import hexlify -from collections import Counter +from collections import Counter, OrderedDict from contextlib import contextmanager import enum from functools import wraps @@ -33,6 +33,10 @@ logger = logging.getLogger(__name__) +# a cache namespace with a size limit is rescanned after this store has inserted more than +# size / CACHE_RESCAN_DIVISOR bytes into it, see Store._cache_scan. +CACHE_RESCAN_DIVISOR = 4 + class CacheMode(enum.Enum): C_OFF = "off" @@ -57,6 +61,44 @@ class CachePolicy(NamedTuple): size: Optional[int] +class CacheIndex: + """ + In-memory view of one cache namespace that has a max_age or size limit. + + It tells the Store what to evict without listing the cache backend for each operation. + It is this store's view only: what other clients sharing the same cache add, use or + evict is only seen when the namespace is scanned, see Store._cache_scan. + """ + + def __init__(self, namespace: str, policy: CachePolicy): + self.namespace = namespace + self.policy = policy + # nested name -> (size, last access timestamp), least recently used entry first. + self.entries: OrderedDict = OrderedDict() + self.total = 0 # sum of the entries' sizes + self.inserted = 0 # bytes this store has put into the cache since the last scan + + def add(self, name: str, size: int, last_access: float) -> None: + """add (or replace) an entry as the most recently used one.""" + self.remove(name) + self.entries[name] = (size, last_access) + self.total += size + + def remove(self, name: str) -> None: + entry = self.entries.pop(name, None) + if entry is not None: + self.total -= entry[0] + + def replace_all(self, entries) -> None: + """replace the contents by entries, an iterable of (name, size, last_access).""" + self.entries = OrderedDict( + (name, (size, last_access)) + for name, size, last_access in sorted(entries, key=lambda entry: (entry[2], entry[0])) + ) + self.total = sum(size for size, _ in self.entries.values()) + self.inserted = 0 + + def get_backend(url, permissions=None, quota=None): """Parse backend URL and return a backend instance (or None).""" backend = get_file_backend(url, permissions=permissions, quota=quota) @@ -153,6 +195,8 @@ def __init__( if self.cache_backend is None: raise BackendURLInvalid(f"Invalid or unsupported Cache Backend URL: {cache_url}") self._cache_disabled = False + # namespace -> CacheIndex, only for namespaces with a max_age or size limit, only while opened. + self._cache_indexes: dict = {} self.cache_namespaces = [ entry for entry in sorted( @@ -207,6 +251,12 @@ def _cache_policy_for(self, name: str) -> CachePolicy: return policy return CachePolicy(mode=CacheMode.C_OFF, max_age=None, size=None) + def _cache_index_for(self, name: str) -> Optional[CacheIndex]: + for namespace, policy in self.cache_namespaces: + if name.startswith(namespace): + return self._cache_indexes.get(namespace) + return None + @_locked def set_levels(self, levels: dict, create: bool = False) -> None: if not levels or not isinstance(levels, dict): @@ -281,14 +331,20 @@ def open(self) -> None: logger.warning(f"borgstore: cache open failed, disabling cache: {err!r}") self._cache_disabled = True else: - self._cache_cleanup_expired() + self._cache_indexes = { + namespace: CacheIndex(namespace, policy) + for namespace, policy in self.cache_namespaces + if policy.max_age is not None or policy.size is not None + } + self._cache_cleanup() @_locked def close(self) -> None: self.backend.close() if self.cache_backend is not None: if not self._cache_disabled: - self._cache_cleanup_expired() + self._cache_cleanup() + self._cache_indexes = {} try: self.cache_backend.close() except Exception as err: @@ -425,10 +481,13 @@ def _cache_load(self, nested_name: str, *, size=None, offset=0) -> Optional[byte if self.cache_backend is None or self._cache_disabled: return None self._stats["cache_load_calls"] += 1 + index = self._cache_index_for(nested_name) try: value = self.cache_backend.load(nested_name, size=size, offset=offset) except ObjectNotFound: self._stats["cache_misses"] += 1 + if index is not None: + index.remove(nested_name) # another client has evicted it return None except Exception as err: logger.warning(f"borgstore: cache load failed for {nested_name!r}: {err!r}") @@ -436,6 +495,17 @@ def _cache_load(self, nested_name: str, *, size=None, offset=0) -> Optional[byte return None self._stats["cache_hits"] += 1 self._stats["cache_load_volume"] += len(value) + if index is not None: + entry = index.entries.get(nested_name) + if entry is not None: + item_size = entry[0] + else: + # another client has cached it. value might be only a part of the item. + try: + item_size = self.cache_backend.info(nested_name).size + except Exception: + item_size = len(value) + index.add(nested_name, item_size, time.time()) return value @_locked @@ -478,6 +548,19 @@ def load(self, name: str, *, size=None, offset=0, deleted=False) -> bytes: def _cache_store(self, nested_name: str, value: StoreValue) -> None: if self.cache_backend is None or self._cache_disabled: return + index = self._cache_index_for(nested_name) + if index is not None: + size_limit = index.policy.size + if size_limit is not None and len(value) > size_limit: + # it can never fit. also make sure the cache does not keep a previous value. + self._cache_delete(nested_name) + return + if size_limit is not None and index.inserted > size_limit / CACHE_RESCAN_DIVISOR: + self._cache_scan(index) + # make room before storing, so the cache does not exceed its size limit. + # the value replaces a previous one (if any), so that does not count. + index.remove(nested_name) + self._cache_evict(index, needed=len(value)) self._stats["cache_store_calls"] += 1 try: self.cache_backend.store(nested_name, value) @@ -485,6 +568,10 @@ def _cache_store(self, nested_name: str, value: StoreValue) -> None: except Exception as err: logger.warning(f"borgstore: cache store failed for {nested_name!r}: {err!r}") self._stats["cache_errors"] += 1 + else: + if index is not None: + index.add(nested_name, len(value), time.time()) + index.inserted += len(value) @_locked def store(self, name: str, value: StoreValue) -> None: @@ -511,6 +598,11 @@ def _cache_delete(self, nested_name: str) -> None: if self.cache_backend is None or self._cache_disabled: return self._stats["cache_delete_calls"] += 1 + index = self._cache_index_for(nested_name) + if index is not None: + # also if deleting fails: the eviction must not try the same item again and again, + # the next scan brings back an item that is still there. + index.remove(nested_name) try: self.cache_backend.delete(nested_name) except ObjectNotFound: @@ -571,13 +663,25 @@ def cache_invalidate(self, name: str, *, deleted: bool = False) -> None: def _cache_move(self, old_nested: str, new_nested: str) -> None: if self.cache_backend is None or self._cache_disabled: return + old_index = self._cache_index_for(old_nested) + entry = old_index.entries.get(old_nested) if old_index is not None else None try: self.cache_backend.move(old_nested, new_nested) except ObjectNotFound: - pass + if old_index is not None: + old_index.remove(old_nested) # another client has evicted it except Exception as err: logger.warning(f"borgstore: cache move failed for {old_nested!r}->{new_nested!r}: {err!r}") self._stats["cache_errors"] += 1 + else: + if old_index is not None: + old_index.remove(old_nested) + new_index = self._cache_index_for(new_nested) + if new_index is not None: + if entry is not None: + new_index.add(new_nested, entry[0], time.time()) # moving counts as using it + else: + new_index.remove(new_nested) # unknown item, the next scan or cache hit adds it @_locked def move( @@ -745,28 +849,52 @@ def defrag(self, sources, *, target=None, algorithm=None, namespace=None, delete ) return unnest(backend_target, namespace=prefix).removeprefix(prefix) - def _cache_cleanup_expired(self) -> None: - now = time.time() - for namespace, policy in self.cache_namespaces: - if policy.max_age is None and policy.size is None: - continue - try: - items = [info for info in self._cache_list(namespace.rstrip("/")) if not info.directory] - if policy.max_age is not None: - remaining_items = [] - for info in items: - if not info.atime or (now - info.atime) > policy.max_age: - self._cache_delete(info.name) - else: - remaining_items.append(info) - items = remaining_items - if policy.size is not None: - total_size = sum(info.size for info in items) - for info in sorted(items, key=lambda entry: (entry.atime, entry.name)): - if total_size <= policy.size: - break - self._cache_delete(info.name) - total_size -= info.size - except Exception as err: - logger.warning(f"borgstore: cache cleanup failed for namespace {namespace!r}: {err!r}") - self._stats["cache_errors"] += 1 + def _cache_scan(self, index: CacheIndex) -> None: + """ + Bring index in line with what the cache backend really has in that namespace. + + Other clients sharing the cache add and evict items, too. Items this store did not + know yet are sorted in by their atime, for known items the more recent one of our last + access and the atime counts (a backend's atime can lag behind or be not implemented). + """ + namespace = index.namespace + index.inserted = 0 # even if scanning fails, do not retry it for each store operation + entries = [] + try: + for info in self._cache_list(namespace.rstrip("/")): + known = index.entries.get(info.name) + last_access = max(known[1], info.atime) if known is not None else info.atime + entries.append((info.name, info.size, last_access)) + index.replace_all(entries) + except ObjectNotFound: + # nothing was cached in this namespace yet (or a directory vanished while listing). + index.replace_all(entries) + except Exception as err: + logger.warning(f"borgstore: cache scan failed for namespace {namespace!r}: {err!r}") + self._stats["cache_errors"] += 1 + + def _cache_evict(self, index: CacheIndex, *, needed: int = 0) -> None: + """ + Evict items that are older than max_age, then evict least recently used items until + more bytes fit into the size limit. + """ + policy = index.policy + if policy.max_age is not None: + now = time.time() + while index.entries: + name, (size, last_access) = next(iter(index.entries.items())) + # last_access is 0 if the item is only known from a backend that has no atime. + if last_access and (now - last_access) <= policy.max_age: + break + index.remove(name) + self._cache_delete(name) + if policy.size is not None: + while index.entries and index.total + needed > policy.size: + name = next(iter(index.entries)) + index.remove(name) + self._cache_delete(name) + + def _cache_cleanup(self) -> None: + for index in self._cache_indexes.values(): + self._cache_scan(index) + self._cache_evict(index) diff --git a/tests/test_cache.py b/tests/test_cache.py index 82fdfad..85c5aa4 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -36,6 +36,19 @@ def make_store(tmp_path, *, config=None, with_cache_backend=True): return Store(**kwargs), cache_root +def fill_shared_cache(tmp_path, names_values): + """Store items using another Store that shares the primary and the cache, but has no cache limits.""" + other, _ = make_store(tmp_path, config=make_config({"data/": {"cache": CacheMode.C_WRITETHROUGH}})) + with other: + for name, value in names_values: + other.store(name, value) + + +def cache_usage(cache_root, namespace="data"): + """Return the total size of the files the cache really has in namespace.""" + return sum(path.stat().st_size for path in (cache_root / namespace).rglob("*") if path.is_file()) + + def test_cache_store_memoryview(tmp_path): """A memoryview value is written to the primary backend as well as to the cache backend.""" store, _ = make_store(tmp_path, config=make_config({"data/": {"cache": "writethrough"}})) @@ -463,8 +476,8 @@ def test_close_cleans_up_lru_cache_items_by_size(tmp_path, monkeypatch): store.create() names_values = [("data/00000000", b"aaaa"), ("data/00000001", b"bbbb"), ("data/00000002", b"cccc")] store.open() - for name, value in names_values: - store.store(name, value) + # another client sharing the cache fills it, so this store only finds the items when it scans at close time. + fill_shared_cache(tmp_path, names_values) nested_names = [store.find(name) for name, _value in names_values] atimes = {nested_names[0]: 100.0, nested_names[1]: 200.0, nested_names[2]: 300.0} @@ -506,8 +519,8 @@ def test_close_cleans_up_expired_before_lru_size_eviction(tmp_path, monkeypatch) store.create() names_values = [("data/00000000", b"aaaa"), ("data/00000001", b"bbbb"), ("data/00000002", b"cccc")] store.open() - for name, value in names_values: - store.store(name, value) + # another client sharing the cache fills it, so this store only finds the items when it scans at close time. + fill_shared_cache(tmp_path, names_values) nested_names = [store.find(name) for name, _value in names_values] now = 1000.0 @@ -726,3 +739,260 @@ def test_public_cache_invalidate(tmp_path): store.cache_invalidate() # type: ignore[call-overload] finally: store.destroy() + + +def make_limited_store(tmp_path, **limits): + """Return (store, cache_root), data/ is cached in writethrough mode with the given limits.""" + return make_store(tmp_path, config=make_config({"data/": {"cache": CacheMode.C_WRITETHROUGH, **limits}})) + + +def data_name(i): + return f"data/{i:08d}" + + +def test_cache_size_limit_holds_while_loading(tmp_path): + """Loading more than the cache size must not grow the cache beyond its size limit, #183.""" + names_values = [(data_name(i), bytes([i]) * 100) for i in range(10)] + store, cache_root = make_limited_store(tmp_path, size=350) + store.create() + try: + with make_store(tmp_path, with_cache_backend=False)[0] as uncached: + for name, value in names_values: + uncached.store(name, value) + with store: + for name, value in names_values: + assert store.load(name) == value + assert cache_usage(cache_root) <= 350 + assert cache_usage(cache_root) == 300 + assert store._cache_indexes["data/"].total == 300 + # the most recently loaded items are still cached: + hits = store.stats["cache_hits"] + for name, value in names_values[-3:]: + assert store.load(name) == value + assert store.stats["cache_hits"] == hits + 3 + finally: + store.destroy() + + +def test_cache_size_limit_holds_while_storing(tmp_path): + store, cache_root = make_limited_store(tmp_path, size=350) + store.create() + try: + with store: + for i in range(10): + store.store(data_name(i), bytes([i]) * 100) + assert cache_usage(cache_root) <= 350 + assert cache_usage(cache_root) == 300 + finally: + store.destroy() + + +def test_cache_evicts_least_recently_used_of_this_session(tmp_path): + """The eviction order follows this store's cache hits, it does not depend on the backend's atime.""" + store, cache_root = make_limited_store(tmp_path, size=300) + store.create() + try: + with store: + for i in range(3): + store.store(data_name(i), bytes([i]) * 100) + assert store.load(data_name(0)) == bytes([0]) * 100 # hit: item 1 is the least recently used now + store.store(data_name(3), bytes([3]) * 100) + cached = {info.name for info in store._cache_list("data")} + assert cached == {store.find(data_name(i)) for i in (0, 2, 3)} + finally: + store.destroy() + + +def test_cache_partial_load_miss_accounts_full_item(tmp_path): + store, cache_root = make_limited_store(tmp_path, size=250) + store.create() + try: + with make_store(tmp_path, with_cache_backend=False)[0] as uncached: + for i in range(3): + uncached.store(data_name(i), bytes([i]) * 100) + with store: + for i in range(3): + assert store.load(data_name(i), offset=10, size=5) == bytes([i]) * 5 + assert store._cache_indexes["data/"].total == cache_usage(cache_root) <= 250 + assert cache_usage(cache_root) == 200 + finally: + store.destroy() + + +def test_cache_does_not_cache_items_bigger_than_size(tmp_path): + store, cache_root = make_limited_store(tmp_path, size=100) + store.create() + try: + with store: + small, big = b"s" * 60, b"b" * 101 + store.store(data_name(0), small) + store.store(data_name(1), big) # can not fit, must not evict anything either + assert cache_usage(cache_root) == 60 + assert store.load(data_name(1)) == big + assert store.load(data_name(1), offset=1, size=2) == b"bb" + assert cache_usage(cache_root) == 60 + # a big value replacing a small one must not leave the previous value in the cache: + store.store(data_name(0), big) + assert cache_usage(cache_root) == 0 + assert store.load(data_name(0)) == big + assert store.stats["cache_errors"] == 0 + finally: + store.destroy() + + +def test_cache_index_accounting(tmp_path): + """The index total follows overwrite, delete, soft delete / undelete and invalidation.""" + store, cache_root = make_limited_store(tmp_path, size=1000) + store.create() + try: + with store: + index = store._cache_indexes["data/"] + store.store(data_name(0), b"a" * 100) + store.store(data_name(1), b"b" * 100) + assert index.total == cache_usage(cache_root) == 200 + store.store(data_name(0), b"a" * 30) # overwrite + assert index.total == cache_usage(cache_root) == 130 + store.move(data_name(0), delete=True) + assert index.total == cache_usage(cache_root) == 130 + assert store.find(data_name(0), deleted=True) in index.entries + assert store.find(data_name(0)) not in index.entries + store.move(data_name(0), undelete=True) + assert index.total == cache_usage(cache_root) == 130 + assert store.find(data_name(0)) in index.entries + store.delete(data_name(0)) + assert index.total == cache_usage(cache_root) == 100 + store.cache_invalidate("data/") + assert index.total == cache_usage(cache_root) == 0 + assert not index.entries + finally: + store.destroy() + + +def test_cache_shared_item_evicted_by_other_client(tmp_path): + """If another client evicted an item, that is a cache miss, the item gets cached again.""" + store, cache_root = make_limited_store(tmp_path, size=1000) + store.create() + try: + with store: + index = store._cache_indexes["data/"] + store.store(data_name(0), b"a" * 100) + store.store(data_name(1), b"b" * 100) + (cache_root / store.find(data_name(0))).unlink() + assert store.load(data_name(0)) == b"a" * 100 + assert store.stats["cache_misses"] == 1 + assert store.stats["cache_errors"] == 0 + assert index.total == cache_usage(cache_root) == 200 + finally: + store.destroy() + + +def test_cache_shared_hit_on_item_of_other_client(tmp_path): + """A cache hit on an item another client has cached adds the full item to the index.""" + store, cache_root = make_limited_store(tmp_path, size=1000) + store.create() + try: + with store: + index = store._cache_indexes["data/"] + fill_shared_cache(tmp_path, [(data_name(0), b"a" * 100)]) + assert index.total == 0 + assert store.load(data_name(0), offset=0, size=10) == b"a" * 10 + assert store.stats["cache_hits"] == 1 + assert index.entries[store.find(data_name(0))][0] == 100 + assert index.total == 100 + finally: + store.destroy() + + +def test_cache_shared_rescan_sees_other_clients_items(tmp_path): + """After inserting more than size / CACHE_RESCAN_DIVISOR bytes, the store scans the shared cache.""" + store, cache_root = make_limited_store(tmp_path, size=1000) + store.create() + try: + with store: + index = store._cache_indexes["data/"] + # another client fills the cache up to the limit: + fill_shared_cache(tmp_path, [(data_name(100 + i), b"o" * 100) for i in range(10)]) + assert cache_usage(cache_root) == 1000 + for i in range(10): + store.store(data_name(i), bytes([i]) * 100) + # this store can only overshoot by what it inserts between 2 scans: + assert cache_usage(cache_root) <= 1000 + 1000 // store_module.CACHE_RESCAN_DIVISOR + 100 + assert index.total == cache_usage(cache_root) <= 1000 + # the other client's items were the least recently used ones, so they got evicted first: + cached = {info.name for info in store._cache_list("data")} + assert {store.find(data_name(i)) for i in range(10)} <= cached + finally: + store.destroy() + + +def test_cache_max_age_eviction_when_storing(tmp_path, monkeypatch): + """Expired items get evicted when something is put into the cache, a cache hit never expires anything.""" + now = 1000.0 + monkeypatch.setattr("borgstore.store.time.time", lambda: now) + store, cache_root = make_limited_store(tmp_path, max_age=5) + store.create() + try: + with store: + store.store(data_name(0), b"a" * 10) + now = 1004.0 + store.store(data_name(1), b"b" * 10) + now = 1007.0 # item 0 is expired now + assert store.load(data_name(0)) == b"a" * 10 + assert store.stats["cache_hits"] == 1 # still served from the cache, and that counts as using it + now = 1010.0 # item 1 is expired now, item 0 was used 3s ago + store.store(data_name(2), b"c" * 10) + cached = {info.name for info in store._cache_list("data")} + assert cached == {store.find(data_name(i)) for i in (0, 2)} + finally: + store.destroy() + + +def test_cache_eviction_errors_do_not_fail_main_operations(tmp_path): + store, cache_root = make_limited_store(tmp_path, size=250) + store.create() + try: + with store: + + def failing_delete(_backend_name): + raise RuntimeError("boom") + + original_delete = store.cache_backend.delete + store.cache_backend.delete = failing_delete + try: + for i in range(5): + store.store(data_name(i), bytes([i]) * 100) + for i in range(5): + assert store.load(data_name(i)) == bytes([i]) * 100 + assert store.stats["cache_errors"] >= 1 + finally: + store.cache_backend.delete = original_delete + # closing the store scans the cache and evicts what could not be evicted before: + assert cache_usage(cache_root) <= 250 + finally: + store.destroy() + + +def test_cache_without_limits_keeps_no_index(tmp_path): + store, cache_root = make_store(tmp_path, config=make_config({"data/": {"cache": CacheMode.C_MIRROR}})) + store.create() + try: + with store: + assert store._cache_indexes == {} + for i in range(5): + store.store(data_name(i), bytes([i]) * 100) + assert cache_usage(cache_root) == 500 + assert cache_usage(cache_root) == 500 + finally: + store.destroy() + + +def test_cache_first_open_does_not_count_errors(tmp_path): + """Scanning a namespace the cache backend does not have yet is not an error.""" + store, cache_root = make_limited_store(tmp_path, size=1000) + store.create() + try: + with store: + pass + assert store.stats["cache_errors"] == 0 + finally: + store.destroy()