diff --git a/CHANGELOG.md b/CHANGELOG.md index c58200102..b1abe8d8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix(cache): preserve disk cache entries on lookup by @remote-controlled-man in #2423 - fix(ci): remove unsupported macOS 14 release runners by @abetlen in #2416 ## [0.3.37] diff --git a/llama_cpp/llama_cache.py b/llama_cpp/llama_cache.py index 5220c7933..b7d68c227 100644 --- a/llama_cpp/llama_cache.py +++ b/llama_cpp/llama_cache.py @@ -132,10 +132,7 @@ def __getitem__(self, key: Sequence[int]) -> "llama_cpp.llama.LlamaState": _key = self._find_longest_prefix_key(key) if _key is None: raise KeyError("Key not found") - value: "llama_cpp.llama.LlamaState" = self.cache.pop(_key) # type: ignore - # NOTE: This puts an integer as key in cache, which breaks, - # Llama.longest_token_prefix(k, key) above since k is not a tuple of ints/tokens - # self.cache.push(_key, side="front") # type: ignore + value: "llama_cpp.llama.LlamaState" = self.cache[_key] # type: ignore return value def __contains__(self, key: Sequence[int]) -> bool: diff --git a/tests/test_llama_cache.py b/tests/test_llama_cache.py new file mode 100644 index 000000000..bb4977896 --- /dev/null +++ b/tests/test_llama_cache.py @@ -0,0 +1,77 @@ +import numpy as np +import pytest + +from llama_cpp import LlamaState +from llama_cpp.llama_cache import LlamaDiskCache, LlamaRAMCache + + +@pytest.fixture(params=[LlamaRAMCache, LlamaDiskCache]) +def cache(request, tmp_path): + if request.param is LlamaDiskCache: + cache = request.param(cache_dir=str(tmp_path / "cache")) + else: + cache = request.param() + try: + yield cache + finally: + if isinstance(cache, LlamaDiskCache): + cache.cache.close() + + +@pytest.fixture +def state(): + return LlamaState( + input_ids=np.array([1, 2, 3], dtype=np.intc), + scores=np.zeros((3, 4), dtype=np.single), + n_tokens=3, + llama_state=b"saved model state", + llama_state_size=17, + seed=42, + ) + + +@pytest.mark.parametrize("query", [(1, 2, 3), (1, 2, 3, 4), (1, 2), (1, 2, 9)]) +def test_cache_hits_remain_available(cache, state, query): + cache[[1, 2, 3]] = state + + for _ in range(3): + assert query in cache + result = cache[query] + np.testing.assert_array_equal(result.input_ids, state.input_ids) + np.testing.assert_array_equal(result.scores, state.scores) + assert result.n_tokens == state.n_tokens + assert result.llama_state == state.llama_state + assert result.llama_state_size == state.llama_state_size + assert result.seed == state.seed + assert [1, 2, 3] in cache + + +@pytest.mark.parametrize("query", [(), (9,)]) +def test_cache_misses_leave_saved_states_available(cache, state, query): + assert query not in cache + with pytest.raises(KeyError): + cache[query] + + cache[[1, 2, 3]] = state + assert query not in cache + with pytest.raises(KeyError): + cache[query] + assert [1, 2, 3] in cache + + +def test_disk_cache_hit_survives_reopening(tmp_path, state): + directory = str(tmp_path / "cache") + cache = LlamaDiskCache(cache_dir=directory) + try: + cache[[1, 2, 3]] = state + assert cache[[1, 2, 3, 4]].llama_state == state.llama_state + finally: + cache.cache.close() + + reopened = LlamaDiskCache(cache_dir=directory) + try: + assert [1, 2, 3] in reopened + assert reopened[[1, 2, 9]].llama_state == state.llama_state + assert [1, 2, 3] in reopened + finally: + reopened.cache.close()