From 1f943b88b80841811430382264b9387737fa329d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 28 Sep 2026 15:48:10 +1300 Subject: [PATCH 1/2] (fix): keep cache owner registrations in one key per collection The document cache and the find() query cache registered every invalidation under its own `#owner:` key and released it with purge(). On the Redis adapters purge() runs LUA_PURGE_BUMP, which DELs the key and then HSETs its next generation, so the key never goes away; the adapter sets no expiry either. Each write therefore left one permanent key per cache: a probe of 200 createDocument() calls against MariaDB and Redis grew the owner keys from 1 to 201 in each cache. A collection's registrations are now fields of one `#owners` hash: save($ownersKey, $token, $token) registers, load() with the token checks, and purge($ownersKey, $token) releases through LUA_PURGE_FIELD, which HDELs the field and bumps the key's generation. The key count is bounded by the number of collections, and the hash only holds the registrations still in flight. The owner checks, fail-closed paths and flush tolerance are unchanged; only where a registration lives moved. Adapters that ignore $hash (Memory, Memcached, Filesystem, Hazelcast) share one slot per collection, so two overlapping invalidations of one collection there would report the earlier owner as invalid after its commit and leave the epoch blocked. The Redis adapters, which are the only Leasable ones, keep fields apart. Keys matching `*#owner:*` from earlier builds are no longer read. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 + src/Database/Cache/QueryCache.php | 14 +- src/Database/Traits/Documents.php | 18 +-- tests/e2e/Adapter/Scopes/GeneralTests.php | 75 +++++++++ tests/unit/Cache/OwnershipCache.php | 26 +++- tests/unit/Cache/QueryCacheTest.php | 64 +++++++- tests/unit/Cache/RedisLeasableCache.php | 146 ++++++++++++++++++ .../unit/Documents/DocumentCacheEpochTest.php | 103 +++++++++++- 8 files changed, 421 insertions(+), 29 deletions(-) create mode 100644 tests/unit/Cache/RedisLeasableCache.php diff --git a/CHANGELOG.md b/CHANGELOG.md index bf0047315..45dec175d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -286,6 +286,10 @@ not change anything for an upgrade from 7.x. document's tenant, whichever tenant is selected on the writing `Database`, and leaves every other tenant's in place. - A written document's own attribute named `options` no longer names a collection to invalidate. + - Writes no longer leave a key behind in Redis each. The document cache and the `find()` query cache registered + every invalidation under a key of its own, and a Redis purge keeps a purged key, holding its generation, with no + expiry. A collection's invalidations now register as fields of one `#owners` key. Keys matching `*#owner:*` + left by earlier builds are no longer read and can be deleted. - **Permissions and tenancy:** - Under shared tables, `upsertDocuments()` and `upsertDocument()` store their permission rows under the tenant, on an adapter with no earlier write and through `Adapter\Pool` alike. diff --git a/src/Database/Cache/QueryCache.php b/src/Database/Cache/QueryCache.php index 2fd280868..9767ea765 100644 --- a/src/Database/Cache/QueryCache.php +++ b/src/Database/Cache/QueryCache.php @@ -169,7 +169,7 @@ public function invalidateCollection(Scope $scope, string $collection): void */ public function blockCollection(string $key, string $token): void { - if ($this->cache->save($this->getOwnerKey($key, $token), $token) === false) { + if ($this->cache->save($this->getOwnersKey($key), $token, $token) === false) { throw new RuntimeException("Failed to register query cache owner for '{$key}'"); } @@ -186,14 +186,14 @@ public function blockCollection(string $key, string $token): void */ public function activateCollection(string $key, string $token): void { - $ownerKey = $this->getOwnerKey($key, $token); - $owner = $this->cache->load($ownerKey, self::PERMANENT); + $ownersKey = $this->getOwnersKey($key); + $owner = $this->cache->load($ownersKey, self::PERMANENT, $token); if ($owner !== false && $owner !== null && $owner !== $token) { throw new RuntimeException("Invalid query cache owner for '{$key}'"); } $owned = $owner === $token; - if ($owned && ! $this->cache->purge($ownerKey)) { - $owner = $this->cache->load($ownerKey, self::PERMANENT); + if ($owned && ! $this->cache->purge($ownersKey, $token)) { + $owner = $this->cache->load($ownersKey, self::PERMANENT, $token); if ($owner !== false && $owner !== null) { throw new RuntimeException("Failed to release query cache owner for '{$key}'"); } @@ -324,9 +324,9 @@ private function getFinishedKey(string $key): string return $key.'#finished'; } - private function getOwnerKey(string $key, string $token): string + private function getOwnersKey(string $key): string { - return $key.'#owner:'.$token; + return $key.'#owners'; } private function getStartedKey(string $key): string diff --git a/src/Database/Traits/Documents.php b/src/Database/Traits/Documents.php index d5daf0130..65367fa08 100644 --- a/src/Database/Traits/Documents.php +++ b/src/Database/Traits/Documents.php @@ -2588,14 +2588,14 @@ private function advanceDocumentCacheEpoch(string $collectionKey): bool private function blockDocumentCacheEpoch(string $collectionKey, string $token): bool { $epochKey = $collectionKey.'#epoch'; - $ownerKey = $collectionKey.'#owner:'.$token; - if ($this->cache->save($ownerKey, $token) === false) { + $ownersKey = $collectionKey.'#owners'; + if ($this->cache->save($ownersKey, $token, $token) === false) { $epoch = $this->cache->load($epochKey, self::TTL); if ($epoch === false || $epoch === null) { return false; } - throw new RuntimeException("Failed to register document cache owner '{$ownerKey}'"); + throw new RuntimeException("Failed to register document cache owner '{$token}' in '{$ownersKey}'"); } $startedKey = $collectionKey.'#started'; @@ -2633,16 +2633,16 @@ protected function activateDocumentInvalidation(array $tokens): void private function activateDocumentCacheEpoch(string $collectionKey, string $token): void { - $ownerKey = $collectionKey.'#owner:'.$token; - $owner = $this->cache->load($ownerKey, self::TTL); + $ownersKey = $collectionKey.'#owners'; + $owner = $this->cache->load($ownersKey, self::TTL, $token); if ($owner !== false && $owner !== null && $owner !== $token) { - throw new RuntimeException("Invalid document cache owner '{$ownerKey}'"); + throw new RuntimeException("Invalid document cache owner '{$token}' in '{$ownersKey}'"); } $owned = $owner === $token; - if ($owned && ! $this->cache->purge($ownerKey)) { - $owner = $this->cache->load($ownerKey, self::TTL); + if ($owned && ! $this->cache->purge($ownersKey, $token)) { + $owner = $this->cache->load($ownersKey, self::TTL, $token); if ($owner !== false && $owner !== null) { - throw new RuntimeException("Failed to release document cache owner '{$ownerKey}'"); + throw new RuntimeException("Failed to release document cache owner '{$token}' in '{$ownersKey}'"); } $owned = false; } diff --git a/tests/e2e/Adapter/Scopes/GeneralTests.php b/tests/e2e/Adapter/Scopes/GeneralTests.php index 99091a805..b2b3eb2e5 100644 --- a/tests/e2e/Adapter/Scopes/GeneralTests.php +++ b/tests/e2e/Adapter/Scopes/GeneralTests.php @@ -847,6 +847,61 @@ public function testNestedTransactionKeepsDocumentCacheEpochBlockedUntilOuterCom } } + public function testCacheInvalidationLeavesOneOwnerKeyPerCollection(): void + { + $database = $this->getDatabase(); + if (! $database->getAdapter()->supports(Capability::Caching)) { + $this->markTestSkipped('Adapter does not use the document cache.'); + } + + $collection = 'ownerKeys'.ID::unique(); + $database->createCollection(new Collection(id: $collection, attributes: [ + Attribute::string(key: 'name', size: 64, required: true), + ], permissions: [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + ])); + + $redis = new Redis(); + $redis->connect('redis', 6379); + $cache = new Cache(new RedisAdapter($redis)); + $original = $database->getCache(); + $queryCache = $database->getQueryCache(); + $destination = $database instanceof Mirror ? $database->getDestination() : null; + $destinationCache = $destination?->getCache(); + $database->setCache($cache)->setQueryCache(new QueryCache($cache)); + + try { + $database->createDocument($collection, new Document(['$id' => 'doc0', 'name' => 'doc 0'])); + $owners = $this->scanOwnerKeys($redis, $collection); + + for ($index = 1; $index <= 10; $index++) { + $database->createDocument($collection, new Document(['$id' => 'doc'.$index, 'name' => 'doc '.$index])); + $this->assertCount($index + 1, $database->find($collection, [Query::limit(100)])); + } + $database->withTransaction(function () use ($database, $collection): void { + $database->updateDocument($collection, 'doc1', new Document(['name' => 'updated'])); + $database->updateDocument($collection, 'doc2', new Document(['name' => 'updated'])); + }); + + $this->assertSame($owners, $this->scanOwnerKeys($redis, $collection), 'A purged key stays behind in Redis, so a write must not register its owner under a key of its own'); + $this->assertNotEmpty($owners); + foreach ($owners as $key) { + $this->assertStringEndsWith('#owners', $key); + $fields = $redis->hKeys($key); + $this->assertIsArray($fields); + $this->assertSame([], \array_values(\array_diff($fields, ['__utopia_gen__', '__utopia_tomb__'])), "Every write must release its registration in '{$key}'"); + } + } finally { + $database->setQueryCache($queryCache)->setCache($original); + if ($destination !== null && $destinationCache !== null) { + $destination->setCache($destinationCache); + } + $database->deleteCollection($collection); + } + } + /** * Test that withTransaction correctly resets inTransaction state * when a known exception (DuplicateException) is thrown after successful rollback. @@ -1068,6 +1123,26 @@ public function testCacheReconnect(): void } } + /** + * @return array + */ + private function scanOwnerKeys(Redis $redis, string $collection): array + { + $keys = []; + $iterator = null; + do { + $batch = $redis->scan($iterator, '*'.\strtolower($collection).'*#owner*', 1000); + if (\is_array($batch)) { + \array_push($keys, ...$batch); + } + } while ($iterator > 0); + + $keys = \array_values(\array_unique($keys)); + \sort($keys); + + return $keys; + } + private function dropRedisConnection(Redis $redis): void { $id = $redis->rawCommand('CLIENT', 'ID'); diff --git a/tests/unit/Cache/OwnershipCache.php b/tests/unit/Cache/OwnershipCache.php index 3b7db5b58..460b8660c 100644 --- a/tests/unit/Cache/OwnershipCache.php +++ b/tests/unit/Cache/OwnershipCache.php @@ -8,7 +8,7 @@ final class OwnershipCache implements CacheAdapter, Leasable { - /** @var array|string}> */ + /** @var array|string}>> */ private array $store = []; /** @var array */ @@ -22,7 +22,7 @@ final class OwnershipCache implements CacheAdapter, Leasable public function load(string $key, int $ttl, string $hash = ''): mixed { - $saved = $this->store[$key] ?? null; + $saved = $this->store[$key][$this->field($key, $hash)] ?? null; return $saved !== null && $saved['time'] + $ttl > \time() ? $saved['data'] : false; } @@ -44,7 +44,7 @@ public function save(string $key, array|string $data, string $hash = ''): bool|s $activation(); } - $this->store[$key] = ['time' => \time(), 'data' => $data]; + $this->store[$key][$this->field($key, $hash)] = ['time' => \time(), 'data' => $data]; return $data; } @@ -65,11 +65,12 @@ public function saveWithLease(string $key, array|string $data, string $hash, str public function touch(string $key, string $hash = ''): bool { - if (! isset($this->store[$key])) { + $field = $this->field($key, $hash); + if (! isset($this->store[$key][$field])) { return false; } - $this->store[$key]['time'] = \time(); + $this->store[$key][$field]['time'] = \time(); return true; } @@ -92,7 +93,11 @@ public function purge(string $key, string $hash = ''): bool } $this->generations[$key] = ($this->generations[$key] ?? 0) + 1; - unset($this->store[$key]); + if ($hash === '') { + unset($this->store[$key]); + } else { + unset($this->store[$key][$hash]); + } return true; } @@ -120,9 +125,9 @@ public function getName(?string $key = null): string return 'ownership'; } - public function has(string $key): bool + public function has(string $key, string $hash = ''): bool { - return isset($this->store[$key]); + return isset($this->store[$key][$this->field($key, $hash)]); } public function pauseNextActivation(Closure $activation): void @@ -139,4 +144,9 @@ public function failDuringActivation(): void { $this->failDuringActivation = true; } + + private function field(string $key, string $hash): string + { + return $hash === '' ? $key : $hash; + } } diff --git a/tests/unit/Cache/QueryCacheTest.php b/tests/unit/Cache/QueryCacheTest.php index dd8524d78..3aee3d8f9 100644 --- a/tests/unit/Cache/QueryCacheTest.php +++ b/tests/unit/Cache/QueryCacheTest.php @@ -448,18 +448,18 @@ public function testConcurrentOwnersCannotEnableCacheEarly(): void $second->blockCollection($key, $secondToken); $this->assertNull($reader->getEntry($scope, 'users', [])); - $this->assertTrue($adapter->has($key.'#owner:'.$secondToken)); + $this->assertTrue($adapter->has($key.'#owners', $secondToken)); }); $first->activateCollection($key, $firstToken); - $this->assertFalse($adapter->has($key.'#owner:'.$firstToken)); - $this->assertTrue($adapter->has($key.'#owner:'.$secondToken)); + $this->assertFalse($adapter->has($key.'#owners', $firstToken)); + $this->assertTrue($adapter->has($key.'#owners', $secondToken)); $this->assertNull($reader->getEntry($scope, 'users', [])); $second->activateCollection($key, $secondToken); - $this->assertFalse($adapter->has($key.'#owner:'.$secondToken)); + $this->assertFalse($adapter->has($key.'#owners', $secondToken)); $fresh = $reader->getEntry($scope, 'users', []); $this->assertNotNull($fresh); $this->assertNull($reader->get($fresh)); @@ -564,6 +564,62 @@ public function testActivationPurgeFailureStillPropagates(): void $this->assertNull($queryCache->getEntry(new Scope(), 'users', [])); } + public function testInvalidationsLeaveOneOwnerKeyPerCollection(): void + { + $adapter = new RedisLeasableCache(); + $queryCache = new QueryCache(new Cache($adapter)); + $scope = new Scope(namespace: 'ns'); + $key = $queryCache->getCollectionKey($scope, 'users'); + + for ($invalidation = 0; $invalidation < 20; $invalidation++) { + $queryCache->invalidateCollection($scope, 'users'); + } + + $this->assertSame( + [$key.'#owners'], + \array_values(\array_filter($adapter->keys(), static fn (string $stored): bool => \str_contains($stored, '#owner'))), + 'A purged key stays behind in Redis, so invalidations must share one owner key instead of leaving one each', + ); + $this->assertSame([], $adapter->list($key.'#owners'), 'Every invalidation must release its registration'); + $this->assertNotNull($queryCache->getEntry($scope, 'users', [])); + } + + public function testActivationRejectsAForeignOwnerRegistration(): void + { + $cache = new Cache(new RedisLeasableCache()); + $queryCache = new QueryCache($cache); + $key = $queryCache->getCollectionKey(new Scope(), 'users'); + $queryCache->blockCollection($key, 'owner'); + $cache->save($key.'#owners', 'intruder', 'owner'); + + try { + $queryCache->activateCollection($key, 'owner'); + $this->fail('A foreign owner registration was accepted'); + } catch (\RuntimeException $error) { + $this->assertStringContainsString('Invalid query cache owner', $error->getMessage()); + } + + $this->assertNull($queryCache->getEntry(new Scope(), 'users', [])); + } + + public function testActivationPropagatesAnOwnerReleaseFailure(): void + { + $adapter = new RedisLeasableCache(); + $queryCache = new QueryCache(new Cache($adapter)); + $key = $queryCache->getCollectionKey(new Scope(), 'users'); + $queryCache->blockCollection($key, 'owner'); + $adapter->failFieldPurges('#owners'); + + try { + $queryCache->activateCollection($key, 'owner'); + $this->fail('An owner release failure was not propagated'); + } catch (\RuntimeException $error) { + $this->assertStringContainsString('Failed to release query cache owner', $error->getMessage()); + } + + $this->assertNull($queryCache->getEntry(new Scope(), 'users', [])); + } + public function testInvalidationPropagatesACacheWriteFailure(): void { $cache = new InvalidationCache(); diff --git a/tests/unit/Cache/RedisLeasableCache.php b/tests/unit/Cache/RedisLeasableCache.php new file mode 100644 index 000000000..080875fe7 --- /dev/null +++ b/tests/unit/Cache/RedisLeasableCache.php @@ -0,0 +1,146 @@ +|string}>> */ + private array $fields = []; + + /** @var array */ + private array $generations = []; + + private ?string $failingFieldPurges = null; + + public function load(string $key, int $ttl, string $hash = ''): mixed + { + $saved = $this->fields[$key][$this->field($key, $hash)] ?? null; + + return $saved !== null && $saved['time'] + $ttl > \time() ? $saved['data'] : false; + } + + public function save(string $key, array|string $data, string $hash = ''): bool|string|array + { + if ($key === '' || empty($data)) { + return false; + } + + $this->fields[$key][$this->field($key, $hash)] = ['time' => \time(), 'data' => $data]; + + return $data; + } + + public function getGeneration(string $key): string + { + return (string) ($this->generations[$key] ?? 0); + } + + public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array + { + if ($this->getGeneration($key) !== $generation) { + return false; + } + + return $this->save($key, $data, $hash); + } + + public function touch(string $key, string $hash = ''): bool + { + $field = $this->field($key, $hash); + if (! isset($this->fields[$key][$field])) { + return false; + } + + $this->fields[$key][$field]['time'] = \time(); + + return true; + } + + /** @return array */ + public function list(string $key): array + { + return \array_map(\strval(...), \array_keys($this->fields[$key] ?? [])); + } + + public function purge(string $key, string $hash = ''): bool + { + if ($hash !== '' && $this->failingFieldPurges !== null && \str_ends_with($key, $this->failingFieldPurges)) { + return false; + } + + $this->generations[$key] = ($this->generations[$key] ?? 0) + 1; + + if ($hash === '') { + $removed = \count($this->fields[$key] ?? []); + unset($this->fields[$key]); + + return $removed > 0; + } + + $removed = isset($this->fields[$key][$hash]); + unset($this->fields[$key][$hash]); + if (($this->fields[$key] ?? null) === []) { + unset($this->fields[$key]); + } + + return $removed; + } + + public function flush(): bool + { + $this->fields = []; + $this->generations = []; + + return true; + } + + public function ping(): bool + { + return true; + } + + public function getSize(): int + { + return \count($this->keys()); + } + + public function getName(?string $key = null): string + { + return 'redis-leasable'; + } + + /** + * Every key the cache holds, including keys that only hold a generation. + * + * @return array + */ + public function keys(): array + { + $keys = \array_map(\strval(...), \array_keys($this->fields + $this->generations)); + \sort($keys); + + return $keys; + } + + /** + * Fail every field purge of a key ending with $suffix, leaving the field in place. + */ + public function failFieldPurges(string $suffix): void + { + $this->failingFieldPurges = $suffix; + } + + private function field(string $key, string $hash): string + { + return $hash === '' ? $key : $hash; + } +} diff --git a/tests/unit/Documents/DocumentCacheEpochTest.php b/tests/unit/Documents/DocumentCacheEpochTest.php index 0699f1a43..c9092c683 100644 --- a/tests/unit/Documents/DocumentCacheEpochTest.php +++ b/tests/unit/Documents/DocumentCacheEpochTest.php @@ -4,12 +4,15 @@ use Closure; use PHPUnit\Framework\TestCase; +use Tests\Unit\Cache\RedisLeasableCache; +use Utopia\Cache\Adapter as CacheAdapter; use Utopia\Cache\Adapter\Memory as MemoryCache; use Utopia\Cache\Cache; use Utopia\Cache\Feature\Leasable; use Utopia\Database\Adapter\Memory as DatabaseMemory; use Utopia\Database\Adapter\SQLite; use Utopia\Database\Attribute; +use Utopia\Database\Cache\QueryCache; use Utopia\Database\Collection; use Utopia\Database\Database; use Utopia\Database\Document; @@ -246,6 +249,104 @@ public function testActivationPurgeFailureStillPropagates(): void $this->assertSame('updated', $database->getDocument('webhooks', 'hook')->getAttribute('name')); } + public function testWritesLeaveOneOwnerKeyPerCollection(): void + { + $cache = new RedisLeasableCache(); + $database = $this->createDatabaseWithCache($cache); + $database->setQueryCache(new QueryCache(new Cache($cache))); + $database->createDocument('webhooks', new Document([ + '$id' => 'hook0', + 'name' => 'hook 0', + ])); + $owners = $this->getOwnerKeys($cache); + + for ($index = 1; $index <= 20; $index++) { + $database->createDocument('webhooks', new Document([ + '$id' => 'hook'.$index, + 'name' => 'hook '.$index, + ])); + } + $database->withTransaction(function () use ($database): void { + $database->updateDocument('webhooks', 'hook1', new Document(['name' => 'updated'])); + $database->updateDocument('webhooks', 'hook2', new Document(['name' => 'updated'])); + }); + + $this->assertSame($owners, $this->getOwnerKeys($cache), 'A purged key stays behind in Redis, so a write must not register its owner under a key of its own'); + [$collectionKey] = $database->getCacheKeys('webhooks'); + $this->assertContains(\strtolower($collectionKey.'#owners'), $owners); + foreach ($owners as $key) { + $this->assertSame([], $cache->list($key), "Every write must release its registration in '{$key}'"); + } + $this->assertSame('updated', $database->getDocument('webhooks', 'hook1')->getAttribute('name')); + } + + public function testActivationRejectsAForeignOwnerRegistration(): void + { + $cache = new RedisLeasableCache(); + $database = $this->createDatabaseWithCache($cache); + $database->createDocument('webhooks', new Document([ + '$id' => 'hook', + 'name' => 'original', + ])); + [$collectionKey] = $database->getCacheKeys('webhooks', 'hook'); + $ownersKey = \strtolower($collectionKey.'#owners'); + + try { + $database->withTransaction(function () use ($database, $cache, $ownersKey): void { + $database->updateDocument('webhooks', 'hook', new Document(['name' => 'updated'])); + foreach ($cache->list($ownersKey) as $token) { + $cache->save($ownersKey, 'intruder', $token); + } + }); + $this->fail('A foreign document cache owner registration was accepted'); + } catch (\RuntimeException $error) { + $this->assertStringContainsString('Invalid document cache owner', $error->getMessage()); + } + + $this->assertDocumentCacheEpochBlocked($database, $collectionKey); + $this->assertSame('updated', $database->getDocument('webhooks', 'hook')->getAttribute('name')); + } + + public function testActivationPropagatesAnOwnerReleaseFailure(): void + { + $cache = new RedisLeasableCache(); + $database = $this->createDatabaseWithCache($cache); + $database->createDocument('webhooks', new Document([ + '$id' => 'hook', + 'name' => 'original', + ])); + $cache->failFieldPurges('#owners'); + + try { + $database->updateDocument('webhooks', 'hook', new Document(['name' => 'updated'])); + $this->fail('A document cache owner release failure was not propagated'); + } catch (\RuntimeException $error) { + $this->assertStringContainsString('Failed to release document cache owner', $error->getMessage()); + } + + [$collectionKey] = $database->getCacheKeys('webhooks', 'hook'); + $this->assertDocumentCacheEpochBlocked($database, $collectionKey); + $this->assertSame('updated', $database->getDocument('webhooks', 'hook')->getAttribute('name')); + } + + /** + * @return array + */ + private function getOwnerKeys(RedisLeasableCache $cache): array + { + return \array_values(\array_filter( + $cache->keys(), + static fn (string $key): bool => \str_contains($key, '#owner'), + )); + } + + private function assertDocumentCacheEpochBlocked(Database $database, string $collectionKey): void + { + $epoch = $database->getCache()->load($collectionKey.'#epoch', Database::TTL); + $this->assertIsString($epoch); + $this->assertStringStartsWith('blocked:', $epoch); + } + /** * @return array{Database, FailPurgeMemory} */ @@ -257,7 +358,7 @@ private function createDatabase(): array return [$database, $adapter]; } - private function createDatabaseWithCache(MemoryCache $cache): Database + private function createDatabaseWithCache(CacheAdapter $cache): Database { $database = new Database(new DatabaseMemory(), new Cache($cache)); $database From b258bdb415de024310b4e7da5f837bc8723fc69e Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 28 Sep 2026 16:48:08 +1300 Subject: [PATCH 2/2] (fix): keep separate owner registrations on caches without fields The first revision put every registration in one `#owners` hash per collection. Memory, Memcached, Filesystem and Hazelcast ignore the field, so there overlapping invalidations of one collection shared a single slot, and the earlier owner read the later owner's token and threw "Invalid document cache owner" after its write had committed. Owners::register() now checks, right after saving its field, whether the adapter lists it. The Redis adapters do, so the token stays a field of the bounded `#owners` hash. Field-less adapters always list nothing, so the token also gets a key of its own, as before this branch; their purge deletes it, so nothing leaks there. Owners::find() locates the registration the same way at activation, so the checks, release and fail-closed paths read the registration the token actually has. The regression tests now assert that the key set does not grow with writes, and inject the owner faults as cache faults, instead of naming the owners key or Redis's reserved fields. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +- src/Database/Cache/Owners.php | 57 ++++++++++++++ src/Database/Cache/QueryCache.php | 15 ++-- src/Database/Cache/Registration.php | 12 +++ src/Database/Traits/Documents.php | 18 ++--- tests/e2e/Adapter/Scopes/GeneralTests.php | 20 ++--- tests/unit/Cache/OwnershipCache.php | 26 ++----- tests/unit/Cache/QueryCacheTest.php | 49 +++++++----- tests/unit/Cache/RedisLeasableCache.php | 24 ++++-- .../unit/Documents/DocumentCacheEpochTest.php | 75 ++++++++++--------- 10 files changed, 192 insertions(+), 109 deletions(-) create mode 100644 src/Database/Cache/Owners.php create mode 100644 src/Database/Cache/Registration.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 45dec175d..aba1583e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -288,8 +288,9 @@ not change anything for an upgrade from 7.x. - A written document's own attribute named `options` no longer names a collection to invalidate. - Writes no longer leave a key behind in Redis each. The document cache and the `find()` query cache registered every invalidation under a key of its own, and a Redis purge keeps a purged key, holding its generation, with no - expiry. A collection's invalidations now register as fields of one `#owners` key. Keys matching `*#owner:*` - left by earlier builds are no longer read and can be deleted. + expiry. On the Redis adapters a collection's invalidations now register as fields of one `#owners` key; + adapters that store no fields keep a key per invalidation, which their purge deletes. On Redis, keys matching + `*#owner:*` left by earlier builds are no longer read and can be deleted. - **Permissions and tenancy:** - Under shared tables, `upsertDocuments()` and `upsertDocument()` store their permission rows under the tenant, on an adapter with no earlier write and through `Adapter\Pool` alike. diff --git a/src/Database/Cache/Owners.php b/src/Database/Cache/Owners.php new file mode 100644 index 000000000..b23af76a0 --- /dev/null +++ b/src/Database/Cache/Owners.php @@ -0,0 +1,57 @@ +getOwnersKey($key); + if ($this->cache->save($owners, $token, $token) === false) { + return false; + } + + if ($this->isField($owners, $token)) { + return true; + } + + return $this->cache->save($this->getOwnerKey($key, $token), $token) !== false; + } + + public function find(string $key, string $token): Registration + { + $owners = $this->getOwnersKey($key); + + return $this->isField($owners, $token) + ? new Registration($owners, $token) + : new Registration($this->getOwnerKey($key, $token)); + } + + private function isField(string $owners, string $token): bool + { + return \in_array($token, $this->cache->list($owners), true); + } + + private function getOwnersKey(string $key): string + { + return $key.'#owners'; + } + + private function getOwnerKey(string $key, string $token): string + { + return $key.'#owner:'.$token; + } +} diff --git a/src/Database/Cache/QueryCache.php b/src/Database/Cache/QueryCache.php index 9767ea765..a0a7f552a 100644 --- a/src/Database/Cache/QueryCache.php +++ b/src/Database/Cache/QueryCache.php @@ -169,7 +169,7 @@ public function invalidateCollection(Scope $scope, string $collection): void */ public function blockCollection(string $key, string $token): void { - if ($this->cache->save($this->getOwnersKey($key), $token, $token) === false) { + if (! (new Owners($this->cache))->register($key, $token)) { throw new RuntimeException("Failed to register query cache owner for '{$key}'"); } @@ -186,14 +186,14 @@ public function blockCollection(string $key, string $token): void */ public function activateCollection(string $key, string $token): void { - $ownersKey = $this->getOwnersKey($key); - $owner = $this->cache->load($ownersKey, self::PERMANENT, $token); + $registration = (new Owners($this->cache))->find($key, $token); + $owner = $this->cache->load($registration->key, self::PERMANENT, $registration->field); if ($owner !== false && $owner !== null && $owner !== $token) { throw new RuntimeException("Invalid query cache owner for '{$key}'"); } $owned = $owner === $token; - if ($owned && ! $this->cache->purge($ownersKey, $token)) { - $owner = $this->cache->load($ownersKey, self::PERMANENT, $token); + if ($owned && ! $this->cache->purge($registration->key, $registration->field)) { + $owner = $this->cache->load($registration->key, self::PERMANENT, $registration->field); if ($owner !== false && $owner !== null) { throw new RuntimeException("Failed to release query cache owner for '{$key}'"); } @@ -324,11 +324,6 @@ private function getFinishedKey(string $key): string return $key.'#finished'; } - private function getOwnersKey(string $key): string - { - return $key.'#owners'; - } - private function getStartedKey(string $key): string { return $key.'#started'; diff --git a/src/Database/Cache/Registration.php b/src/Database/Cache/Registration.php new file mode 100644 index 000000000..6c07723a0 --- /dev/null +++ b/src/Database/Cache/Registration.php @@ -0,0 +1,12 @@ +cache->save($ownersKey, $token, $token) === false) { + if (! (new Owners($this->cache))->register($collectionKey, $token)) { $epoch = $this->cache->load($epochKey, self::TTL); if ($epoch === false || $epoch === null) { return false; } - throw new RuntimeException("Failed to register document cache owner '{$token}' in '{$ownersKey}'"); + throw new RuntimeException("Failed to register document cache owner '{$token}' for '{$collectionKey}'"); } $startedKey = $collectionKey.'#started'; @@ -2633,16 +2633,16 @@ protected function activateDocumentInvalidation(array $tokens): void private function activateDocumentCacheEpoch(string $collectionKey, string $token): void { - $ownersKey = $collectionKey.'#owners'; - $owner = $this->cache->load($ownersKey, self::TTL, $token); + $registration = (new Owners($this->cache))->find($collectionKey, $token); + $owner = $this->cache->load($registration->key, self::TTL, $registration->field); if ($owner !== false && $owner !== null && $owner !== $token) { - throw new RuntimeException("Invalid document cache owner '{$token}' in '{$ownersKey}'"); + throw new RuntimeException("Invalid document cache owner '{$token}' for '{$collectionKey}'"); } $owned = $owner === $token; - if ($owned && ! $this->cache->purge($ownersKey, $token)) { - $owner = $this->cache->load($ownersKey, self::TTL, $token); + if ($owned && ! $this->cache->purge($registration->key, $registration->field)) { + $owner = $this->cache->load($registration->key, self::TTL, $registration->field); if ($owner !== false && $owner !== null) { - throw new RuntimeException("Failed to release document cache owner '{$token}' in '{$ownersKey}'"); + throw new RuntimeException("Failed to release document cache owner '{$token}' for '{$collectionKey}'"); } $owned = false; } diff --git a/tests/e2e/Adapter/Scopes/GeneralTests.php b/tests/e2e/Adapter/Scopes/GeneralTests.php index b2b3eb2e5..ed4d5367e 100644 --- a/tests/e2e/Adapter/Scopes/GeneralTests.php +++ b/tests/e2e/Adapter/Scopes/GeneralTests.php @@ -847,7 +847,7 @@ public function testNestedTransactionKeepsDocumentCacheEpochBlockedUntilOuterCom } } - public function testCacheInvalidationLeavesOneOwnerKeyPerCollection(): void + public function testCacheInvalidationDoesNotAddRedisKeysPerWrite(): void { $database = $this->getDatabase(); if (! $database->getAdapter()->supports(Capability::Caching)) { @@ -874,25 +874,19 @@ public function testCacheInvalidationLeavesOneOwnerKeyPerCollection(): void try { $database->createDocument($collection, new Document(['$id' => 'doc0', 'name' => 'doc 0'])); - $owners = $this->scanOwnerKeys($redis, $collection); + $keys = $this->scanKeys($redis, $collection); for ($index = 1; $index <= 10; $index++) { $database->createDocument($collection, new Document(['$id' => 'doc'.$index, 'name' => 'doc '.$index])); - $this->assertCount($index + 1, $database->find($collection, [Query::limit(100)])); } $database->withTransaction(function () use ($database, $collection): void { $database->updateDocument($collection, 'doc1', new Document(['name' => 'updated'])); $database->updateDocument($collection, 'doc2', new Document(['name' => 'updated'])); }); - $this->assertSame($owners, $this->scanOwnerKeys($redis, $collection), 'A purged key stays behind in Redis, so a write must not register its owner under a key of its own'); - $this->assertNotEmpty($owners); - foreach ($owners as $key) { - $this->assertStringEndsWith('#owners', $key); - $fields = $redis->hKeys($key); - $this->assertIsArray($fields); - $this->assertSame([], \array_values(\array_diff($fields, ['__utopia_gen__', '__utopia_tomb__'])), "Every write must release its registration in '{$key}'"); - } + $this->assertNotEmpty($keys); + $this->assertSame($keys, $this->scanKeys($redis, $collection), 'Redis keeps a purged key with no expiry, so a write must not leave a key of its own'); + $this->assertCount(11, $database->find($collection, [Query::limit(100)])); } finally { $database->setQueryCache($queryCache)->setCache($original); if ($destination !== null && $destinationCache !== null) { @@ -1126,12 +1120,12 @@ public function testCacheReconnect(): void /** * @return array */ - private function scanOwnerKeys(Redis $redis, string $collection): array + private function scanKeys(Redis $redis, string $collection): array { $keys = []; $iterator = null; do { - $batch = $redis->scan($iterator, '*'.\strtolower($collection).'*#owner*', 1000); + $batch = $redis->scan($iterator, '*'.\strtolower($collection).'*', 1000); if (\is_array($batch)) { \array_push($keys, ...$batch); } diff --git a/tests/unit/Cache/OwnershipCache.php b/tests/unit/Cache/OwnershipCache.php index 460b8660c..3b7db5b58 100644 --- a/tests/unit/Cache/OwnershipCache.php +++ b/tests/unit/Cache/OwnershipCache.php @@ -8,7 +8,7 @@ final class OwnershipCache implements CacheAdapter, Leasable { - /** @var array|string}>> */ + /** @var array|string}> */ private array $store = []; /** @var array */ @@ -22,7 +22,7 @@ final class OwnershipCache implements CacheAdapter, Leasable public function load(string $key, int $ttl, string $hash = ''): mixed { - $saved = $this->store[$key][$this->field($key, $hash)] ?? null; + $saved = $this->store[$key] ?? null; return $saved !== null && $saved['time'] + $ttl > \time() ? $saved['data'] : false; } @@ -44,7 +44,7 @@ public function save(string $key, array|string $data, string $hash = ''): bool|s $activation(); } - $this->store[$key][$this->field($key, $hash)] = ['time' => \time(), 'data' => $data]; + $this->store[$key] = ['time' => \time(), 'data' => $data]; return $data; } @@ -65,12 +65,11 @@ public function saveWithLease(string $key, array|string $data, string $hash, str public function touch(string $key, string $hash = ''): bool { - $field = $this->field($key, $hash); - if (! isset($this->store[$key][$field])) { + if (! isset($this->store[$key])) { return false; } - $this->store[$key][$field]['time'] = \time(); + $this->store[$key]['time'] = \time(); return true; } @@ -93,11 +92,7 @@ public function purge(string $key, string $hash = ''): bool } $this->generations[$key] = ($this->generations[$key] ?? 0) + 1; - if ($hash === '') { - unset($this->store[$key]); - } else { - unset($this->store[$key][$hash]); - } + unset($this->store[$key]); return true; } @@ -125,9 +120,9 @@ public function getName(?string $key = null): string return 'ownership'; } - public function has(string $key, string $hash = ''): bool + public function has(string $key): bool { - return isset($this->store[$key][$this->field($key, $hash)]); + return isset($this->store[$key]); } public function pauseNextActivation(Closure $activation): void @@ -144,9 +139,4 @@ public function failDuringActivation(): void { $this->failDuringActivation = true; } - - private function field(string $key, string $hash): string - { - return $hash === '' ? $key : $hash; - } } diff --git a/tests/unit/Cache/QueryCacheTest.php b/tests/unit/Cache/QueryCacheTest.php index 3aee3d8f9..46b040efa 100644 --- a/tests/unit/Cache/QueryCacheTest.php +++ b/tests/unit/Cache/QueryCacheTest.php @@ -448,18 +448,18 @@ public function testConcurrentOwnersCannotEnableCacheEarly(): void $second->blockCollection($key, $secondToken); $this->assertNull($reader->getEntry($scope, 'users', [])); - $this->assertTrue($adapter->has($key.'#owners', $secondToken)); + $this->assertTrue($adapter->has($key.'#owner:'.$secondToken)); }); $first->activateCollection($key, $firstToken); - $this->assertFalse($adapter->has($key.'#owners', $firstToken)); - $this->assertTrue($adapter->has($key.'#owners', $secondToken)); + $this->assertFalse($adapter->has($key.'#owner:'.$firstToken)); + $this->assertTrue($adapter->has($key.'#owner:'.$secondToken)); $this->assertNull($reader->getEntry($scope, 'users', [])); $second->activateCollection($key, $secondToken); - $this->assertFalse($adapter->has($key.'#owners', $secondToken)); + $this->assertFalse($adapter->has($key.'#owner:'.$secondToken)); $fresh = $reader->getEntry($scope, 'users', []); $this->assertNotNull($fresh); $this->assertNull($reader->get($fresh)); @@ -564,37 +564,50 @@ public function testActivationPurgeFailureStillPropagates(): void $this->assertNull($queryCache->getEntry(new Scope(), 'users', [])); } - public function testInvalidationsLeaveOneOwnerKeyPerCollection(): void + public function testInvalidationsDoNotAddKeysToACacheThatKeepsPurgedKeys(): void { $adapter = new RedisLeasableCache(); $queryCache = new QueryCache(new Cache($adapter)); $scope = new Scope(namespace: 'ns'); - $key = $queryCache->getCollectionKey($scope, 'users'); + $queryCache->invalidateCollection($scope, 'users'); + $keys = $adapter->keys(); for ($invalidation = 0; $invalidation < 20; $invalidation++) { $queryCache->invalidateCollection($scope, 'users'); } - $this->assertSame( - [$key.'#owners'], - \array_values(\array_filter($adapter->keys(), static fn (string $stored): bool => \str_contains($stored, '#owner'))), - 'A purged key stays behind in Redis, so invalidations must share one owner key instead of leaving one each', - ); - $this->assertSame([], $adapter->list($key.'#owners'), 'Every invalidation must release its registration'); + $this->assertSame($keys, $adapter->keys(), 'A purged key stays behind in Redis, so an invalidation must not leave a key of its own'); $this->assertNotNull($queryCache->getEntry($scope, 'users', [])); } - public function testActivationRejectsAForeignOwnerRegistration(): void + public function testOverlappingInvalidationsSucceedOnACacheWithoutFields(): void { - $cache = new Cache(new RedisLeasableCache()); - $queryCache = new QueryCache($cache); + $queryCache = new QueryCache(new Cache(new Memory())); + $key = $queryCache->getCollectionKey(new Scope(), 'users'); + + $queryCache->blockCollection($key, 'first'); + $queryCache->blockCollection($key, 'second'); + $queryCache->activateCollection($key, 'first'); + $this->assertNull($queryCache->getEntry(new Scope(), 'users', []), 'The second writer is still in flight'); + $queryCache->activateCollection($key, 'second'); + + $this->assertNull( + $queryCache->getEntry(new Scope(), 'users', []), + 'A cache without generations cannot prove a new epoch fresh, so it stays blocked', + ); + } + + public function testActivationRejectsACorruptedOwnerRegistration(): void + { + $adapter = new RedisLeasableCache(); + $queryCache = new QueryCache(new Cache($adapter)); $key = $queryCache->getCollectionKey(new Scope(), 'users'); + $adapter->corruptFieldWrites(); $queryCache->blockCollection($key, 'owner'); - $cache->save($key.'#owners', 'intruder', 'owner'); try { $queryCache->activateCollection($key, 'owner'); - $this->fail('A foreign owner registration was accepted'); + $this->fail('A corrupted owner registration was accepted'); } catch (\RuntimeException $error) { $this->assertStringContainsString('Invalid query cache owner', $error->getMessage()); } @@ -608,7 +621,7 @@ public function testActivationPropagatesAnOwnerReleaseFailure(): void $queryCache = new QueryCache(new Cache($adapter)); $key = $queryCache->getCollectionKey(new Scope(), 'users'); $queryCache->blockCollection($key, 'owner'); - $adapter->failFieldPurges('#owners'); + $adapter->failFieldPurges(); try { $queryCache->activateCollection($key, 'owner'); diff --git a/tests/unit/Cache/RedisLeasableCache.php b/tests/unit/Cache/RedisLeasableCache.php index 080875fe7..a28fca95e 100644 --- a/tests/unit/Cache/RedisLeasableCache.php +++ b/tests/unit/Cache/RedisLeasableCache.php @@ -19,7 +19,9 @@ final class RedisLeasableCache implements CacheAdapter, Leasable /** @var array */ private array $generations = []; - private ?string $failingFieldPurges = null; + private bool $failingFieldPurges = false; + + private bool $corruptingFieldWrites = false; public function load(string $key, int $ttl, string $hash = ''): mixed { @@ -34,6 +36,10 @@ public function save(string $key, array|string $data, string $hash = ''): bool|s return false; } + if ($hash !== '' && $this->corruptingFieldWrites) { + $data = 'corrupted'; + } + $this->fields[$key][$this->field($key, $hash)] = ['time' => \time(), 'data' => $data]; return $data; @@ -73,7 +79,7 @@ public function list(string $key): array public function purge(string $key, string $hash = ''): bool { - if ($hash !== '' && $this->failingFieldPurges !== null && \str_ends_with($key, $this->failingFieldPurges)) { + if ($hash !== '' && $this->failingFieldPurges) { return false; } @@ -132,11 +138,19 @@ public function keys(): array } /** - * Fail every field purge of a key ending with $suffix, leaving the field in place. + * Fail every purge of a single field, leaving the field in place. + */ + public function failFieldPurges(): void + { + $this->failingFieldPurges = true; + } + + /** + * Store a different value than the one given on every write to a single field. */ - public function failFieldPurges(string $suffix): void + public function corruptFieldWrites(): void { - $this->failingFieldPurges = $suffix; + $this->corruptingFieldWrites = true; } private function field(string $key, string $hash): string diff --git a/tests/unit/Documents/DocumentCacheEpochTest.php b/tests/unit/Documents/DocumentCacheEpochTest.php index c9092c683..3eabcef0e 100644 --- a/tests/unit/Documents/DocumentCacheEpochTest.php +++ b/tests/unit/Documents/DocumentCacheEpochTest.php @@ -249,7 +249,7 @@ public function testActivationPurgeFailureStillPropagates(): void $this->assertSame('updated', $database->getDocument('webhooks', 'hook')->getAttribute('name')); } - public function testWritesLeaveOneOwnerKeyPerCollection(): void + public function testWritesDoNotAddKeysToACacheThatKeepsPurgedKeys(): void { $cache = new RedisLeasableCache(); $database = $this->createDatabaseWithCache($cache); @@ -258,7 +258,7 @@ public function testWritesLeaveOneOwnerKeyPerCollection(): void '$id' => 'hook0', 'name' => 'hook 0', ])); - $owners = $this->getOwnerKeys($cache); + $keys = $cache->keys(); for ($index = 1; $index <= 20; $index++) { $database->createDocument('webhooks', new Document([ @@ -271,16 +271,39 @@ public function testWritesLeaveOneOwnerKeyPerCollection(): void $database->updateDocument('webhooks', 'hook2', new Document(['name' => 'updated'])); }); - $this->assertSame($owners, $this->getOwnerKeys($cache), 'A purged key stays behind in Redis, so a write must not register its owner under a key of its own'); - [$collectionKey] = $database->getCacheKeys('webhooks'); - $this->assertContains(\strtolower($collectionKey.'#owners'), $owners); - foreach ($owners as $key) { - $this->assertSame([], $cache->list($key), "Every write must release its registration in '{$key}'"); - } + $this->assertSame($keys, $cache->keys(), 'A purged key stays behind in Redis, so a write must not leave a key of its own'); $this->assertSame('updated', $database->getDocument('webhooks', 'hook1')->getAttribute('name')); } - public function testActivationRejectsAForeignOwnerRegistration(): void + public function testOverlappingWritesSucceedOnACacheWithoutFields(): void + { + $cache = new FailDocumentEpochMemory(); + $writer = $this->createDatabaseWithCache($cache); + $other = $this->createDatabaseWithCache($cache, $writer->getNamespace()); + foreach ([$writer, $other] as $database) { + $database->createDocument('webhooks', new Document([ + '$id' => 'hook', + 'name' => 'original', + ])); + } + + $writer->withTransaction(function () use ($writer, $other, $cache): void { + $writer->updateDocument('webhooks', 'hook', new Document(['name' => 'updated'])); + $cache->failBlocks(); + try { + $other->updateDocument('webhooks', 'hook', new Document(['name' => 'updated'])); + $this->fail('The other writer\'s block did not fail'); + } catch (\RuntimeException $error) { + $this->assertStringContainsString('block document cache epoch', $error->getMessage()); + } finally { + $cache->failBlocks(false); + } + }); + + $this->assertSame('updated', $writer->getDocument('webhooks', 'hook')->getAttribute('name')); + } + + public function testActivationRejectsACorruptedOwnerRegistration(): void { $cache = new RedisLeasableCache(); $database = $this->createDatabaseWithCache($cache); @@ -288,21 +311,16 @@ public function testActivationRejectsAForeignOwnerRegistration(): void '$id' => 'hook', 'name' => 'original', ])); - [$collectionKey] = $database->getCacheKeys('webhooks', 'hook'); - $ownersKey = \strtolower($collectionKey.'#owners'); + $cache->corruptFieldWrites(); try { - $database->withTransaction(function () use ($database, $cache, $ownersKey): void { - $database->updateDocument('webhooks', 'hook', new Document(['name' => 'updated'])); - foreach ($cache->list($ownersKey) as $token) { - $cache->save($ownersKey, 'intruder', $token); - } - }); - $this->fail('A foreign document cache owner registration was accepted'); + $database->updateDocument('webhooks', 'hook', new Document(['name' => 'updated'])); + $this->fail('A corrupted document cache owner registration was accepted'); } catch (\RuntimeException $error) { $this->assertStringContainsString('Invalid document cache owner', $error->getMessage()); } + [$collectionKey] = $database->getCacheKeys('webhooks', 'hook'); $this->assertDocumentCacheEpochBlocked($database, $collectionKey); $this->assertSame('updated', $database->getDocument('webhooks', 'hook')->getAttribute('name')); } @@ -315,7 +333,7 @@ public function testActivationPropagatesAnOwnerReleaseFailure(): void '$id' => 'hook', 'name' => 'original', ])); - $cache->failFieldPurges('#owners'); + $cache->failFieldPurges(); try { $database->updateDocument('webhooks', 'hook', new Document(['name' => 'updated'])); @@ -329,17 +347,6 @@ public function testActivationPropagatesAnOwnerReleaseFailure(): void $this->assertSame('updated', $database->getDocument('webhooks', 'hook')->getAttribute('name')); } - /** - * @return array - */ - private function getOwnerKeys(RedisLeasableCache $cache): array - { - return \array_values(\array_filter( - $cache->keys(), - static fn (string $key): bool => \str_contains($key, '#owner'), - )); - } - private function assertDocumentCacheEpochBlocked(Database $database, string $collectionKey): void { $epoch = $database->getCache()->load($collectionKey.'#epoch', Database::TTL); @@ -358,12 +365,12 @@ private function createDatabase(): array return [$database, $adapter]; } - private function createDatabaseWithCache(CacheAdapter $cache): Database + private function createDatabaseWithCache(CacheAdapter $cache, ?string $namespace = null): Database { $database = new Database(new DatabaseMemory(), new Cache($cache)); $database ->setDatabase('utopiaTests') - ->setNamespace('epoch_'.\uniqid()); + ->setNamespace($namespace ?? 'epoch_'.\uniqid()); $database->create(); $database->createCollection(new Collection(id: 'webhooks', attributes: [ Attribute::string(key: 'name'), @@ -480,9 +487,9 @@ final class FailDocumentEpochMemory extends MemoryCache private ?string $activationFailure = null; - public function failBlocks(): void + public function failBlocks(bool $failing = true): void { - $this->failingBlocks = true; + $this->failingBlocks = $failing; } public function failActivations(?string $key = null): void