From b42aaa278da679a0df8a54703c4cdf9b33301f42 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 28 Sep 2026 15:07:03 +1300 Subject: [PATCH 1/3] fix(cache): stop mistaking a cache flush for a lost invalidation purge activateDocumentCacheEpoch() read the #started and #finished generations, published the new epoch, purged #finished, and threw "Failed to finish document cache invalidation" when #finished still held the generation it had read. A cache flush resets every generation, so a flush landing after that read let the purge bring #finished back to exactly that value: from 0 to 1 whenever the collection had finished one invalidation since the last flush. The only guard was that the epoch must still hold the value just published, and that value was written after the flush, so it survived and the committed write threw. Database::delete() flushes the whole cache, and a lane's four paratest workers share one Redis database, so sibling workers' flushes (9 to 20 per MariaDB lane run) made deleteCollection() in testKeywords throw intermittently. Under an injected FLUSHDB loop the MariaDB and SharedTables/MariaDB lanes failed the same way. Purge #finished before publishing the new epoch, and report a lost purge only while the epoch read alongside the generations is still in place. Every epoch value is a fresh random token, so it survives only if nothing flushed the cache in between. A purge that is actually lost still throws, and the activation makes the same number of cache round trips. QueryCache::activateCollection() already proves a lost purge against its own tombstone the same way. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 ++ src/Database/Traits/Documents.php | 30 +++++++-------- .../unit/Documents/DocumentCacheEpochTest.php | 37 ++++++++++++++++++- 3 files changed, 53 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 822c3aede..bf0047315 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -327,6 +327,9 @@ not change anything for an upgrade from 7.x. dropped, as the SQL adapters do, instead of throwing `Exception\Structure`. - The Redis adapter throws `Exception\Unique`, not a plain `Duplicate`, for unique index violations. - Custom types stay on the handles that share their `TypeRegistry` instead of replacing global filters. + - A write no longer throws `Failed to finish document cache invalidation` after it commits when the cache is + flushed while the write invalidates its collection's cached documents, for example by `delete()` of another + database that shares the cache. - **SQL adapters:** - A transparent reconnect of `Utopia\Database\PDO` keeps MariaDB and MySQL statement timeouts; the timeout also applies to the statement retried after the reconnect. diff --git a/src/Database/Traits/Documents.php b/src/Database/Traits/Documents.php index c977dd998..d5daf0130 100644 --- a/src/Database/Traits/Documents.php +++ b/src/Database/Traits/Documents.php @@ -2669,25 +2669,23 @@ private function activateDocumentCacheEpoch(string $collectionKey, string $token return; } - $active = \bin2hex(\random_bytes(16)); - if ($this->cache->save($epochKey, $active) === false) { - throw new RuntimeException("Failed to activate document cache epoch '{$epochKey}'"); - } + if ($started !== $finished) { + $this->cache->purge($finishedKey); - if ($started === $finished) { - return; - } - - $this->cache->purge($finishedKey); - if ($this->cache->getGeneration($finishedKey) === $finished) { - $nextStarted = $this->cache->getGeneration($startedKey); - $nextFinished = $this->cache->getGeneration($finishedKey); - $nextEpoch = $this->cache->load($epochKey, self::TTL); - if ($nextStarted === $nextFinished || $nextEpoch !== $active) { - return; + // A cache flush restarts generations, so an unchanged #finished + // only proves this purge was lost while the epoch read with it is + // still in place; publish the new epoch after this check, not before. + if ( + $this->cache->getGeneration($finishedKey) === $finished + && \is_string($epoch) + && $this->cache->load($epochKey, self::TTL) === $epoch + ) { + throw new RuntimeException("Failed to finish document cache invalidation '{$epochKey}'"); } + } - throw new RuntimeException("Failed to finish document cache invalidation '{$epochKey}'"); + if ($this->cache->save($epochKey, \bin2hex(\random_bytes(16))) === false) { + throw new RuntimeException("Failed to activate document cache epoch '{$epochKey}'"); } } diff --git a/tests/unit/Documents/DocumentCacheEpochTest.php b/tests/unit/Documents/DocumentCacheEpochTest.php index 81cd5b7d8..749d4db6b 100644 --- a/tests/unit/Documents/DocumentCacheEpochTest.php +++ b/tests/unit/Documents/DocumentCacheEpochTest.php @@ -213,6 +213,28 @@ public function testCacheFlushDuringActivationDoesNotFailTheCommittedMutation(): $this->assertSame('updated', $database->getDocument('webhooks', 'hook')->getAttribute('name')); } + public function testCacheFlushAfterActivationReadsGenerationsDoesNotFailTheCommittedMutation(): void + { + $cache = new FlushDuringActivationMemory(); + $database = $this->createDatabaseWithCache($cache); + $this->assertTrue($cache->flush()); + $database->createDocument('webhooks', new Document([ + '$id' => 'hook', + 'name' => 'original', + ])); + + [$collectionKey] = $database->getCacheKeys('webhooks'); + $this->assertSame( + '1', + $database->getCache()->getGeneration($collectionKey.'#finished'), + "The flush and the activation's own purge must bring #finished back to the generation it read" + ); + $cache->flushAfterReading('collection:webhooks#finished'); + + $this->assertTrue($database->deleteCollection('webhooks')); + $this->assertTrue($database->getCollection('webhooks')->isEmpty()); + } + public function testActivationPurgeFailureStillPropagates(): void { $cache = new FlushDuringActivationMemory(); @@ -406,6 +428,8 @@ final class FlushDuringActivationMemory extends MemoryCache implements Leasable private bool $failDuringActivation = false; + private ?string $flushAfterReading = null; + public function flushDuringActivation(): void { $this->flushDuringActivation = true; @@ -416,9 +440,20 @@ public function failDuringActivation(): void $this->failDuringActivation = true; } + public function flushAfterReading(string $key): void + { + $this->flushAfterReading = $key; + } + public function getGeneration(string $key): string { - return (string) ($this->generations[$key] ?? 0); + $generation = (string) ($this->generations[$key] ?? 0); + if ($this->flushAfterReading !== null && \str_ends_with($key, $this->flushAfterReading)) { + $this->flushAfterReading = null; + $this->flush(); + } + + return $generation; } #[\Override] From 073b9bf0616248483701e7c3d290b235e98bfff7 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 28 Sep 2026 15:07:03 +1300 Subject: [PATCH 2/3] test(collections): scope testTransformations' hook to the table it rewrites The hook appended AND 1 = 0 to every document read, including the _metadata read getCollection() makes, so the test only passed while the docs definition was cached. A flush from another paratest worker between find() and getDocument() sent that read through the hook and failed the test with Collection not found: 13 of 15 runs under a FLUSHDB loop, with or without the cache fix. Rewriting only reads of the docs table keeps the test independent of the cache. Co-Authored-By: Claude Opus 5.5 --- tests/e2e/Adapter/Scopes/CollectionTests.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/CollectionTests.php b/tests/e2e/Adapter/Scopes/CollectionTests.php index 4323cad70..02177a0c4 100644 --- a/tests/e2e/Adapter/Scopes/CollectionTests.php +++ b/tests/e2e/Adapter/Scopes/CollectionTests.php @@ -1293,12 +1293,16 @@ public function testTransformations(): void $database->setMetadata('scope', 'api.users'); - $hook = new class () implements Transform { + $hook = new class ($database->getNamespace().'_docs') implements Transform { public string $query = ''; + public function __construct(private readonly string $table) + { + } + public function transform(Event $event, string $query): string { - if ($event !== Event::DocumentRead) { + if ($event !== Event::DocumentRead || ! \str_contains($query, $this->table)) { return $query; } From cfa339288b4caafad2ddbd998298d9fc0f903bb8 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 28 Sep 2026 15:18:13 +1300 Subject: [PATCH 3/3] test(cache): arrange the flush regression with purgeCachedCollection() The regression needs #finished at generation 1 when the flush lands, and asserted it by reading the cache double's generation. That coupled the test to how many invalidations createDocument() runs. One purgeCachedCollection() after the flush is exactly one invalidation, so it reaches the same state through the public API with nothing to assert about cache internals. The test still fails without the fix, with the same deleteCollection() stack. Co-Authored-By: Claude Opus 5.5 --- tests/unit/Documents/DocumentCacheEpochTest.php | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/tests/unit/Documents/DocumentCacheEpochTest.php b/tests/unit/Documents/DocumentCacheEpochTest.php index 749d4db6b..0699f1a43 100644 --- a/tests/unit/Documents/DocumentCacheEpochTest.php +++ b/tests/unit/Documents/DocumentCacheEpochTest.php @@ -218,17 +218,7 @@ public function testCacheFlushAfterActivationReadsGenerationsDoesNotFailTheCommi $cache = new FlushDuringActivationMemory(); $database = $this->createDatabaseWithCache($cache); $this->assertTrue($cache->flush()); - $database->createDocument('webhooks', new Document([ - '$id' => 'hook', - 'name' => 'original', - ])); - - [$collectionKey] = $database->getCacheKeys('webhooks'); - $this->assertSame( - '1', - $database->getCache()->getGeneration($collectionKey.'#finished'), - "The flush and the activation's own purge must bring #finished back to the generation it read" - ); + $this->assertTrue($database->purgeCachedCollection('webhooks')); $cache->flushAfterReading('collection:webhooks#finished'); $this->assertTrue($database->deleteCollection('webhooks'));