From 6eebc58a373a08bbff72a5ac15f4c640476f435f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 16:44:03 +1300 Subject: [PATCH 01/55] feat(joins): join on a to-many relationship attribute A one-to-many relationship attribute, and the child side of a two-way many-to-one, holds no column on its collection: the related collection stores this document's ID. appwrite rewrote such join conditions app-side (Databases/Joins.php) so `on('orders', '$id')` reads the related rows; the library now owns that rewrite. find, aggregate, count, sum and getDocument rewrite the condition to `$id .` on a copy of the join before validation, keeping the operator and alias. Only a condition that compares the attribute with the `$id` of the collection the relationship relates is rewritten. appwrite rewrote regardless of the joined collection or the right column, which silently changed the meaning of a join of another collection; those, the parent side of a many-to-one, one-to-one, many-to-many, a one-way many-to-one child side, a relationship without a two-way key and a column of an earlier join stay refused exactly as before. Co-Authored-By: Claude Opus 5.5 --- src/Database/Trait/Documents.php | 105 +++++++ tests/unit/Joins/RelationshipJoinTest.php | 334 ++++++++++++++++++++++ 2 files changed, 439 insertions(+) create mode 100644 tests/unit/Joins/RelationshipJoinTest.php diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 16e43b7f3..2cb72f145 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -60,6 +60,7 @@ use Utopia\Query\CursorDirection; use Utopia\Query\Method; use Utopia\Query\OrderDirection; +use Utopia\Query\Query as BaseQuery; use Utopia\Query\Schema\ColumnType; use Utopia\Query\Schema\IndexType; use WeakMap; @@ -493,6 +494,7 @@ private function readDocument(Collection $collection, string $id, array $queries $attributes = $collection->attributes(); $this->checkQueryTypes($queries); + $queries = $this->rewriteRelationshipJoins($collection, $queries); $joinedCollections = null; @@ -3811,6 +3813,7 @@ private function fetch(Collection $collection, array $queries, PermissionType $f $queryCacheQueries = $queries; $this->checkQueryTypes($queries); + $queries = $this->rewriteRelationshipJoins($collection, $queries); $joinedCollectionsById = null; @@ -4181,6 +4184,7 @@ public function count(string $collection, array $queries = [], ?int $max = null) $collection = $this->silent(fn () => $this->getCollection($collection)); $this->checkQueryTypes($queries); + $queries = $this->rewriteRelationshipJoins($collection, $queries); $joinedCollections = null; @@ -4237,6 +4241,7 @@ public function sum(string $collection, string $attribute, array $queries = [], $collection = $this->silent(fn () => $this->getCollection($collection)); $this->checkQueryTypes($queries); + $queries = $this->rewriteRelationshipJoins($collection, $queries); $joinedCollections = null; @@ -4864,6 +4869,106 @@ private function validateDocumentsQueries(Document $collection, array $queries, } } + /** + * A one-to-many relationship attribute, and the child side of a two-way many-to-one, holds no column: the + * related collection stores this document's ID. A join of that related collection comparing such an attribute + * with the related `$id` is rewritten to compare this document's `$id` with the column storing it, keeping the + * operator and alias. The rewritten joins are copies; the caller's queries are left as given. + * + * @param array $queries + * @return array + */ + private function rewriteRelationshipJoins(Collection $collection, array $queries): array + { + $relationships = null; + + foreach ($queries as $index => $query) { + if (! $query->getMethod()->isJoin()) { + continue; + } + + $relationships ??= self::joinableRelationships($collection); + if ($relationships === []) { + return $queries; + } + + $joined = $query->getAttribute(); + $alias = $query->getAlias(); + $rewritten = null; + + foreach ($query->getJoinOnQueries() as $position => $condition) { + $column = self::relationshipJoinColumn($condition, $joined, $alias, $relationships); + if ($column === null) { + continue; + } + + $rewritten ??= clone $query; + $rewritten->getJoinOnQueries()[$position]->setValues([Document::ID, $condition->getValues()[1] ?? '=', $alias.'.'.$column]); + } + + if ($rewritten !== null) { + $queries[$index] = $rewritten; + } + } + + return $queries; + } + + /** + * The relationship attributes whose ID the related collection stores, by key. + * + * @return array + */ + private static function joinableRelationships(Collection $collection): array + { + $relationships = []; + foreach (self::relationshipAttributes($collection) as $attribute) { + $relationship = $attribute->relationship; + if ($relationship === null || $relationship->twoWayKey === null || $relationship->twoWayKey === '') { + continue; + } + + $storedByRelated = match ($relationship->type) { + RelationshipType::OneToMany => $attribute->side === RelationshipSide::Parent, + RelationshipType::ManyToOne => $attribute->side === RelationshipSide::Child && $relationship->twoWay, + RelationshipType::OneToOne, RelationshipType::ManyToMany => false, + }; + + if ($storedByRelated) { + $relationships[$attribute->key] = $attribute; + } + } + + return $relationships; + } + + /** + * The related column a join condition compares with this document's `$id`, or null when the condition does not + * compare a relationship attribute of this collection with the `$id` of the collection that relationship relates. + * + * @param array $relationships + */ + private static function relationshipJoinColumn(BaseQuery $condition, string $joined, string $alias, array $relationships): ?string + { + if ($condition->getMethod() !== Method::On) { + return null; + } + + $values = $condition->getValues(); + $left = $values[0] ?? null; + $right = $values[2] ?? null; + if (! \is_string($left) || ($right !== Document::ID && $right !== $alias.'.'.Document::ID)) { + return null; + } + + $relationship = ($relationships[$left] ?? null)?->relationship; + if ($relationship === null || $relationship->relatedCollection !== $joined) { + return null; + } + + return $relationship->twoWayKey; + } + /** * The collections the join queries name, each loaded once, by the id the join names it with. A * read resolves them once and hands them to validation, authorization, the adapter and decoding. diff --git a/tests/unit/Joins/RelationshipJoinTest.php b/tests/unit/Joins/RelationshipJoinTest.php new file mode 100644 index 000000000..9b8d20641 --- /dev/null +++ b/tests/unit/Joins/RelationshipJoinTest.php @@ -0,0 +1,334 @@ +database = new Database(new SQLite(new PDO('sqlite::memory:')), new Cache(new NoCache())); + $this->database + ->setDatabase('relationship_join') + ->setNamespace('relationship_join_'.\uniqid()) + ->setAuthorization(new Authorization()); + $this->database->addHook(new Relationships()); + $this->database->addHook(new Permissions()); + $this->database->create(); + + $this->createCollection('customers', [Attribute::string(key: 'name', size: 16)]); + $this->createCollection('orders', [ + Attribute::integer(key: 'amount'), + Attribute::string(key: 'status', size: 16), + Attribute::string(key: 'region', size: 16), + ]); + $this->createCollection('regions', [Attribute::string(key: 'name', size: 16)]); + + $this->database->createRelationship('customers', Relationship::oneToMany(relatedCollection: 'orders', key: 'orders', twoWayKey: 'customer')); + $this->database->createRelationship('customers', Relationship::manyToOne(relatedCollection: 'regions', key: 'region', twoWay: true, twoWayKey: 'customers')); + + $this->createDocument('regions', 'r1', ['name' => 'north']); + $this->createDocument('regions', 'r2', ['name' => 'south']); + + foreach ([ + 'o1' => [10, 'paid', 'r1'], + 'o2' => [20, 'open', 'r1'], + 'o3' => [30, 'paid', 'r1'], + 'o4' => [40, 'paid', 'r2'], + ] as $id => [$amount, $status, $region]) { + $this->createDocument('orders', $id, ['amount' => $amount, 'status' => $status, 'region' => $region]); + } + + $this->createDocument('customers', 'c1', ['name' => 'Ada', 'region' => 'r1', 'orders' => ['o1', 'o2']]); + $this->createDocument('customers', 'c2', ['name' => 'Bob', 'region' => 'r2', 'orders' => ['o3', 'o4']]); + $this->createDocument('customers', 'c3', ['name' => 'Cy', 'region' => 'r1']); + } + + public function testLeftJoinOnAOneToManyRelationshipPairsEachCustomerWithItsOrders(): void + { + $this->assertSame( + ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4', 'c3/-'], + $this->pairs('customers', [Query::leftJoin('orders', 'ord', [Query::on('orders', '$id')])], 'ord'), + ); + } + + public function testJoinOnAOneToManyRelationshipMatchesTheMainIdAgainstTheRelatedKey(): void + { + $this->assertSame( + ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], + $this->pairs('customers', [Query::join('orders', 'ord', [Query::on('orders', '$id')])], 'ord'), + 'a one-to-many relationship is stored on the related side, in its two-way key', + ); + } + + public function testJoinNamingTheRelatedIdUnderItsAliasIsRewritten(): void + { + $this->assertSame( + ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], + $this->pairs('customers', [Query::join('orders', 'ord', [Query::on('orders', 'ord.$id')])], 'ord'), + ); + } + + public function testRewrittenJoinKeepsItsAlias(): void + { + $rows = $this->database->find('customers', [ + Query::join('orders', 'purchase', [Query::on('orders', '$id')]), + Query::select(['name', 'purchase.amount']), + Query::orderAsc('purchase.amount'), + ]); + + $this->assertSame([10, 20, 30, 40], \array_map(static fn (Document $row): mixed => $row->getAttribute('purchase.amount'), $rows)); + $this->assertSame(['Ada', 'Ada', 'Bob', 'Bob'], \array_map(static fn (Document $row): mixed => $row->getAttribute('name'), $rows)); + } + + public function testOnlyTheRelationshipConditionOfTheOnListIsRewritten(): void + { + $this->assertSame( + ['c1/o1', 'c2/o4'], + $this->pairs('customers', [Query::join('orders', 'ord', [ + Query::on('orders', '$id'), + Query::on('region', 'region'), + Query::equal('ord.status', ['paid']), + ])], 'ord'), + 'the many-to-one column and the ON filter keep comparing what they named', + ); + } + + public function testOperatorOfTheRewrittenConditionIsKept(): void + { + $this->assertSame( + ['c1/o3', 'c1/o4', 'c2/o1', 'c2/o2', 'c3/o1', 'c3/o2', 'c3/o3', 'c3/o4'], + $this->pairs('customers', [Query::join('orders', 'ord', [Query::on('orders', '$id', '!=')])], 'ord'), + ); + } + + public function testConditionOnAnEarlierJoinsColumnIsNotTheMainRelationship(): void + { + $this->expectException(QueryException::class); + $this->expectExceptionMessage(self::VIRTUAL.'peer.orders'); + + $this->database->find('customers', [ + Query::join('customers', 'peer', [Query::on('$id', '$id')]), + Query::join('orders', 'sub', [Query::on('peer.orders', '$id')]), + ]); + } + + public function testOneToManyWithoutATwoWayKeyIsNotRewritten(): void + { + $this->createCollection('ledgers', [Attribute::relationship('orders', Relationship::oneToMany(relatedCollection: 'orders'), RelationshipSide::Parent)]); + + $this->expectException(QueryException::class); + $this->expectExceptionMessage(self::VIRTUAL.'orders'); + + $this->database->find('ledgers', [Query::join('orders', 'ord', [Query::on('orders', '$id')])]); + } + + public function testRelationshipDeclaredInTheCollectionDefinitionIsRewritten(): void + { + $this->createCollection('areas', [ + Attribute::string(key: 'name', size: 16), + Attribute::relationship('orders', Relationship::oneToMany(relatedCollection: 'orders', twoWayKey: 'region'), RelationshipSide::Parent), + ]); + $this->createDocument('areas', 'r1', ['name' => 'north']); + $this->createDocument('areas', 'r2', ['name' => 'south']); + + $this->assertSame( + ['r1/o1', 'r1/o2', 'r1/o3', 'r2/o4'], + $this->pairs('areas', [Query::join('orders', 'ord', [Query::on('orders', '$id')])], 'ord'), + ); + } + + public function testJoinOnAManyToOneRelationshipKeepsItsColumns(): void + { + $this->assertSame( + ['c1/r1', 'c2/r2', 'c3/r1'], + $this->pairs('customers', [Query::join('regions', 'reg', [Query::on('region', '$id')])], 'reg'), + 'the parent side of a many-to-one holds the related ID in its own column', + ); + } + + public function testJoinOnTheChildSideOfATwoWayManyToOneMatchesTheMainIdAgainstTheParentKey(): void + { + $this->assertSame( + ['r1/c1', 'r1/c3', 'r2/c2'], + $this->pairs('regions', [Query::leftJoin('customers', 'resident', [Query::on('customers', '$id')])], 'resident'), + ); + } + + public function testJoinOnTheChildSideOfAOneWayManyToOneIsRefused(): void + { + $this->createCollection('warehouses', [Attribute::string(key: 'name', size: 16)]); + $this->database->createRelationship('orders', Relationship::manyToOne(relatedCollection: 'warehouses', key: 'warehouse', twoWayKey: 'stock')); + + $this->expectException(QueryException::class); + $this->expectExceptionMessage(self::VIRTUAL.'stock'); + + $this->database->find('warehouses', [Query::join('orders', 'ord', [Query::on('stock', '$id')])]); + } + + public function testJoinOfAnotherCollectionThanTheRelatedOneIsRefused(): void + { + $this->expectException(QueryException::class); + $this->expectExceptionMessage(self::VIRTUAL.'orders'); + + $this->database->find('customers', [Query::join('regions', 'reg', [Query::on('orders', '$id')])]); + } + + public function testRelationshipComparedWithAnotherColumnThanTheRelatedIdIsRefused(): void + { + $this->expectException(QueryException::class); + $this->expectExceptionMessage(self::VIRTUAL.'orders'); + + $this->database->find('customers', [Query::join('orders', 'ord', [Query::on('orders', 'status')])]); + } + + public function testJoinOnAManyToManyRelationshipIsRefused(): void + { + $this->createCollection('tags', [Attribute::string(key: 'name', size: 16)]); + $this->database->createRelationship('customers', Relationship::manyToMany(relatedCollection: 'tags', key: 'tags', twoWay: true, twoWayKey: 'customers')); + + $this->expectException(QueryException::class); + $this->expectExceptionMessage(self::VIRTUAL.'tags'); + + $this->database->find('customers', [Query::join('tags', 'tag', [Query::on('tags', '$id')])]); + } + + public function testJoinOnTheChildSideOfAOneWayOneToOneIsRefused(): void + { + $this->createCollection('profiles', [Attribute::string(key: 'name', size: 16)]); + $this->database->createRelationship('customers', Relationship::oneToOne(relatedCollection: 'profiles', key: 'profile', twoWayKey: 'owner')); + + $this->expectException(QueryException::class); + $this->expectExceptionMessage(self::VIRTUAL.'owner'); + + $this->database->find('profiles', [Query::join('customers', 'customer', [Query::on('owner', '$id')])]); + } + + /** + * @return iterable + */ + public static function reads(): iterable + { + $join = Query::join('orders', 'ord', [Query::on('orders', '$id')]); + + yield 'count' => [static fn (Database $database): int => $database->count('customers', [$join]), 4]; + yield 'sum' => [static fn (Database $database): int => (int) $database->sum('customers', 'ord.amount', [$join]), 100]; + yield 'aggregate' => [ + static fn (Database $database): array => \array_map( + static fn (array $row): array => [$row['name'], (int) $row['total']], + $database->aggregate('customers', [$join, Query::sum('ord.amount', 'total'), Query::groupBy(['name']), Query::orderAsc('name')]), + ), + [['Ada', 30], ['Bob', 70]], + ]; + yield 'getDocument' => [ + static fn (Database $database): mixed => $database->getDocument('customers', 'c1', [ + Query::join('orders', 'ord', [Query::on('orders', '$id'), Query::equal('ord.status', ['open'])]), + Query::select(['name', 'ord.amount']), + ])->getAttribute('ord.amount'), + 20, + ]; + } + + /** + * @param Closure(Database): mixed $read + */ + #[DataProvider('reads')] + public function testEveryReadOfARelationshipJoinMatchesTheRelatedKey(Closure $read, mixed $expected): void + { + $this->assertSame($expected, $read($this->database)); + } + + public function testRelationshipJoinIsRewrittenWithoutValidation(): void + { + $pairs = $this->database->skipValidation(fn (): array => $this->pairs('customers', [Query::join('orders', 'ord', [Query::on('orders', '$id')])], 'ord')); + + $this->assertSame(['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], $pairs); + } + + public function testCallersJoinIsLeftAsGivenForTheNextRead(): void + { + $condition = Query::on('orders', '$id'); + $join = Query::join('orders', 'ord', [$condition]); + + $this->assertSame(4, $this->database->count('customers', [$join])); + $this->assertSame(['orders', '=', '$id'], $condition->getValues()); + $this->assertSame($condition, $join->getJoinOnQueries()[0]); + $this->assertSame(['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], $this->pairs('customers', [$join], 'ord')); + } + + /** + * Each row as `
/`, sorted, with `-` for a row the outer join did not match. + * + * @param list $queries + * @return list + */ + private function pairs(string $collection, array $queries, string $alias): array + { + $pairs = \array_map( + static function (Document $row) use ($alias): string { + $joined = $row->getAttribute($alias.'.$id'); + + return $row->getId().'/'.(\is_string($joined) ? $joined : '-'); + }, + $this->database->find($collection, [...$queries, Query::limit(100)]), + ); + \sort($pairs); + + return $pairs; + } + + /** + * @param list $attributes + */ + private function createCollection(string $id, array $attributes): void + { + $this->database->createCollection(Collection::create( + id: $id, + attributes: $attributes, + permissions: [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], + )); + } + + /** + * @param array $attributes + */ + private function createDocument(string $collection, string $id, array $attributes): void + { + $this->database->createDocument($collection, new Document([ + '$id' => $id, + '$permissions' => [Permission::read(Role::any())], + ...$attributes, + ])); + } +} From 5f27c007ff522561d79791e510a3121660690ec3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 16:44:04 +1300 Subject: [PATCH 02/55] test(joins): cover relationship joins and the aggregate join read rule Exercises the relationship join rewrite on every Joins adapter, and pins the join read rule for aggregate(), which no test covered: a joined collection readable neither at collection nor at document level is refused, authorization disabled reads every row, and under document security an aggregate over the join reads the rows a direct aggregate of the joined collection would. The read-rule cases pass on 8.0.3; they are coverage for behaviour appwrite relied on app-side. Co-Authored-By: Claude Opus 5.5 --- tests/e2e/Adapter/Base.php | 2 + .../Adapter/Scopes/JoinRelationshipTests.php | 391 ++++++++++++++++++ 2 files changed, 393 insertions(+) create mode 100644 tests/e2e/Adapter/Scopes/JoinRelationshipTests.php diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 7e8058f9c..429c3a9a4 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -13,6 +13,7 @@ use Tests\E2E\Adapter\Scopes\GeneralTests; use Tests\E2E\Adapter\Scopes\IndexTests; use Tests\E2E\Adapter\Scopes\JoinComboTests; +use Tests\E2E\Adapter\Scopes\JoinRelationshipTests; use Tests\E2E\Adapter\Scopes\JoinTests; use Tests\E2E\Adapter\Scopes\MetadataCacheTests; use Tests\E2E\Adapter\Scopes\ObjectAttributeTests; @@ -51,6 +52,7 @@ abstract class Base extends TestCase use GeneralTests; use IndexTests; use JoinComboTests; + use JoinRelationshipTests; use JoinTests; use MetadataCacheTests; use ObjectAttributeTests; diff --git a/tests/e2e/Adapter/Scopes/JoinRelationshipTests.php b/tests/e2e/Adapter/Scopes/JoinRelationshipTests.php new file mode 100644 index 000000000..d13f54cfe --- /dev/null +++ b/tests/e2e/Adapter/Scopes/JoinRelationshipTests.php @@ -0,0 +1,391 @@ +getDatabase(); + if (! $this->supportsJoinRelationships($database)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $this->seedJoinRelationships($database); + + try { + $this->assertSame( + ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4', 'c3/-'], + $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, Query::leftJoin(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id')]), 'purchase'), + 'a left join keeps the customer without orders', + ); + $this->assertSame( + ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], + $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', 'purchase.$id')]), 'purchase'), + ); + $this->assertSame( + ['c1/o3', 'c1/o4', 'c2/o1', 'c2/o2', 'c3/o1', 'c3/o2', 'c3/o3', 'c3/o4'], + $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id', '!=')]), 'purchase'), + 'the operator of the condition is kept', + ); + $this->assertSame( + ['c1/o1', 'c2/o4'], + $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [ + Query::on('orders', '$id'), + Query::on('region', 'region'), + Query::equal('purchase.status', ['paid']), + ]), 'purchase'), + 'the other conditions of the ON list compare what they named', + ); + + $rows = $database->find(self::JOIN_RELATIONSHIP_CUSTOMERS, [ + Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id')]), + Query::select(['name', 'purchase.amount']), + Query::orderAsc('purchase.amount'), + ]); + $this->assertSame([10, 20, 30, 40], \array_map(static fn (Document $row): int => (int) $row->getAttribute('purchase.amount'), $rows)); + $this->assertSame(['Ada', 'Ada', 'Bob', 'Bob'], \array_map(static fn (Document $row): mixed => $row->getAttribute('name'), $rows)); + } finally { + $this->deleteJoinRelationshipCollections($database, ...self::JOIN_RELATIONSHIP_COLLECTIONS); + } + } + + public function testJoinRelationshipOnTheChildSideOfATwoWayManyToOneMatchesTheParentKey(): void + { + $database = $this->getDatabase(); + if (! $this->supportsJoinRelationships($database)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $this->seedJoinRelationships($database); + + try { + $this->assertSame( + ['r1/c1', 'r1/c3', 'r2/c2'], + $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_REGIONS, Query::join(self::JOIN_RELATIONSHIP_CUSTOMERS, 'resident', [Query::on('customers', '$id')]), 'resident'), + ); + $this->assertSame( + ['c1/r1', 'c2/r2', 'c3/r1'], + $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, Query::join(self::JOIN_RELATIONSHIP_REGIONS, 'area', [Query::on('region', '$id')]), 'area'), + 'the parent side holds the related ID in its own column', + ); + } finally { + $this->deleteJoinRelationshipCollections($database, ...self::JOIN_RELATIONSHIP_COLLECTIONS); + } + } + + public function testJoinRelationshipIsMatchedByEveryRead(): void + { + $database = $this->getDatabase(); + if (! $this->supportsJoinRelationships($database)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $this->seedJoinRelationships($database); + + try { + $join = Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id')]); + + $this->assertSame(4, $database->count(self::JOIN_RELATIONSHIP_CUSTOMERS, [$join])); + $this->assertSame(100, (int) $database->sum(self::JOIN_RELATIONSHIP_CUSTOMERS, 'purchase.amount', [$join])); + $this->assertSame( + [['Ada', 30], ['Bob', 70]], + \array_map( + static fn (array $row): array => [$row['name'], (int) $row['total']], + $database->aggregate(self::JOIN_RELATIONSHIP_CUSTOMERS, [$join, Query::sum('purchase.amount', 'total'), Query::groupBy(['name']), Query::orderAsc('name')]), + ), + ); + $this->assertSame( + 20, + (int) $database->getDocument(self::JOIN_RELATIONSHIP_CUSTOMERS, 'c1', [ + Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id'), Query::equal('purchase.status', ['open'])]), + Query::select(['name', 'purchase.amount']), + ])->getAttribute('purchase.amount'), + ); + } finally { + $this->deleteJoinRelationshipCollections($database, ...self::JOIN_RELATIONSHIP_COLLECTIONS); + } + } + + public function testJoinRelationshipKeepsRefusingAttributesWithoutAColumnToMatch(): void + { + $database = $this->getDatabase(); + if (! $this->supportsJoinRelationships($database)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $this->seedJoinRelationships($database); + + try { + $database->createCollection(Collection::create( + id: self::JOIN_RELATIONSHIP_WAREHOUSES, + attributes: [Attribute::string(key: 'name', size: 16)], + permissions: [Permission::create(Role::any()), Permission::read(Role::any())], + )); + $database->createRelationship(self::JOIN_RELATIONSHIP_ORDERS, Relationship::manyToOne(relatedCollection: self::JOIN_RELATIONSHIP_WAREHOUSES, key: 'warehouse', twoWayKey: 'stock')); + + $refusals = [ + 'the child side of a one-way many-to-one' => [self::JOIN_RELATIONSHIP_WAREHOUSES, Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('stock', '$id')]), 'stock'], + 'a join of another collection than the related one' => [self::JOIN_RELATIONSHIP_CUSTOMERS, Query::join(self::JOIN_RELATIONSHIP_REGIONS, 'area', [Query::on('orders', '$id')]), 'orders'], + 'a comparison with another column than the related $id' => [self::JOIN_RELATIONSHIP_CUSTOMERS, Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', 'status')]), 'orders'], + ]; + + foreach ($refusals as $case => [$collection, $join, $attribute]) { + try { + $database->find($collection, [$join]); + $this->fail("{$case} must be refused"); + } catch (QueryException $exception) { + $this->assertStringContainsString('Cannot join on virtual relationship attribute: '.$attribute, $exception->getMessage(), $case); + } + } + } finally { + $this->deleteJoinRelationshipCollections($database, ...self::JOIN_RELATIONSHIP_COLLECTIONS); + } + } + + public function testJoinRelationshipAggregateRefusesAJoinedCollectionWithoutGrantOrDocumentSecurity(): void + { + $database = $this->getDatabase(); + if (! $database->getAdapter()->supports(Capability::Joins) || ! $database->getAdapter()->supports(Capability::Aggregations)) { + $this->expectNotToPerformAssertions(); + + return; + } + + [$main, $joined] = $this->seedJoinRelationshipReadRule($database, 'jrel_deny', joinedPermissions: [Permission::create(Role::any())], joinedDocumentSecurity: false); + + try { + $queries = [Query::join($joined, 'entry', [Query::on('$id', 'mainId')]), Query::count('*', 'rows')]; + + $this->withJoinRelationshipRoles($database, [Role::any()->toString()], function () use ($database, $main, $queries): void { + try { + $database->aggregate($main, $queries); + $this->fail('aggregate() must refuse a joined collection readable neither at collection nor at document level'); + } catch (AuthorizationException $exception) { + $this->assertStringContainsString('jrel_deny_j', $exception->getMessage()); + } + }); + + $rows = $database->getAuthorization()->skip(fn (): array => $database->aggregate($main, $queries)); + $this->assertSame(2, (int) $rows[0]['rows'], 'with authorization disabled every joined row is read'); + } finally { + $this->deleteJoinRelationshipCollections($database, $main, $joined); + } + } + + public function testJoinRelationshipAggregateFiltersAJoinedCollectionPerDocument(): void + { + $database = $this->getDatabase(); + if (! $database->getAdapter()->supports(Capability::Joins) || ! $database->getAdapter()->supports(Capability::Aggregations)) { + $this->expectNotToPerformAssertions(); + + return; + } + + [$main, $joined] = $this->seedJoinRelationshipReadRule($database, 'jrel_owned', joinedPermissions: [Permission::create(Role::any())], joinedDocumentSecurity: true); + + try { + $this->withJoinRelationshipRoles($database, [Role::any()->toString(), Role::user('alice')->toString()], function () use ($database, $main, $joined): void { + $joinedRows = $database->aggregate($joined, [Query::count('*', 'rows'), Query::sum('amount', 'total')]); + $rows = $database->aggregate($main, [ + Query::join($joined, 'entry', [Query::on('$id', 'mainId')]), + Query::count('*', 'rows'), + Query::sum('entry.amount', 'total'), + ]); + + $this->assertSame([1, 5], [(int) $joinedRows[0]['rows'], (int) $joinedRows[0]['total']], 'a direct aggregate reads only the rows the caller holds read on'); + $this->assertSame([1, 5], [(int) $rows[0]['rows'], (int) $rows[0]['total']], 'an aggregate over the join reads the same rows'); + }); + } finally { + $this->deleteJoinRelationshipCollections($database, $main, $joined); + } + } + + private function supportsJoinRelationships(Database $database): bool + { + $adapter = $database->getAdapter(); + + return $adapter->supports(Capability::Joins) + && $adapter->supports(Capability::Aggregations) + && $adapter->hasFeature(Feature\Relationships::class); + } + + /** + * Customers c1 (region r1, orders o1 and o2), c2 (region r2, orders o3 and o4) and c3 (region r1, no orders). + */ + private function seedJoinRelationships(Database $database): void + { + $this->deleteJoinRelationshipCollections($database, ...self::JOIN_RELATIONSHIP_COLLECTIONS); + + $permissions = [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())]; + $database->createCollection(Collection::create( + id: self::JOIN_RELATIONSHIP_CUSTOMERS, + attributes: [Attribute::string(key: 'name', size: 16)], + permissions: $permissions, + )); + $database->createCollection(Collection::create( + id: self::JOIN_RELATIONSHIP_ORDERS, + attributes: [ + Attribute::integer(key: 'amount'), + Attribute::string(key: 'status', size: 16), + Attribute::string(key: 'region', size: 16), + ], + permissions: $permissions, + )); + $database->createCollection(Collection::create( + id: self::JOIN_RELATIONSHIP_REGIONS, + attributes: [Attribute::string(key: 'name', size: 16)], + permissions: $permissions, + )); + + $database->createRelationship(self::JOIN_RELATIONSHIP_CUSTOMERS, Relationship::oneToMany(relatedCollection: self::JOIN_RELATIONSHIP_ORDERS, key: 'orders', twoWayKey: 'customer')); + $database->createRelationship(self::JOIN_RELATIONSHIP_CUSTOMERS, Relationship::manyToOne(relatedCollection: self::JOIN_RELATIONSHIP_REGIONS, key: 'region', twoWay: true, twoWayKey: 'customers')); + + $documents = [ + [self::JOIN_RELATIONSHIP_REGIONS, 'r1', ['name' => 'north']], + [self::JOIN_RELATIONSHIP_REGIONS, 'r2', ['name' => 'south']], + [self::JOIN_RELATIONSHIP_ORDERS, 'o1', ['amount' => 10, 'status' => 'paid', 'region' => 'r1']], + [self::JOIN_RELATIONSHIP_ORDERS, 'o2', ['amount' => 20, 'status' => 'open', 'region' => 'r1']], + [self::JOIN_RELATIONSHIP_ORDERS, 'o3', ['amount' => 30, 'status' => 'paid', 'region' => 'r1']], + [self::JOIN_RELATIONSHIP_ORDERS, 'o4', ['amount' => 40, 'status' => 'paid', 'region' => 'r2']], + [self::JOIN_RELATIONSHIP_CUSTOMERS, 'c1', ['name' => 'Ada', 'region' => 'r1', 'orders' => ['o1', 'o2']]], + [self::JOIN_RELATIONSHIP_CUSTOMERS, 'c2', ['name' => 'Bob', 'region' => 'r2', 'orders' => ['o3', 'o4']]], + [self::JOIN_RELATIONSHIP_CUSTOMERS, 'c3', ['name' => 'Cy', 'region' => 'r1']], + ]; + + foreach ($documents as [$collection, $id, $attributes]) { + $database->createDocument($collection, new Document([ + '$id' => $id, + '$permissions' => [Permission::read(Role::any())], + ...$attributes, + ])); + } + } + + private function deleteJoinRelationshipCollections(Database $database, string ...$collections): void + { + foreach ($collections as $collection) { + if ($database->collectionExists($collection)) { + $database->deleteCollection($collection); + } + } + } + + /** + * Each row as `
/`, sorted, with `-` for a row the outer join did not match. + * + * @return list + */ + private function joinRelationshipPairs(Database $database, string $collection, Query $join, string $alias): array + { + $pairs = \array_map( + static function (Document $row) use ($alias): string { + $joined = $row->getAttribute($alias.'.$id'); + + return $row->getId().'/'.(\is_string($joined) ? $joined : '-'); + }, + $database->find($collection, [$join, Query::limit(100)]), + ); + \sort($pairs); + + return $pairs; + } + + /** + * A main collection every caller reads, with one document, and a joined collection holding two entries for it: + * one readable by user alice and one by user bob. + * + * @param list $joinedPermissions + * @return array{string, string} + */ + private function seedJoinRelationshipReadRule(Database $database, string $prefix, array $joinedPermissions, bool $joinedDocumentSecurity): array + { + $main = $prefix.'_m'; + $joined = $prefix.'_j'; + $this->deleteJoinRelationshipCollections($database, $main, $joined); + + $database->createCollection(Collection::create( + id: $main, + attributes: [Attribute::string(key: 'name', size: 16)], + permissions: [Permission::create(Role::any()), Permission::read(Role::any())], + )); + $database->createCollection(Collection::create( + id: $joined, + attributes: [Attribute::string(key: 'mainId', size: 36), Attribute::integer(key: 'amount')], + permissions: $joinedPermissions, + documentSecurity: $joinedDocumentSecurity, + )); + + $database->createDocument($main, new Document(['$id' => 'customer', 'name' => 'Customer', '$permissions' => [Permission::read(Role::any())]])); + $database->getAuthorization()->skip(function () use ($database, $joined): void { + $database->createDocument($joined, new Document(['$id' => 'alice-entry', 'mainId' => 'customer', 'amount' => 5, '$permissions' => [Permission::read(Role::user('alice'))]])); + $database->createDocument($joined, new Document(['$id' => 'bob-entry', 'mainId' => 'customer', 'amount' => 7, '$permissions' => [Permission::read(Role::user('bob'))]])); + }); + + return [$main, $joined]; + } + + /** + * @param list $roles + * @param callable(): void $callback + */ + private function withJoinRelationshipRoles(Database $database, array $roles, callable $callback): void + { + $authorization = $database->getAuthorization(); + $previous = $authorization->getRoles(); + $authorization->cleanRoles(); + foreach ($roles as $role) { + $authorization->addRole($role); + } + + try { + $callback(); + } finally { + $authorization->cleanRoles(); + foreach ($previous as $role) { + $authorization->addRole($role); + } + } + } +} From 03b3814b8edc04aa2e3758b0a8872fd1a0f6e002 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 16:45:43 +1300 Subject: [PATCH 03/55] feat(arithmetic): add exact rounding and decimal reading for integer columns Database 7.4.1 handed fractional amounts and limits on integer columns to the engine, and each engine stored the result rounded its own way. 8.0 refuses them, so appwrite reimplemented the 7.4.1 answers app-side (Appwrite\Databases\{Rounding,Decimal,Engine}). The library owns the engines, so the knowledge belongs here. Arithmetic is the value object an adapter will declare: the rounding of a counter change, an operator, a division by a whole number and a power, plus whether the engine refused fractional counter changes, which operators it computed exactly and which it refused without a limit. Rounding applies a fraction to a base without passing the base through a float, so integers beyond 2^53 stay exact. Decimal reads number text exactly at any size (merging appwrite's Decimal with the private decimal/order/isTiny readers of IntegerOperators) so huge or tiny exponents never expand into billions of digits. Nothing calls these yet; defaults are unchanged. Co-Authored-By: Claude Opus 5.5 --- src/Database/Arithmetic.php | 41 ++++++ src/Database/Arithmetic/Decimal.php | 183 +++++++++++++++++++++++++ src/Database/Arithmetic/Rounding.php | 69 ++++++++++ tests/unit/Arithmetic/DecimalTest.php | 126 +++++++++++++++++ tests/unit/Arithmetic/RoundingTest.php | 83 +++++++++++ 5 files changed, 502 insertions(+) create mode 100644 src/Database/Arithmetic.php create mode 100644 src/Database/Arithmetic/Decimal.php create mode 100644 src/Database/Arithmetic/Rounding.php create mode 100644 tests/unit/Arithmetic/DecimalTest.php create mode 100644 tests/unit/Arithmetic/RoundingTest.php diff --git a/src/Database/Arithmetic.php b/src/Database/Arithmetic.php new file mode 100644 index 000000000..9d8a716bf --- /dev/null +++ b/src/Database/Arithmetic.php @@ -0,0 +1,41 @@ + $exact + * @param list $refusedWithoutLimit + */ + public function __construct( + public Rounding $counter, + public Rounding $operators, + public Rounding $division, + public Rounding $power, + public bool $refusesCounterFractions = false, + public array $exact = [], + public array $refusedWithoutLimit = [], + ) { + } + + /** + * The rounding of an operator's result, by its method and whether its operand is a whole number. + */ + public function rounding(OperatorType $method, bool $whole): Rounding + { + return match (true) { + $method === OperatorType::Power => $this->power, + $method === OperatorType::Divide && $whole => $this->division, + default => $this->operators, + }; + } +} diff --git a/src/Database/Arithmetic/Decimal.php b/src/Database/Arithmetic/Decimal.php new file mode 100644 index 000000000..6a1b908fc --- /dev/null +++ b/src/Database/Arithmetic/Decimal.php @@ -0,0 +1,183 @@ + \strlen(BigInt::UNSIGNED_MAX)) { + return new self($negative, self::SATURATED, false); + } + + $whole = $point > 0 ? \str_pad(\substr($digits, 0, $point), $point, '0') : '0'; + $fractional = $point <= 0 || \ltrim(\substr($digits, $point), '0') !== ''; + + return new self($negative, $whole, $fractional); + } + + public function floor(): int|string + { + return $this->fractional && $this->negative + ? BigInt::subtract($this->truncated(), 1) + : BigInt::toNative($this->truncated()); + } + + public function ceil(): int|string + { + return $this->fractional && ! $this->negative + ? BigInt::add($this->truncated(), 1) + : BigInt::toNative($this->truncated()); + } + + /** + * A number as an integer numerator and the power of ten that divides it, from its text; null for text that is + * not numeric or a number whose leading digit is more than MAXIMUM_DIGITS powers of ten from the units. + * + * @return array{int|string, int}|null + */ + public static function fraction(int|float|string $value): ?array + { + if (\is_string($value) && ! \is_numeric($value)) { + return null; + } + + $order = self::order($value); + if ($order === null) { + return [0, 0]; + } + if (\abs($order) > self::MAXIMUM_DIGITS) { + return null; + } + + [$mantissa, $exponent] = self::split($value); + [$whole, $fraction] = \explode('.', $mantissa, 2) + [1 => '']; + $scale = \strlen($fraction) - (int) $exponent; + $digits = \ltrim($whole, '+') . $fraction; + + if ($scale < 0) { + $digits .= \str_repeat('0', -$scale); + $scale = 0; + } + + return [BigInt::toNative(BigInt::normalizeInteger($digits)), $scale]; + } + + /** + * Whether a number fraction() leaves out is closer to zero than any result rather than beyond every result. + */ + public static function isTiny(int|float|string $value): bool + { + $order = self::order($value); + + return $order !== null && $order < -self::MAXIMUM_DIGITS; + } + + /** + * The power of ten of a number's leading digit (2 for 123, -3 for 0.001), from its text; null for zero. An + * exponent too long to read as an integer saturates. + */ + public static function order(int|float|string $value): ?int + { + [$mantissa, $exponent] = self::split($value); + [$whole, $fraction] = \explode('.', \ltrim($mantissa, '+-'), 2) + [1 => '']; + if (\ltrim($whole . $fraction, '0') === '') { + return null; + } + + $magnitude = \ltrim(\ltrim($exponent, '+-'), '0'); + if (\strlen($magnitude) > self::MAXIMUM_EXPONENT_DIGITS) { + return \str_starts_with($exponent, '-') ? \PHP_INT_MIN : \PHP_INT_MAX; + } + + $significant = \ltrim($whole, '0'); + $leading = $significant !== '' ? \strlen($significant) - 1 : -(\strspn($fraction, '0') + 1); + + return $leading + (int) $exponent; + } + + /** + * @return array{string, string} + */ + private static function split(int|float|string $value): array + { + $parts = \explode('e', \strtolower(\trim((string) $value)), 2); + + return [$parts[0], $parts[1] ?? '0']; + } + + private function truncated(): string + { + return ($this->negative ? '-' : '') . $this->whole; + } + + private static function unsigned(string $value): string + { + return \str_starts_with($value, '+') || \str_starts_with($value, '-') ? \substr($value, 1) : $value; + } + + private static function isDigits(string $value): bool + { + return $value === '' || \ctype_digit($value); + } + + private static function exponent(string $exponent, int $limit): int + { + $digits = \ltrim($exponent, '+-0'); + $magnitude = \strlen($digits) > \strlen((string) $limit) ? $limit : \min((int) $digits, $limit); + + return \str_starts_with($exponent, '-') ? -$magnitude : $magnitude; + } +} diff --git a/src/Database/Arithmetic/Rounding.php b/src/Database/Arithmetic/Rounding.php new file mode 100644 index 000000000..df1c31db9 --- /dev/null +++ b/src/Database/Arithmetic/Rounding.php @@ -0,0 +1,69 @@ + $fraction > 0.5 || ($fraction === 0.5 && self::isOdd($base)), + self::HalfAwayFromZero => $fraction > 0.5 || ($fraction === 0.5 && BigInt::compare($base, 0) >= 0), + self::TowardZero => BigInt::compare($base, 0) < 0, + }; + + return $up ? BigInt::add($base, 1) : $base; + } + + /** + * The whole number stored for a fractional result computed in floating point. + */ + public function round(float $value): int|string + { + $floor = \floor($value); + + return $this->apply(BigInt::toNative(\sprintf('%.0f', $floor)), $value - $floor); + } + + /** + * The whole number stored for $numerator / $denominator, computed exactly; $denominator is positive. + */ + public function ratio(int|string $numerator, int|string $denominator): int|string + { + $quotient = BigInt::divide($numerator, $denominator); + $remainder = BigInt::modulo($numerator, $denominator); + if (BigInt::compare($remainder, 0) === 0 || $this === self::TowardZero) { + return $quotient; + } + + $half = BigInt::compare(BigInt::multiply(\ltrim((string) $remainder, '-'), 2), $denominator); + $away = $half > 0 || ($half === 0 && ($this === self::HalfAwayFromZero || self::isOdd($quotient))); + if (! $away) { + return $quotient; + } + + return BigInt::compare($numerator, 0) < 0 ? BigInt::subtract($quotient, 1) : BigInt::add($quotient, 1); + } + + private static function isOdd(int|string $value): bool + { + return (int) \substr((string) $value, -1) % 2 === 1; + } +} diff --git a/tests/unit/Arithmetic/DecimalTest.php b/tests/unit/Arithmetic/DecimalTest.php new file mode 100644 index 000000000..86947d28a --- /dev/null +++ b/tests/unit/Arithmetic/DecimalTest.php @@ -0,0 +1,126 @@ + + */ + public static function nonDecimals(): iterable + { + yield 'empty' => ['']; + yield 'whitespace' => [' ']; + yield 'sign without digits' => ['+']; + yield 'point without digits' => ['.']; + yield 'exponent without a mantissa' => ['e5']; + yield 'exponent without digits' => ['1e']; + yield 'exponent without a mantissa digit' => ['.e5']; + yield 'exponent with two signs' => ['1e-+5']; + yield 'two exponent markers' => ['1ee5']; + yield 'fractional exponent' => ['1e5.0']; + yield 'hexadecimal' => ['0x1A']; + yield 'digit separators' => ['1_000']; + yield 'two decimal points' => ['1.2.3']; + yield 'two signs' => ['--5']; + yield 'mixed signs' => ['+-1']; + yield 'infinity' => ['INF']; + yield 'padded with a null byte' => ["10.5\0"]; + yield 'integer padded with a null byte' => ["1\0"]; + } + + #[DataProvider('nonDecimals')] + public function testTextThatIsNotADecimalIsNotParsed(string $value): void + { + $this->assertNull(Decimal::parse($value)); + } + + /** + * @return iterable + */ + public static function decimals(): iterable + { + yield 'fractional' => ['10.5', 10, 11]; + yield 'negative fraction of one' => ['-0.5', -1, 0]; + yield 'whole' => ['10.000', 10, 10]; + yield 'padded with every numeric whitespace' => [" \t\n\r\v\f10.5 \t\n\r\v\f", 10, 11]; + yield 'zero with an exponent' => ['-0.000e50', 0, 0]; + yield 'explicitly positive exponent' => ['1e+5', 100000, 100000]; + yield 'upper case exponent marker' => ['1.5E+2', 150, 150]; + yield 'point before the exponent' => ['1.e5', 100000, 100000]; + yield 'fraction without a whole part and an exponent' => ['-.5e1', -5, -5]; + yield 'negative exponent' => ['1.5e-1', 0, 1]; + yield 'fractional at the unsigned limit' => ['18446744073709551615.9', '18446744073709551615', '18446744073709551616']; + yield 'beyond the unsigned limit' => ['1e400', '184467440737095516150', '184467440737095516150']; + yield 'below the negative unsigned limit' => ['-1e400', '-184467440737095516150', '-184467440737095516150']; + yield 'an integer' => [-7, -7, -7]; + yield 'a float' => [-2.5, -3, -2]; + } + + #[DataProvider('decimals')] + public function testADecimalRoundsDownByFloorAndUpByCeiling(int|float|string $value, int|string $floor, int|string $ceiling): void + { + $decimal = Decimal::parse($value); + + $this->assertInstanceOf(Decimal::class, $decimal); + $this->assertSame($floor, $decimal->floor()); + $this->assertSame($ceiling, $decimal->ceil()); + } + + /** + * @return iterable + */ + public static function fractions(): iterable + { + yield 'fractional text' => ['1.5', [15, 1]]; + yield 'fractional float' => [1.5, [15, 1]]; + yield 'negative fraction' => ['-2.75', [-275, 2]]; + yield 'signed fraction' => ['+2.75', [275, 2]]; + yield 'exponent scaling a fraction up' => ['1.5e2', [150, 0]]; + yield 'exponent scaling down' => ['15e-3', [15, 3]]; + yield 'integer' => [10, [10, 0]]; + yield 'zero' => ['0.000', [0, 0]]; + yield 'beyond float precision' => ['9007199254740993.5', [90071992547409935, 1]]; + yield 'beyond the native integer range' => ['18446744073709551615.5', ['184467440737095516155', 1]]; + yield 'too large to write out' => ['1e1000000000', null]; + yield 'too close to zero to write out' => ['1e-1000000000', null]; + } + + /** + * @param array{int|string, int}|null $expected + */ + #[DataProvider('fractions')] + public function testANumberIsAnIntegerOverAPowerOfTen(int|float|string $value, ?array $expected): void + { + $this->assertSame($expected, Decimal::fraction($value)); + } + + /** + * @return iterable + */ + public static function orders(): iterable + { + yield 'hundreds' => [123, 2, false]; + yield 'thousandths' => ['0.001', -3, false]; + yield 'a negative fraction' => [-0.5, -1, false]; + yield 'zero' => [0, null, false]; + yield 'a zero with an exponent' => ['0e5', null, false]; + yield 'a huge exponent' => ['1e1000000000', 1000000000, false]; + yield 'a vanishing exponent' => ['1e-1000000000', -1000000000, true]; + yield 'just inside the written-out range' => ['1e-64', -64, false]; + yield 'just past the written-out range' => ['1e-65', -65, true]; + yield 'an exponent too long for an integer' => ['1e99999999999999999999', \PHP_INT_MAX, false]; + yield 'a negative exponent too long for an integer' => ['1e-99999999999999999999', \PHP_INT_MIN, true]; + } + + #[DataProvider('orders')] + public function testTheOrderOfANumberIsThePowerOfTenOfItsLeadingDigit(int|float|string $value, ?int $order, bool $tiny): void + { + $this->assertSame($order, Decimal::order($value)); + $this->assertSame($tiny, Decimal::isTiny($value)); + } +} diff --git a/tests/unit/Arithmetic/RoundingTest.php b/tests/unit/Arithmetic/RoundingTest.php new file mode 100644 index 000000000..9d08be752 --- /dev/null +++ b/tests/unit/Arithmetic/RoundingTest.php @@ -0,0 +1,83 @@ + + */ + public static function values(): iterable + { + yield 'a half below an even number' => [2.5, 2, 3, 2]; + yield 'a half below an odd number' => [3.5, 4, 4, 3]; + yield 'a negative half' => [-2.5, -2, -3, -2]; + yield 'a negative half below an odd number' => [-3.5, -4, -4, -3]; + yield 'above a half' => [2.6, 3, 3, 2]; + yield 'below a half' => [2.4, 2, 2, 2]; + yield 'a negative value above a half' => [-2.6, -3, -3, -2]; + yield 'a negative value below a half' => [-2.4, -2, -2, -2]; + yield 'a whole number' => [7.0, 7, 7, 7]; + yield 'a half of zero' => [0.5, 0, 1, 0]; + yield 'a negative half of zero' => [-0.5, 0, -1, 0]; + } + + #[DataProvider('values')] + public function testAFloatIsRoundedAsEachEngineStoredIt(float $value, int $halfEven, int $halfAwayFromZero, int $towardZero): void + { + $this->assertSame($halfEven, Rounding::HalfEven->round($value)); + $this->assertSame($halfAwayFromZero, Rounding::HalfAwayFromZero->round($value)); + $this->assertSame($towardZero, Rounding::TowardZero->round($value)); + } + + /** + * @return iterable + */ + public static function ratios(): iterable + { + yield 'a half below an even number' => [5, 2, 2, 3, 2]; + yield 'a half below an odd number' => [7, 2, 4, 4, 3]; + yield 'a negative half' => [-5, 2, -2, -3, -2]; + yield 'a negative half below an odd number' => [-7, 2, -4, -4, -3]; + yield 'above a half' => [11, 4, 3, 3, 2]; + yield 'below a half' => [9, 4, 2, 2, 2]; + yield 'a negative value above a half' => [-11, 4, -3, -3, -2]; + yield 'exact' => [6, 3, 2, 2, 2]; + yield 'a half beyond float precision' => ['18014398509481987', 2, 9007199254740994, 9007199254740994, 9007199254740993]; + yield 'a half beyond the native integer range' => ['36893488147419103231', 2, '18446744073709551616', '18446744073709551616', '18446744073709551615']; + } + + #[DataProvider('ratios')] + public function testARatioIsRoundedExactly(int|string $numerator, int|string $denominator, int|string $halfEven, int|string $halfAwayFromZero, int|string $towardZero): void + { + $this->assertSame($halfEven, Rounding::HalfEven->ratio($numerator, $denominator)); + $this->assertSame($halfAwayFromZero, Rounding::HalfAwayFromZero->ratio($numerator, $denominator)); + $this->assertSame($towardZero, Rounding::TowardZero->ratio($numerator, $denominator)); + } + + /** + * @return iterable + */ + public static function fractions(): iterable + { + yield 'no fraction keeps the base' => [9007199254740993, 0.0, 9007199254740993, 9007199254740993, 9007199254740993]; + yield 'a half above an odd base beyond float precision' => [9007199254740993, 0.5, 9007199254740994, 9007199254740994, 9007199254740993]; + yield 'a half above an even base beyond float precision' => [9007199254740994, 0.5, 9007199254740994, 9007199254740995, 9007199254740994]; + yield 'above a half beyond float precision' => [9007199254740993, 0.75, 9007199254740994, 9007199254740994, 9007199254740993]; + yield 'a half above a negative base' => [-9007199254740995, 0.5, -9007199254740994, -9007199254740995, -9007199254740994]; + yield 'a half above the unsigned limit' => ['18446744073709551615', 0.5, '18446744073709551616', '18446744073709551616', '18446744073709551615']; + yield 'a half above the native integer maximum' => [\PHP_INT_MAX, 0.5, '9223372036854775808', '9223372036854775808', \PHP_INT_MAX]; + } + + #[DataProvider('fractions')] + public function testABaseAndItsFractionAreRoundedWithoutPassingThroughAFloat(int|string $base, float $fraction, int|string $halfEven, int|string $halfAwayFromZero, int|string $towardZero): void + { + $this->assertSame($halfEven, Rounding::HalfEven->apply($base, $fraction)); + $this->assertSame($halfAwayFromZero, Rounding::HalfAwayFromZero->apply($base, $fraction)); + $this->assertSame($towardZero, Rounding::TowardZero->apply($base, $fraction)); + } +} From 374ae2e4672de1207efbc875648608d0b1baf5a1 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 16:45:43 +1300 Subject: [PATCH 04/55] feat(adapter): declare engine arithmetic and resolve fractional operands Engine knowledge now lives in the adapters instead of an app-side engine enum sniffed from the PDO driver name: Adapter::arithmetic() answers MariaDB/MySQL rules (also for SQLite, Memory and Redis, which had no 7.4.1 engine of their own), Postgres declares NUMERIC rounding, its exact operators and its refusals (7.4.1 bound the amount as text next to an integer column, so a fractional counter change was refused), and Mongo cuts toward zero as every 7.4.1 read of its stored double did. Pool asks the pooled adapter once per pool, like capabilities(), and ReadWritePool answers it where reads go without opening the sticky window. Arithmetic\Fractional resolves a fractional operator against the stored value into the whole-number increment or decrement that stores what the engine stored, computes the whole change of a fractional counter change, rounds fractional bounds to the whole number they allow, and cuts fractional integer-array items toward zero. It returns a new operator instead of mutating the given one, keeps the operator's attribute, and leaves refused, non-numeric or non-finite cases unchanged so the strict validator still refuses them. The fixture holds appwrite's 204 rows measured against 7.4.1 on MariaDB 10.11, PostgreSQL and MongoDB 8.2.5, plus counter rows per engine, so the wiring can replay them end to end. Nothing calls Fractional yet; defaults are unchanged. Co-Authored-By: Claude Opus 5.5 --- src/Database/Adapter.php | 16 + src/Database/Adapter/Mongo.php | 16 + src/Database/Adapter/Pool.php | 22 + src/Database/Adapter/Postgres.php | 21 + src/Database/Adapter/ReadWritePool.php | 1 + src/Database/Arithmetic/Fractional.php | 363 ++++++++++++++++ tests/unit/Adapter/ArithmeticTest.php | 167 ++++++++ .../Adapter/ReadWritePoolMetadataTest.php | 4 + tests/unit/Adapter/ReadWritePoolTest.php | 2 + tests/unit/Arithmetic/FractionalTest.php | 400 ++++++++++++++++++ tests/unit/Support/FractionalEngine.php | 13 + tests/unit/Support/FractionalResults.php | 267 ++++++++++++ 12 files changed, 1292 insertions(+) create mode 100644 src/Database/Arithmetic/Fractional.php create mode 100644 tests/unit/Adapter/ArithmeticTest.php create mode 100644 tests/unit/Arithmetic/FractionalTest.php create mode 100644 tests/unit/Support/FractionalEngine.php create mode 100644 tests/unit/Support/FractionalResults.php diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 769bba3ea..5acb19da6 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -5,6 +5,7 @@ use Exception; use Throwable; use Utopia\Database\Adapter\Limits; +use Utopia\Database\Arithmetic\Rounding; use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Contention as ContentionException; use Utopia\Database\Exception\Duplicate as DuplicateException; @@ -811,6 +812,21 @@ abstract public function getSequences(Document $collection, array $documents): a abstract public function limits(): Limits; + /** + * How this adapter's engine stored a fractional result in an integer column on database 7.4.1. MariaDB and + * MySQL computed a fractional operand in DOUBLE, rounded half to even, and a division by a whole number in + * exact DECIMAL, rounded half away from zero; adapters without a 7.4.1 engine of their own answer the same. + */ + public function arithmetic(): Arithmetic + { + return new Arithmetic( + counter: Rounding::HalfEven, + operators: Rounding::HalfEven, + division: Rounding::HalfAwayFromZero, + power: Rounding::HalfEven, + ); + } + /** * Get Collection Size of the raw data * diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 1f87731d6..9b4a717b8 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -12,6 +12,8 @@ use Swoole\Coroutine; use Throwable; use Utopia\Database\Adapter; +use Utopia\Database\Arithmetic; +use Utopia\Database\Arithmetic\Rounding; use Utopia\Database\Attribute; use Utopia\Database\Capability; use Utopia\Database\Change; @@ -2912,6 +2914,20 @@ private function sequenceKey(mixed $tenant, string $id): string return $tenant."\0".$id; } + /** + * MongoDB stored a fractional result as a double, and every read cast it to an integer, cutting it toward zero. + */ + #[\Override] + public function arithmetic(): Arithmetic + { + return new Arithmetic( + counter: Rounding::TowardZero, + operators: Rounding::TowardZero, + division: Rounding::TowardZero, + power: Rounding::TowardZero, + ); + } + /** * Collections hold any number of attributes and documents of any width, so both caps are 0. */ diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index 4b2c66301..2878360d4 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -5,6 +5,7 @@ use Throwable; use Utopia\Database\Adapter; use Utopia\Database\Adapter\SQL\Wkt; +use Utopia\Database\Arithmetic; use Utopia\Database\Attribute; use Utopia\Database\Builder\Scoping; use Utopia\Database\Capability; @@ -68,6 +69,11 @@ class Pool extends Adapter implements Feature\Timeouts */ private static ?\WeakMap $features = null; + /** + * @var \WeakMap, Arithmetic>|null + */ + private static ?\WeakMap $arithmetic = null; + /** * @var \WeakMap, array>|null */ @@ -900,6 +906,22 @@ public function getSizeOfCollectionOnDisk(string $collection): int return $result; } + #[\Override] + public function arithmetic(): Arithmetic + { + $remembered = self::$arithmetic[$this->pool] ?? null; + if ($remembered !== null) { + return $remembered; + } + + /** @var Arithmetic $result */ + $result = $this->delegate(__FUNCTION__, \func_get_args()); + self::$arithmetic ??= new \WeakMap(); + self::$arithmetic[$this->pool] = $result; + + return $result; + } + #[\Override] public function limits(): Limits { diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 7f8d5757a..54968eec7 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -11,6 +11,8 @@ use Utopia\Database\Adapter\SQL\Expression; use Utopia\Database\Adapter\SQL\Hook\Permission; use Utopia\Database\Adapter\SQL\Wkt; +use Utopia\Database\Arithmetic; +use Utopia\Database\Arithmetic\Rounding; use Utopia\Database\Attribute; use Utopia\Database\Builder\Postgres as PostgresBuilder; use Utopia\Database\Builder\Scoping; @@ -73,6 +75,25 @@ class Postgres extends SQL implements Feature\Spatial, Feature\Timeouts ', ' => ',', ]; + /** + * PostgreSQL computed in NUMERIC, rounding half away from zero, except POWER() in double precision, rounded half + * to even. Next to an integer column it read a bound amount as an integer, so it refused a fractional counter + * change, and a fractional increment, decrement, multiplication or division without a limit. + */ + #[\Override] + public function arithmetic(): Arithmetic + { + return new Arithmetic( + counter: Rounding::HalfEven, + operators: Rounding::HalfAwayFromZero, + division: Rounding::HalfAwayFromZero, + power: Rounding::HalfEven, + refusesCounterFractions: true, + exact: [OperatorType::Multiply, OperatorType::Divide, OperatorType::Modulo], + refusedWithoutLimit: [OperatorType::Increment, OperatorType::Decrement, OperatorType::Multiply, OperatorType::Divide], + ); + } + /** * @return array */ diff --git a/src/Database/Adapter/ReadWritePool.php b/src/Database/Adapter/ReadWritePool.php index 18016ea19..f071a6c6d 100644 --- a/src/Database/Adapter/ReadWritePool.php +++ b/src/Database/Adapter/ReadWritePool.php @@ -31,6 +31,7 @@ class ReadWritePool extends Pool * the sticky window, so a call that writes, or must see the latest write, never belongs here. */ final protected const array METADATA_METHODS = [ + 'arithmetic', 'supports', 'capabilities', 'hasFeature', diff --git a/src/Database/Arithmetic/Fractional.php b/src/Database/Arithmetic/Fractional.php new file mode 100644 index 000000000..d34062920 --- /dev/null +++ b/src/Database/Arithmetic/Fractional.php @@ -0,0 +1,363 @@ + \is_finite($value) && \floor($value) !== $value, + \is_string($value) && \is_numeric($value) => ! BigInt::isIntegerString($value) && \floor((float) $value) !== (float) $value, + default => false, + }; + } + + /** + * Whether a value is an integer, integer text or a finite float without a fractional part. + */ + public static function isWhole(int|float|string $value): bool + { + return match (true) { + \is_int($value) => true, + \is_string($value) => BigInt::isIntegerString($value), + default => \is_finite($value) && \floor($value) === $value, + }; + } + + /** + * Whether operator() needs the stored value: a numeric operator with a fractional amount or limit. + */ + public static function needsStored(Operator $operator): bool + { + if (! $operator->isNumericOperation()) { + return false; + } + + foreach (\array_slice($operator->getValues(), 0, 2) as $value) { + if (self::isFractional($value)) { + return true; + } + } + + return false; + } + + /** + * The whole-number increment or decrement that stores what the engine stored for $operator applied to + * $stored, or an increment by 0 where its limit held the value back. The operator comes back unchanged where + * the engine refused it, an operand is not a number, or the result is not finite, so the caller refuses it. + */ + public function operator(Operator $operator, int|float|string|null $stored): Operator + { + $method = $operator->getMethod(); + $values = $operator->getValues(); + $amount = self::number($values[0] ?? 1); + $limit = isset($values[1]) ? self::number($values[1]) : null; + $current = self::integer($stored ?? 0); + + if (! $method->isNumeric() || $current === null || $amount === null || ($limit === null && isset($values[1]))) { + return $operator; + } + + $whole = ! self::isFractional($amount); + if ($limit === null && ! $whole && \in_array($method, $this->arithmetic->refusedWithoutLimit, true)) { + return $operator; + } + + $rounding = $this->arithmetic->rounding($method, $whole); + if (\in_array($method, $this->arithmetic->exact, true)) { + $exact = self::rational($method, $current, $amount); + if ($exact === null) { + return $operator; + } + if ($limit !== null && self::isBeyondRational($method, $exact, $amount, $limit)) { + return self::change($operator, 0); + } + $stores = $rounding->ratio(...$exact); + } else { + $exact = self::compute($method, (float) $current, (float) $amount); + if ($limit !== null && (self::isUndefinedPower($method, $current, $amount) || self::isBeyond($method, $exact, (float) $amount, (float) $limit))) { + return self::change($operator, 0); + } + if (! \is_finite($exact)) { + return $operator; + } + $stores = \in_array($method, self::SHIFTS, true) + ? self::shift($rounding, $current, (float) $amount, $method === OperatorType::Increment) + : $rounding->round($exact); + } + + return self::change($operator, BigInt::subtract($stores, $current)); + } + + /** + * The whole-number change, never negative, that stores $stored plus or minus $value as the engine stored a + * counter change. $stored never passes through a float; only the fraction of $value decides the rounding. + * + * @throws TypeException when $value is not a number greater than 0, or is fractional and the engine refused + * a fractional counter change + */ + public function counter(int|string $stored, int|float|string $value, bool $increase): int|string + { + if (! \is_numeric($value) || (BigInt::isIntegerValue($value) ? BigInt::compare($value, 0) <= 0 : (float) $value <= 0)) { + throw new TypeException('Value must be numeric and greater than 0'); + } + + $whole = BigInt::integralValue($value); + if ($whole !== null) { + return $whole; + } + + if (! self::isFractional($value) || $this->arithmetic->refusesCounterFractions) { + throw new TypeException('Change value must be an integer.'); + } + + $value = (float) $value; + $fraction = $value - \floor($value); + $wholePart = self::integral(\floor($value)); + $result = $increase + ? $this->arithmetic->counter->apply(BigInt::add($stored, $wholePart), $fraction) + : $this->arithmetic->counter->apply(BigInt::subtract(BigInt::subtract($stored, $wholePart), 1), 1 - $fraction); + + return BigInt::compare($result, $stored) >= 0 ? BigInt::subtract($result, $stored) : BigInt::subtract($stored, $result); + } + + /** + * A maximum on an integer column rounded down to the whole number it allows. Integer text and anything that + * is not a finite number come back unchanged. + */ + public static function maximum(int|float|string|null $bound): int|float|string|null + { + return match (true) { + \is_string($bound) => self::decimal($bound)?->floor() ?? $bound, + \is_float($bound) && \is_finite($bound) => self::integral(\floor($bound)), + default => $bound, + }; + } + + /** + * A minimum on an integer column rounded up to the whole number it allows. Integer text and anything that + * is not a finite number come back unchanged. + */ + public static function minimum(int|float|string|null $bound): int|float|string|null + { + return match (true) { + \is_string($bound) => self::decimal($bound)?->ceil() ?? $bound, + \is_float($bound) && \is_finite($bound) => self::integral(\ceil($bound)), + default => $bound, + }; + } + + /** + * An operator adding items to an integer array, with each fractional item cut toward zero as every engine + * read it back. + */ + public static function whole(Operator $operator): Operator + { + $values = $operator->getValues(); + + return match ($operator->getMethod()) { + OperatorType::ArrayAppend, OperatorType::ArrayPrepend => (clone $operator)->setValues(\array_map( + static fn (mixed $value): mixed => \is_array($value) ? \array_map(self::truncate(...), $value) : self::truncate($value), + $values, + )), + OperatorType::ArrayInsert => \array_key_exists(1, $values) + ? (clone $operator)->setValues([$values[0], self::truncate($values[1])]) + : $operator, + default => $operator, + }; + } + + private static function truncate(mixed $item): mixed + { + return (\is_float($item) || \is_string($item)) && self::isFractional($item) ? (int) (float) $item : $item; + } + + private static function change(Operator $operator, int|string $change): Operator + { + return BigInt::compare($change, 0) >= 0 + ? new Operator(OperatorType::Increment, $operator->getAttribute(), [$change]) + : new Operator(OperatorType::Decrement, $operator->getAttribute(), [BigInt::negate($change)]); + } + + private static function number(mixed $value): int|float|string|null + { + return \is_int($value) || \is_float($value) || (\is_string($value) && \is_numeric($value)) ? $value : null; + } + + /** + * A stored integer, or a stored fraction cut toward zero as every read of it was; null for anything else. + */ + private static function integer(int|float|string $stored): int|string|null + { + if (\is_string($stored) && ! \is_numeric($stored)) { + return null; + } + + return BigInt::integralValue($stored) ?? (self::isFractional($stored) ? self::integral((float) $stored < 0 ? \ceil((float) $stored) : \floor((float) $stored)) : null); + } + + private static function compute(OperatorType $method, float $current, float $amount): float + { + return match ($method) { + OperatorType::Increment => $current + $amount, + OperatorType::Decrement => $current - $amount, + OperatorType::Multiply => $current * $amount, + OperatorType::Divide => $amount === 0.0 ? \NAN : $current / $amount, + OperatorType::Modulo => $amount === 0.0 ? \NAN : \fmod($current, $amount), + OperatorType::Power => $current === 0.0 && $amount < 0 ? \NAN : $current ** $amount, + default => \NAN, + }; + } + + /** + * The exact result as a numerator and a positive denominator, or null for a division or modulo by zero or an + * operand too long to write out. + * + * @return array{int|string, int|string}|null + */ + private static function rational(OperatorType $method, int|string $current, int|float|string $amount): ?array + { + $decimal = Decimal::fraction($amount); + if ($decimal === null) { + return Decimal::isTiny($amount) && $method !== OperatorType::Divide ? [0, 1] : null; + } + [$numerator, $scale] = $decimal; + $power = self::powerOfTen($scale); + + if ($method === OperatorType::Multiply) { + return [BigInt::multiply($current, $numerator), $power]; + } + if (BigInt::compare($numerator, 0) === 0) { + return null; + } + if ($method === OperatorType::Modulo) { + return [BigInt::modulo(BigInt::multiply($current, $power), $numerator), $power]; + } + + $dividend = BigInt::multiply($current, $power); + + return BigInt::compare($numerator, 0) < 0 + ? [BigInt::negate($dividend), BigInt::negate($numerator)] + : [$dividend, $numerator]; + } + + /** + * @param array{int|string, int|string} $exact + */ + private static function isBeyondRational(OperatorType $method, array $exact, int|float|string $amount, int|float|string $limit): bool + { + $factor = Decimal::fraction($amount); + if ($factor === null || BigInt::compare($factor[0], 0) === 0) { + return false; + } + + [$numerator, $denominator] = $exact; + $bound = Decimal::fraction($limit); + $comparison = $bound === null + ? self::compareWithOversized($numerator, $limit) + : BigInt::compare( + BigInt::multiply($numerator, self::powerOfTen($bound[1])), + BigInt::multiply($bound[0], $denominator), + ); + + return match (true) { + \in_array($method, self::LIMITED_BY_MAXIMUM, true) => $comparison > 0, + \in_array($method, self::LIMITED_BY_MINIMUM, true) => $comparison < 0, + default => false, + }; + } + + /** + * How a result compares with a limit Decimal::fraction() leaves out: one beyond every result in magnitude, or + * one closer to zero than any of them but zero. + */ + private static function compareWithOversized(int|string $numerator, int|float|string $limit): int + { + $negative = \str_starts_with(\ltrim((string) $limit), '-'); + if (! Decimal::isTiny($limit)) { + return $negative ? 1 : -1; + } + + $sign = BigInt::compare($numerator, 0); + + return $sign !== 0 ? $sign : ($negative ? 1 : -1); + } + + private static function isBeyond(OperatorType $method, float $exact, float $amount, float $limit): bool + { + return match (true) { + $method === OperatorType::Multiply && $amount === 0.0, $method === OperatorType::Divide && $amount === 0.0 => false, + \in_array($method, self::LIMITED_BY_MAXIMUM, true) => $exact > $limit, + \in_array($method, self::LIMITED_BY_MINIMUM, true) => $exact < $limit, + default => false, + }; + } + + /** + * Where a limit came with it, 7.4.1 left the value as it was for a power without a value: zero to a negative + * power, or a negative base to a fractional one. + */ + private static function isUndefinedPower(OperatorType $method, int|string $current, int|float|string $amount): bool + { + return $method === OperatorType::Power + && ((BigInt::compare($current, 0) === 0 && (float) $amount < 0) || (BigInt::compare($current, 0) < 0 && self::isFractional($amount))); + } + + /** + * The stored value plus or minus a fractional amount, exact at any size of the stored value. + */ + private static function shift(Rounding $rounding, int|string $current, float $amount, bool $increase): int|string + { + $whole = self::integral(\floor($amount)); + $fraction = $amount - \floor($amount); + + if ($increase) { + return $rounding->apply(BigInt::add($current, $whole), $fraction); + } + + return $fraction === 0.0 + ? BigInt::subtract($current, $whole) + : $rounding->apply(BigInt::subtract(BigInt::subtract($current, $whole), 1), 1 - $fraction); + } + + private static function decimal(string $bound): ?Decimal + { + return BigInt::isIntegerString($bound) ? null : Decimal::parse($bound); + } + + private static function powerOfTen(int $exponent): int|string + { + return BigInt::toNative('1' . \str_repeat('0', $exponent)); + } + + private static function integral(float $whole): int|string + { + return BigInt::toNative(\sprintf('%.0f', $whole)); + } +} diff --git a/tests/unit/Adapter/ArithmeticTest.php b/tests/unit/Adapter/ArithmeticTest.php new file mode 100644 index 000000000..b2fb0423c --- /dev/null +++ b/tests/unit/Adapter/ArithmeticTest.php @@ -0,0 +1,167 @@ +}> + */ + public static function adapters(): iterable + { + $mariaDB = [Rounding::HalfEven, Rounding::HalfEven, Rounding::HalfAwayFromZero, Rounding::HalfEven, false, [], []]; + + yield 'MariaDB' => [static fn (): Adapter => new MariaDB(new stdClass()), $mariaDB]; + yield 'MySQL' => [static fn (): Adapter => new MySQL(new stdClass()), $mariaDB]; + yield 'SQLite' => [static fn (): Adapter => new SQLite(new PDO('sqlite::memory:')), $mariaDB]; + yield 'Memory' => [static fn (): Adapter => new Memory(), $mariaDB]; + yield 'Redis' => [static fn (): Adapter => new Redis(new RedisClient()), $mariaDB]; + yield 'PostgreSQL' => [static fn (): Adapter => new Postgres(new stdClass()), [ + Rounding::HalfEven, + Rounding::HalfAwayFromZero, + Rounding::HalfAwayFromZero, + Rounding::HalfEven, + true, + [OperatorType::Multiply, OperatorType::Divide, OperatorType::Modulo], + [OperatorType::Increment, OperatorType::Decrement, OperatorType::Multiply, OperatorType::Divide], + ]]; + yield 'MongoDB' => [self::mongo(...), [Rounding::TowardZero, Rounding::TowardZero, Rounding::TowardZero, Rounding::TowardZero, false, [], []]]; + } + + /** + * @param callable(): Adapter $adapter + * @param list $expected + */ + #[DataProvider('adapters')] + public function testEachAdapterDeclaresHowItsEngineStoredAFractionalResult(callable $adapter, array $expected): void + { + $this->assertSame($expected, self::describe($adapter()->arithmetic())); + } + + /** + * @param callable(): Adapter $adapter + * @param list $expected + */ + #[DataProvider('adapters')] + public function testAPoolAnswersForItsPooledAdapter(callable $adapter, array $expected): void + { + $pool = new Pool($this->connections($adapter())); + $pool->setAuthorization(new Authorization()); + + $this->assertSame($expected, self::describe($pool->arithmetic())); + $this->assertSame($expected, self::describe($pool->arithmetic())); + } + + /** + * @param callable(): Adapter $adapter + * @param list $expected + */ + #[DataProvider('adapters')] + public function testAReadWritePoolAnswersForItsPooledAdapter(callable $adapter, array $expected): void + { + $pool = new ReadWritePool($this->connections($adapter()), $this->connections($adapter())); + $pool->setAuthorization(new Authorization()); + + $this->assertSame($expected, self::describe($pool->arithmetic())); + } + + /** + * @return iterable + */ + public static function methods(): iterable + { + yield 'a power' => [OperatorType::Power, false, Rounding::TowardZero]; + yield 'a power to a whole exponent' => [OperatorType::Power, true, Rounding::TowardZero]; + yield 'a division by a whole number' => [OperatorType::Divide, true, Rounding::HalfAwayFromZero]; + yield 'a division by a fraction' => [OperatorType::Divide, false, Rounding::HalfEven]; + yield 'a multiplication by a whole number' => [OperatorType::Multiply, true, Rounding::HalfEven]; + yield 'an increment' => [OperatorType::Increment, false, Rounding::HalfEven]; + yield 'a modulo' => [OperatorType::Modulo, false, Rounding::HalfEven]; + } + + #[DataProvider('methods')] + public function testAnOperatorIsRoundedByItsMethodAndOperand(OperatorType $method, bool $whole, Rounding $expected): void + { + $arithmetic = new Arithmetic( + counter: Rounding::TowardZero, + operators: Rounding::HalfEven, + division: Rounding::HalfAwayFromZero, + power: Rounding::TowardZero, + ); + + $this->assertSame($expected, $arithmetic->rounding($method, $whole)); + } + + /** + * @return list + */ + private static function describe(Arithmetic $arithmetic): array + { + return [ + $arithmetic->counter, + $arithmetic->operators, + $arithmetic->division, + $arithmetic->power, + $arithmetic->refusesCounterFractions, + $arithmetic->exact, + $arithmetic->refusedWithoutLimit, + ]; + } + + private static function mongo(): Mongo + { + return new Mongo(new class () extends Client { + public function __construct() + { + } + + #[\Override] + public function connect(): self + { + return $this; + } + + #[\Override] + public function close(): void + { + } + }); + } + + /** + * @return UtopiaPool + */ + private function connections(Adapter $adapter): UtopiaPool + { + /** @var UtopiaPool&Stub $connections */ + $connections = self::createStub(UtopiaPool::class); + $connections->method('use')->willReturnCallback( + static fn (callable $callback): mixed => $callback($adapter), + ); + + return $connections; + } +} diff --git a/tests/unit/Adapter/ReadWritePoolMetadataTest.php b/tests/unit/Adapter/ReadWritePoolMetadataTest.php index fa81f7200..a80c35d6b 100644 --- a/tests/unit/Adapter/ReadWritePoolMetadataTest.php +++ b/tests/unit/Adapter/ReadWritePoolMetadataTest.php @@ -41,6 +41,7 @@ final class ReadWritePoolMetadataTest extends TestCase public static function metadataCalls(): iterable { $calls = [ + 'arithmetic' => [], 'supports' => [Capability::IndexKey], 'capabilities' => [], 'getAttributeWidth' => [new Document()], @@ -95,6 +96,9 @@ public function testMetadataCallIsAnsweredWhereReadsGoWithoutOpeningTheStickyWin if ($method === 'builder') { $answer->willReturn(new SQLiteBuilder()); } + if ($method === 'arithmetic') { + $answer->willReturn((new Memory())->arithmetic()); + } $primary->expects($this->never())->method($method); $replica->expects($this->once())->method('ping')->willReturn(true); $primary->expects($this->never())->method('ping'); diff --git a/tests/unit/Adapter/ReadWritePoolTest.php b/tests/unit/Adapter/ReadWritePoolTest.php index 34d72ec08..2251d3145 100644 --- a/tests/unit/Adapter/ReadWritePoolTest.php +++ b/tests/unit/Adapter/ReadWritePoolTest.php @@ -110,6 +110,7 @@ public function testReadMethodsRouteToReadPool(): void 'getCountOfAttributes', 'getCountOfIndexes', 'limits', + 'arithmetic', 'supports', 'capabilities', 'hasFeature', @@ -606,6 +607,7 @@ private function getDefaultReturnForMethod(string $method): mixed 'exists', 'collectionExists', 'ping', 'supports', 'hasFeature' => true, 'id' => 'string', 'limits' => (new Memory())->limits(), + 'arithmetic' => (new Memory())->arithmetic(), 'getSchemaAttributes', 'getSchemaIndexes', 'capabilities', 'decode' => [], 'encode' => 'POINT(1 2)', 'builder' => new \Utopia\Database\Builder\SQLite(), diff --git a/tests/unit/Arithmetic/FractionalTest.php b/tests/unit/Arithmetic/FractionalTest.php new file mode 100644 index 000000000..27ee93b52 --- /dev/null +++ b/tests/unit/Arithmetic/FractionalTest.php @@ -0,0 +1,400 @@ +, int|null}> + */ + public static function operators(): iterable + { + return FractionalResults::operators(); + } + + /** + * @param list $values + */ + #[DataProvider('operators')] + public function testAFractionalOperandStoresWhatTheEngineStored(FractionalEngine $engine, int $stored, OperatorType $method, array $values, ?int $expected): void + { + $operator = new Operator($method, 'count', $values); + $this->assertTrue(Fractional::needsStored($operator)); + + $resolved = self::fractional($engine)->operator($operator, $stored); + + if ($expected === null) { + $this->assertSame($operator, $resolved, 'The engine refused it, so the caller must see it unchanged'); + + return; + } + + $this->assertContains($resolved->getMethod(), [OperatorType::Increment, OperatorType::Decrement]); + $this->assertSame('count', $resolved->getAttribute()); + $this->assertCount(1, $resolved->getValues(), 'A resolved change carries no limit'); + $this->assertSame($expected, self::stores($stored, $resolved)); + } + + public function testAWholeOperandAndLimitNeedNoStoredValue(): void + { + $this->assertFalse(Fractional::needsStored(Operator::increment(2, 10))); + $this->assertFalse(Fractional::needsStored(Operator::multiply(2.0))); + $this->assertFalse(Fractional::needsStored(Operator::divide('4'))); + $this->assertFalse(Fractional::needsStored(Operator::arrayAppend([1.5]))); + $this->assertFalse(Fractional::needsStored(Operator::increment(\INF))); + $this->assertTrue(Fractional::needsStored(Operator::increment(2, 10.5))); + $this->assertTrue(Fractional::needsStored(Operator::divide('2.5'))); + } + + public function testAMissingStoredValueCountsAsZero(): void + { + $resolved = self::fractional(FractionalEngine::MariaDB)->operator(Operator::increment(1.5), null); + + $this->assertSame(OperatorType::Increment, $resolved->getMethod()); + $this->assertSame(2, $resolved->getValue()); + } + + public function testAFractionalChangeBeyondFloatPrecisionStaysExact(): void + { + $resolved = self::fractional(FractionalEngine::MariaDB)->operator(Operator::increment(1.5), 9007199254740993); + + $this->assertSame(1, $resolved->getValue(), '9007199254740993 + 1.5 rounds half to even to 9007199254740994'); + } + + /** + * @return iterable + */ + public static function numericOnPostgreSQLBeyondFloatPrecision(): iterable + { + yield 'multiply' => [Operator::multiply(1.5, 99999999999999999), 9007199254740993, 13510798882111490]; + yield 'divide' => [Operator::divide(0.5, 0), 9007199254740993, 18014398509481986]; + yield 'modulo' => [Operator::modulo(2.5), 9007199254740993, 1]; + } + + #[DataProvider('numericOnPostgreSQLBeyondFloatPrecision')] + public function testPostgreSQLComputesFractionalFactorsExactlyAsNumericDid(Operator $operator, int $stored, int $expected): void + { + $resolved = self::fractional(FractionalEngine::PostgreSQL)->operator($operator, $stored); + + $this->assertSame($expected, self::stores($stored, $resolved)); + } + + public function testAnOperandTooLongForAnIntegerIsLeftForTheCallerToRefuse(): void + { + $fractional = self::fractional(FractionalEngine::PostgreSQL); + + foreach ([Operator::multiply('1e1000000000', 1.5), new Operator(OperatorType::Divide, 'count', ['1e-1000000000', 0.5]), Operator::multiply('1e99999999999999999999', 1.5)] as $operator) { + $this->assertSame($operator, $fractional->operator($operator, 10)); + } + } + + public function testAResultThatIsNotFiniteIsLeftForTheCallerToRefuse(): void + { + $operator = Operator::power(100.5); + + $this->assertSame($operator, self::fractional(FractionalEngine::MariaDB)->operator($operator, \PHP_INT_MAX)); + } + + /** + * @return iterable + */ + public static function oversizedLimits(): iterable + { + yield 'a huge maximum holds nothing back' => [Operator::multiply(1.5, '1e1000000000'), 15]; + yield 'a huge negative maximum holds everything back' => [Operator::multiply(1.5, '-1e1000000000'), 10]; + yield 'a tiny positive maximum holds back a positive result' => [Operator::multiply(1.5, '1e-1000000000'), 10]; + yield 'a huge negative minimum holds nothing back' => [Operator::divide(0.5, '-1e1000000000'), 20]; + yield 'a tiny negative minimum lets a positive result through' => [Operator::divide(0.5, '-1e-1000000000'), 20]; + yield 'a maximum with many fractional digits just above one holds back 15' => [Operator::multiply(1.5, '1.' . \str_repeat('0', 80) . '1'), 10]; + yield 'a maximum with many fractional digits just below sixteen lets 15 through' => [Operator::multiply(1.5, '15.' . \str_repeat('9', 80)), 15]; + yield 'a tiny factor leaves zero' => [Operator::multiply('1e-1000000000', 100), 0]; + yield 'an exponent written with leading zeros is read by its value' => [Operator::multiply('1.5e-00000', 100), 15]; + yield 'a long fraction scaled up by its exponent is read by its value' => [Operator::multiply('0.' . \str_repeat('0', 99) . '15e100', 100), 15]; + } + + #[DataProvider('oversizedLimits')] + public function testALimitTooLongToWriteOutStillBoundsTheResult(Operator $operator, int $expected): void + { + $resolved = self::fractional(FractionalEngine::PostgreSQL)->operator($operator, 10); + + $this->assertSame($expected, self::stores(10, $resolved)); + } + + public function testFractionalArrayItemsAreCutTowardZero(): void + { + $this->assertSame([1, -1, -2, 2, 0, 7], Fractional::whole(Operator::arrayAppend([1.5, -1.5, -2.5, 2.5, 0.9999, 7]))->getValues()); + $this->assertSame([0, 3], Fractional::whole(Operator::arrayPrepend([0.5, 3.7]))->getValues()); + $this->assertSame([[1, 2], 3], Fractional::whole(Operator::arrayAppend([[1.5, 2], '3.9']))->getValues()); + $this->assertSame([1, 3], Fractional::whole(Operator::arrayInsert(1, 3.5))->getValues()); + $this->assertSame(['a', 2], Fractional::whole(Operator::arrayAppend(['a', 2]))->getValues()); + $this->assertSame([1.5], Fractional::whole(Operator::increment(1.5))->getValues()); + } + + public function testCuttingArrayItemsLeavesTheGivenOperatorAsItWas(): void + { + $operator = Operator::arrayAppend([1.5]); + + Fractional::whole($operator); + + $this->assertSame([1.5], $operator->getValues()); + } + + /** + * @return iterable + */ + public static function counters(): iterable + { + return FractionalResults::counters(); + } + + #[DataProvider('counters')] + public function testAFractionalCounterChangeStoresWhatTheEngineStored(FractionalEngine $engine, bool $increase, int $start, float $value, int|float|null $bound, ?int $stored): void + { + $fractional = self::fractional($engine); + + if ($stored === null) { + $this->expectException(TypeException::class); + $this->expectExceptionMessage('Change value must be an integer.'); + } + + $change = $fractional->counter($start, $value, $increase); + + $this->assertSame($stored, $increase ? BigInt::add($start, $change) : BigInt::subtract($start, $change)); + } + + /** + * @return iterable + */ + public static function wholeCounterChanges(): iterable + { + yield 'integer' => [2, 2]; + yield 'whole float' => [2.0, 2]; + yield 'integer text' => ['2', 2]; + yield 'integer text beyond the native range' => ['18446744073709551615', '18446744073709551615']; + } + + #[DataProvider('wholeCounterChanges')] + public function testAWholeCounterChangeIsTheChangeOnEveryEngine(int|float|string $value, int|string $expected): void + { + foreach (FractionalEngine::cases() as $engine) { + $this->assertSame($expected, self::fractional($engine)->counter(5, $value, true), $engine->name); + } + } + + public function testFractionalTextChangesAsItsNumber(): void + { + $this->assertSame(1, self::fractional(FractionalEngine::MariaDB)->counter(5, '1.5', true)); + } + + /** + * @return iterable + */ + public static function unusableCounterChanges(): iterable + { + yield 'infinite' => [\INF, 'Change value must be an integer.']; + yield 'not a number' => [\NAN, 'Change value must be an integer.']; + yield 'non-numeric text' => ['ten', 'Value must be numeric and greater than 0']; + yield 'zero' => [0, 'Value must be numeric and greater than 0']; + yield 'negative' => [-1, 'Value must be numeric and greater than 0']; + yield 'negative fraction' => [-1.5, 'Value must be numeric and greater than 0']; + yield 'negative fractional text' => ['-0.5', 'Value must be numeric and greater than 0']; + yield 'negative integer text' => ['-18446744073709551615', 'Value must be numeric and greater than 0']; + } + + #[DataProvider('unusableCounterChanges')] + public function testACounterChangeThatIsNotAPositiveNumberIsRefused(int|float|string $value, string $message): void + { + $this->expectException(TypeException::class); + $this->expectExceptionMessage($message); + + self::fractional(FractionalEngine::MariaDB)->counter(5, $value, true); + } + + public function testAFractionalStoredValueIsReadCutTowardZero(): void + { + $resolved = self::fractional(FractionalEngine::MongoDB)->operator(Operator::increment(1.5), 6.5); + + $this->assertSame(7, self::stores(6, $resolved)); + } + + public function testAnOperatorThatIsNotNumericOrHasANonNumericOperandIsUnchanged(): void + { + $fractional = self::fractional(FractionalEngine::MariaDB); + + foreach ([Operator::arrayAppend([1.5]), Operator::increment('ten', 10.5), Operator::multiply(1.5, 'ten')] as $operator) { + $this->assertSame($operator, $fractional->operator($operator, 10)); + } + + $operator = Operator::increment(1.5); + $this->assertSame($operator, $fractional->operator($operator, 'ten'), 'A stored value that is not a number is left for the caller'); + } + + /** + * @return iterable + */ + public static function changes(): iterable + { + yield 'integer' => [2, true]; + yield 'whole float' => [2.0, true]; + yield 'integer text' => ['2', true]; + yield 'fractional' => [1.5, false]; + yield 'fractional text' => ['1.5', false]; + yield 'infinite' => [\INF, false]; + } + + #[DataProvider('changes')] + public function testOnlyAWholeChangeIsWhole(int|float|string $value, bool $whole): void + { + $this->assertSame($whole, Fractional::isWhole($value)); + } + + /** + * @return iterable + */ + public static function values(): iterable + { + yield 'fractional float' => [1.5, true]; + yield 'fractional text' => ['-0.25', true]; + yield 'fractional exponent text' => ['1.25e1', true]; + yield 'integer' => [2, false]; + yield 'whole float' => [2.0, false]; + yield 'whole decimal text' => ['2.000', false]; + yield 'integer text' => ['2', false]; + yield 'infinite' => [\INF, false]; + yield 'not a number' => [\NAN, false]; + yield 'non-numeric text' => ['1.5x', false]; + yield 'null' => [null, false]; + yield 'array' => [[1.5], false]; + } + + #[DataProvider('values')] + public function testOnlyAFiniteNumberWithAFractionIsFractional(mixed $value, bool $fractional): void + { + $this->assertSame($fractional, Fractional::isFractional($value)); + } + + /** + * @return iterable + */ + public static function integerBounds(): iterable + { + yield 'fractional' => [10.5, 10, 11]; + yield 'negative fractional' => [-2.5, -3, -2]; + yield 'just under a whole number' => [9.999, 9, 10]; + yield 'whole float' => [10.0, 10, 10]; + yield 'integer' => [10, 10, 10]; + yield 'fractional text' => ['10.5', 10, 11]; + yield 'integer text' => ['10', '10', '10']; + yield 'beyond the native integer range' => [1.0e19, '10000000000000000000', '10000000000000000000']; + yield 'fractional text beyond float precision' => ['9007199254740993.5', 9007199254740993, 9007199254740994]; + yield 'negative fractional text beyond float precision' => ['-9007199254740993.5', -9007199254740994, -9007199254740993]; + yield 'negative fractional text' => ['-5.5', -6, -5]; + yield 'negative fraction of one text' => ['-0.5', -1, 0]; + yield 'fraction of one text' => ['0.5', 0, 1]; + yield 'whole decimal text' => ['10.000', 10, 10]; + yield 'leading zeros text' => ['0000000000000000000005.5', 5, 6]; + yield 'fractional text above the native integer range' => ['9223372036854775807.5', 9223372036854775807, '9223372036854775808']; + yield 'fractional text below the native integer range' => ['-9223372036854775808.5', '-9223372036854775809', \PHP_INT_MIN]; + yield 'fractional text at the unsigned limit' => ['18446744073709551615.9', '18446744073709551615', '18446744073709551616']; + yield 'exponent text' => ['1e3', 1000, 1000]; + yield 'capital exponent text' => ['1E3', 1000, 1000]; + yield 'fractional exponent text' => ['1.25e1', 12, 13]; + yield 'negative fractional exponent text' => ['-1.25e1', -13, -12]; + yield 'negative exponent text' => ['1e-3', 0, 1]; + yield 'negative value with a negative exponent text' => ['-1e-3', -1, 0]; + yield 'vanishing exponent text' => ['1e-999999999', 0, 1]; + yield 'negative value with a vanishing exponent text' => ['-1e-999999999', -1, 0]; + yield 'exponent beyond the native integer range text' => ['5e-9223372036854775808', 0, 1]; + yield 'zero exponent text beyond float precision' => ['9007199254740993.5e0', 9007199254740993, 9007199254740994]; + yield 'padded fractional text' => [' 10.5 ', 10, 11]; + yield 'fractional text padded with form feed and vertical tab' => ["\f10.5\v", 10, 11]; + yield 'signed fractional text' => ['+10.5', 10, 11]; + yield 'fraction without a whole part text' => ['.5', 0, 1]; + yield 'whole part without a fraction text' => ['5.', 5, 5]; + yield 'negative zero text' => ['-0.0', 0, 0]; + yield 'zero with a huge exponent text' => ['0e999999999', 0, 0]; + yield 'zero decimal with an exponent text' => ['0.000e50', 0, 0]; + yield 'negative zero with an exponent text' => ['-0e5', 0, 0]; + yield 'no bound' => [null, null, null]; + } + + #[DataProvider('integerBounds')] + public function testAFractionalMaximumRoundsDownAndAMinimumUp(int|float|string|null $bound, int|float|string|null $maximum, int|float|string|null $minimum): void + { + $this->assertSame($maximum, Fractional::maximum($bound)); + $this->assertSame($minimum, Fractional::minimum($bound)); + } + + /** + * @return iterable + */ + public static function nonNumericBounds(): iterable + { + yield 'empty' => ['']; + yield 'exponent without digits' => ['1e']; + yield 'hexadecimal' => ['0x1A']; + yield 'digit separators' => ['1_000']; + yield 'two decimal points' => ['1.2.3']; + yield 'two signs' => ['--5']; + yield 'a word' => ['ten']; + yield 'infinite' => [\INF]; + } + + #[DataProvider('nonNumericBounds')] + public function testABoundThatIsNotAFiniteNumberIsPassedOnUnchanged(int|float|string $bound): void + { + $this->assertSame($bound, Fractional::maximum($bound)); + $this->assertSame($bound, Fractional::minimum($bound)); + } + + private static function fractional(FractionalEngine $engine): Fractional + { + $adapter = match ($engine) { + FractionalEngine::MariaDB => new MariaDB(new stdClass()), + FractionalEngine::PostgreSQL => new Postgres(new stdClass()), + FractionalEngine::MongoDB => new Mongo(new class () extends Client { + public function __construct() + { + } + + #[\Override] + public function connect(): self + { + return $this; + } + + #[\Override] + public function close(): void + { + } + }), + }; + + return new Fractional($adapter->arithmetic()); + } + + private static function stores(int $stored, Operator $resolved): int|string + { + $change = $resolved->getValue(); + self::assertTrue(BigInt::isIntegerValue($change), 'A resolved change is a whole number'); + + return $resolved->getMethod() === OperatorType::Increment + ? BigInt::add($stored, $change) + : BigInt::subtract($stored, $change); + } +} diff --git a/tests/unit/Support/FractionalEngine.php b/tests/unit/Support/FractionalEngine.php new file mode 100644 index 000000000..291b66b98 --- /dev/null +++ b/tests/unit/Support/FractionalEngine.php @@ -0,0 +1,13 @@ +, int|null}> + */ + public static function operators(): iterable + { + yield 'MariaDB: inc 11+1.5' => [FractionalEngine::MariaDB, 11, OperatorType::Increment, [1.5], 12]; + yield 'MariaDB: inc 10+1.5' => [FractionalEngine::MariaDB, 10, OperatorType::Increment, [1.5], 12]; + yield 'MariaDB: inc 10+1.25' => [FractionalEngine::MariaDB, 10, OperatorType::Increment, [1.25], 11]; + yield 'MariaDB: inc 10+1.75' => [FractionalEngine::MariaDB, 10, OperatorType::Increment, [1.75], 12]; + yield 'MariaDB: inc -11+0.5' => [FractionalEngine::MariaDB, -11, OperatorType::Increment, [0.5], -10]; + yield 'MariaDB: inc -12+0.5' => [FractionalEngine::MariaDB, -12, OperatorType::Increment, [0.5], -12]; + yield 'MariaDB: dec 11-0.5' => [FractionalEngine::MariaDB, 11, OperatorType::Decrement, [0.5], 10]; + yield 'MariaDB: dec 12-0.5' => [FractionalEngine::MariaDB, 12, OperatorType::Decrement, [0.5], 12]; + yield 'MariaDB: dec -10-0.5' => [FractionalEngine::MariaDB, -10, OperatorType::Decrement, [0.5], -10]; + yield 'MariaDB: inc 10+2 max13.9' => [FractionalEngine::MariaDB, 10, OperatorType::Increment, [2, 13.9], 12]; + yield 'MariaDB: inc 12+2 max13.9' => [FractionalEngine::MariaDB, 12, OperatorType::Increment, [2, 13.9], 12]; + yield 'MariaDB: inc 12+1.5 max13.9' => [FractionalEngine::MariaDB, 12, OperatorType::Increment, [1.5, 13.9], 14]; + yield 'MariaDB: inc 12+1.5 max13.5' => [FractionalEngine::MariaDB, 12, OperatorType::Increment, [1.5, 13.5], 14]; + yield 'MariaDB: dec 12-1 min11.1' => [FractionalEngine::MariaDB, 12, OperatorType::Decrement, [1, 11.1], 12]; + yield 'MariaDB: dec 13-1 min11.1' => [FractionalEngine::MariaDB, 13, OperatorType::Decrement, [1, 11.1], 12]; + yield 'MariaDB: dec 12-0.5 min11.5' => [FractionalEngine::MariaDB, 12, OperatorType::Decrement, [0.5, 11.5], 12]; + yield 'MariaDB: mul 11*1.5' => [FractionalEngine::MariaDB, 11, OperatorType::Multiply, [1.5], 16]; + yield 'MariaDB: mul 5*2.5' => [FractionalEngine::MariaDB, 5, OperatorType::Multiply, [2.5], 12]; + yield 'MariaDB: mul 7*1.5' => [FractionalEngine::MariaDB, 7, OperatorType::Multiply, [1.5], 10]; + yield 'MariaDB: mul 7*-1.5' => [FractionalEngine::MariaDB, 7, OperatorType::Multiply, [-1.5], -10]; + yield 'MariaDB: mul 3*1.25' => [FractionalEngine::MariaDB, 3, OperatorType::Multiply, [1.25], 4]; + yield 'MariaDB: mul 12*2 max25.5' => [FractionalEngine::MariaDB, 12, OperatorType::Multiply, [2, 25.5], 24]; + yield 'MariaDB: mul 13*2 max25.5' => [FractionalEngine::MariaDB, 13, OperatorType::Multiply, [2, 25.5], 13]; + yield 'MariaDB: mul 10*1.5 max15' => [FractionalEngine::MariaDB, 10, OperatorType::Multiply, [1.5, 15], 15]; + yield 'MariaDB: div 10/1.5' => [FractionalEngine::MariaDB, 10, OperatorType::Divide, [1.5], 7]; + yield 'MariaDB: div 9/2.5' => [FractionalEngine::MariaDB, 9, OperatorType::Divide, [2.5], 4]; + yield 'MariaDB: div 10/2 min5.5' => [FractionalEngine::MariaDB, 10, OperatorType::Divide, [2, 5.5], 10]; + yield 'MariaDB: div 10/0.5' => [FractionalEngine::MariaDB, 10, OperatorType::Divide, [0.5], 20]; + yield 'MariaDB: mod 10%2.75' => [FractionalEngine::MariaDB, 10, OperatorType::Modulo, [2.75], 2]; + yield 'MariaDB: mod 11%4.5' => [FractionalEngine::MariaDB, 11, OperatorType::Modulo, [4.5], 2]; + yield 'MariaDB: pow 16^0.5' => [FractionalEngine::MariaDB, 16, OperatorType::Power, [0.5], 4]; + yield 'MariaDB: pow 10^0.5' => [FractionalEngine::MariaDB, 10, OperatorType::Power, [0.5], 3]; + yield 'MariaDB: pow 2^2.5' => [FractionalEngine::MariaDB, 2, OperatorType::Power, [2.5], 6]; + yield 'MariaDB: pow 4^1.5' => [FractionalEngine::MariaDB, 4, OperatorType::Power, [1.5], 8]; + yield 'MariaDB: pow 3^2 max8.5' => [FractionalEngine::MariaDB, 3, OperatorType::Power, [2, 8.5], 3]; + yield 'MariaDB: pow 2^3 max8.5' => [FractionalEngine::MariaDB, 2, OperatorType::Power, [3, 8.5], 8]; + yield 'MariaDB: inc 11+1.5 max20' => [FractionalEngine::MariaDB, 11, OperatorType::Increment, [1.5, 20], 12]; + yield 'MariaDB: inc 10+1.5 max20' => [FractionalEngine::MariaDB, 10, OperatorType::Increment, [1.5, 20], 12]; + yield 'MariaDB: inc -12+0.5 max20' => [FractionalEngine::MariaDB, -12, OperatorType::Increment, [0.5, 20], -12]; + yield 'MariaDB: inc 10+1.25 max20' => [FractionalEngine::MariaDB, 10, OperatorType::Increment, [1.25, 20], 11]; + yield 'MariaDB: dec 11-0.5 min0' => [FractionalEngine::MariaDB, 11, OperatorType::Decrement, [0.5, 0], 10]; + yield 'MariaDB: dec 12-0.5 min0' => [FractionalEngine::MariaDB, 12, OperatorType::Decrement, [0.5, 0], 12]; + yield 'MariaDB: dec -10-0.5 min-100' => [FractionalEngine::MariaDB, -10, OperatorType::Decrement, [0.5, -100], -10]; + yield 'MariaDB: mul 5*2.5 max100' => [FractionalEngine::MariaDB, 5, OperatorType::Multiply, [2.5, 100], 12]; + yield 'MariaDB: mul 11*1.5 max100' => [FractionalEngine::MariaDB, 11, OperatorType::Multiply, [1.5, 100], 16]; + yield 'MariaDB: mul 7*-1.5 max100' => [FractionalEngine::MariaDB, 7, OperatorType::Multiply, [-1.5, 100], -10]; + yield 'MariaDB: mul 3*1.25 max100' => [FractionalEngine::MariaDB, 3, OperatorType::Multiply, [1.25, 100], 4]; + yield 'MariaDB: div 9/2.5 min0' => [FractionalEngine::MariaDB, 9, OperatorType::Divide, [2.5, 0], 4]; + yield 'MariaDB: div 10/1.5 min0' => [FractionalEngine::MariaDB, 10, OperatorType::Divide, [1.5, 0], 7]; + yield 'MariaDB: div 5/0.4 min0' => [FractionalEngine::MariaDB, 5, OperatorType::Divide, [0.4, 0], 12]; + yield 'MariaDB: pow 2^2.5 max100' => [FractionalEngine::MariaDB, 2, OperatorType::Power, [2.5, 100], 6]; + yield 'MariaDB: pow 10^0.5 max100' => [FractionalEngine::MariaDB, 10, OperatorType::Power, [0.5, 100], 3]; + yield 'MariaDB: mod -10%2.75' => [FractionalEngine::MariaDB, -10, OperatorType::Modulo, [2.75], -2]; + yield 'MariaDB: mod 10%2.5' => [FractionalEngine::MariaDB, 10, OperatorType::Modulo, [2.5], 0]; + yield 'MariaDB: mod 11%2.5' => [FractionalEngine::MariaDB, 11, OperatorType::Modulo, [2.5], 1]; + yield 'MariaDB: mod 7%-2.5' => [FractionalEngine::MariaDB, 7, OperatorType::Modulo, [-2.5], 2]; + yield 'MariaDB: pow 4^-0.5' => [FractionalEngine::MariaDB, 4, OperatorType::Power, [-0.5], 0]; + yield 'MariaDB: pow 4^-0.5 max100' => [FractionalEngine::MariaDB, 4, OperatorType::Power, [-0.5, 100], 0]; + yield 'MariaDB: pow 16^-0.25' => [FractionalEngine::MariaDB, 16, OperatorType::Power, [-0.25], 0]; + yield 'MariaDB: div 5/2 min0.5' => [FractionalEngine::MariaDB, 5, OperatorType::Divide, [2, 0.5], 3]; + yield 'MariaDB: div 7/2 min0.5' => [FractionalEngine::MariaDB, 7, OperatorType::Divide, [2, 0.5], 4]; + yield 'MariaDB: div -5/2 min-10.5' => [FractionalEngine::MariaDB, -5, OperatorType::Divide, [2, -10.5], -3]; + yield 'MariaDB: div 5/2 min2.6' => [FractionalEngine::MariaDB, 5, OperatorType::Divide, [2, 2.6], 5]; + yield 'MariaDB: mod 11%3.5' => [FractionalEngine::MariaDB, 11, OperatorType::Modulo, [3.5], 0]; + yield 'MariaDB: mod 13%3.5' => [FractionalEngine::MariaDB, 13, OperatorType::Modulo, [3.5], 2]; + yield 'MariaDB: mod -11%3.5' => [FractionalEngine::MariaDB, -11, OperatorType::Modulo, [3.5], 0]; + yield 'MariaDB: inc 10+2 max10.5' => [FractionalEngine::MariaDB, 10, OperatorType::Increment, [2, 10.5], 10]; + yield 'MariaDB: mul 5*3 max15.5' => [FractionalEngine::MariaDB, 5, OperatorType::Multiply, [3, 15.5], 15]; + yield 'PostgreSQL: inc 11+1.5' => [FractionalEngine::PostgreSQL, 11, OperatorType::Increment, [1.5], null]; + yield 'PostgreSQL: inc 10+1.5' => [FractionalEngine::PostgreSQL, 10, OperatorType::Increment, [1.5], null]; + yield 'PostgreSQL: inc 10+1.25' => [FractionalEngine::PostgreSQL, 10, OperatorType::Increment, [1.25], null]; + yield 'PostgreSQL: inc 10+1.75' => [FractionalEngine::PostgreSQL, 10, OperatorType::Increment, [1.75], null]; + yield 'PostgreSQL: inc -11+0.5' => [FractionalEngine::PostgreSQL, -11, OperatorType::Increment, [0.5], null]; + yield 'PostgreSQL: inc -12+0.5' => [FractionalEngine::PostgreSQL, -12, OperatorType::Increment, [0.5], null]; + yield 'PostgreSQL: dec 11-0.5' => [FractionalEngine::PostgreSQL, 11, OperatorType::Decrement, [0.5], null]; + yield 'PostgreSQL: dec 12-0.5' => [FractionalEngine::PostgreSQL, 12, OperatorType::Decrement, [0.5], null]; + yield 'PostgreSQL: dec -10-0.5' => [FractionalEngine::PostgreSQL, -10, OperatorType::Decrement, [0.5], null]; + yield 'PostgreSQL: inc 10+2 max13.9' => [FractionalEngine::PostgreSQL, 10, OperatorType::Increment, [2, 13.9], 12]; + yield 'PostgreSQL: inc 12+2 max13.9' => [FractionalEngine::PostgreSQL, 12, OperatorType::Increment, [2, 13.9], 12]; + yield 'PostgreSQL: inc 12+1.5 max13.9' => [FractionalEngine::PostgreSQL, 12, OperatorType::Increment, [1.5, 13.9], 14]; + yield 'PostgreSQL: inc 12+1.5 max13.5' => [FractionalEngine::PostgreSQL, 12, OperatorType::Increment, [1.5, 13.5], 14]; + yield 'PostgreSQL: dec 12-1 min11.1' => [FractionalEngine::PostgreSQL, 12, OperatorType::Decrement, [1, 11.1], 12]; + yield 'PostgreSQL: dec 13-1 min11.1' => [FractionalEngine::PostgreSQL, 13, OperatorType::Decrement, [1, 11.1], 12]; + yield 'PostgreSQL: dec 12-0.5 min11.5' => [FractionalEngine::PostgreSQL, 12, OperatorType::Decrement, [0.5, 11.5], 12]; + yield 'PostgreSQL: mul 11*1.5' => [FractionalEngine::PostgreSQL, 11, OperatorType::Multiply, [1.5], null]; + yield 'PostgreSQL: mul 5*2.5' => [FractionalEngine::PostgreSQL, 5, OperatorType::Multiply, [2.5], null]; + yield 'PostgreSQL: mul 7*1.5' => [FractionalEngine::PostgreSQL, 7, OperatorType::Multiply, [1.5], null]; + yield 'PostgreSQL: mul 7*-1.5' => [FractionalEngine::PostgreSQL, 7, OperatorType::Multiply, [-1.5], null]; + yield 'PostgreSQL: mul 3*1.25' => [FractionalEngine::PostgreSQL, 3, OperatorType::Multiply, [1.25], null]; + yield 'PostgreSQL: mul 12*2 max25.5' => [FractionalEngine::PostgreSQL, 12, OperatorType::Multiply, [2, 25.5], 24]; + yield 'PostgreSQL: mul 13*2 max25.5' => [FractionalEngine::PostgreSQL, 13, OperatorType::Multiply, [2, 25.5], 13]; + yield 'PostgreSQL: mul 10*1.5 max15' => [FractionalEngine::PostgreSQL, 10, OperatorType::Multiply, [1.5, 15], 15]; + yield 'PostgreSQL: div 10/1.5' => [FractionalEngine::PostgreSQL, 10, OperatorType::Divide, [1.5], null]; + yield 'PostgreSQL: div 9/2.5' => [FractionalEngine::PostgreSQL, 9, OperatorType::Divide, [2.5], null]; + yield 'PostgreSQL: div 10/2 min5.5' => [FractionalEngine::PostgreSQL, 10, OperatorType::Divide, [2, 5.5], 10]; + yield 'PostgreSQL: div 10/0.5' => [FractionalEngine::PostgreSQL, 10, OperatorType::Divide, [0.5], null]; + yield 'PostgreSQL: mod 10%2.75' => [FractionalEngine::PostgreSQL, 10, OperatorType::Modulo, [2.75], 2]; + yield 'PostgreSQL: mod 11%4.5' => [FractionalEngine::PostgreSQL, 11, OperatorType::Modulo, [4.5], 2]; + yield 'PostgreSQL: pow 16^0.5' => [FractionalEngine::PostgreSQL, 16, OperatorType::Power, [0.5], 4]; + yield 'PostgreSQL: pow 10^0.5' => [FractionalEngine::PostgreSQL, 10, OperatorType::Power, [0.5], 3]; + yield 'PostgreSQL: pow 2^2.5' => [FractionalEngine::PostgreSQL, 2, OperatorType::Power, [2.5], 6]; + yield 'PostgreSQL: pow 4^1.5' => [FractionalEngine::PostgreSQL, 4, OperatorType::Power, [1.5], 8]; + yield 'PostgreSQL: pow 3^2 max8.5' => [FractionalEngine::PostgreSQL, 3, OperatorType::Power, [2, 8.5], 3]; + yield 'PostgreSQL: pow 2^3 max8.5' => [FractionalEngine::PostgreSQL, 2, OperatorType::Power, [3, 8.5], 8]; + yield 'PostgreSQL: inc 11+1.5 max20' => [FractionalEngine::PostgreSQL, 11, OperatorType::Increment, [1.5, 20], 13]; + yield 'PostgreSQL: inc 10+1.5 max20' => [FractionalEngine::PostgreSQL, 10, OperatorType::Increment, [1.5, 20], 12]; + yield 'PostgreSQL: inc -12+0.5 max20' => [FractionalEngine::PostgreSQL, -12, OperatorType::Increment, [0.5, 20], -12]; + yield 'PostgreSQL: inc 10+1.25 max20' => [FractionalEngine::PostgreSQL, 10, OperatorType::Increment, [1.25, 20], 11]; + yield 'PostgreSQL: dec 11-0.5 min0' => [FractionalEngine::PostgreSQL, 11, OperatorType::Decrement, [0.5, 0], 11]; + yield 'PostgreSQL: dec 12-0.5 min0' => [FractionalEngine::PostgreSQL, 12, OperatorType::Decrement, [0.5, 0], 12]; + yield 'PostgreSQL: dec -10-0.5 min-100' => [FractionalEngine::PostgreSQL, -10, OperatorType::Decrement, [0.5, -100], -11]; + yield 'PostgreSQL: mul 5*2.5 max100' => [FractionalEngine::PostgreSQL, 5, OperatorType::Multiply, [2.5, 100], 13]; + yield 'PostgreSQL: mul 11*1.5 max100' => [FractionalEngine::PostgreSQL, 11, OperatorType::Multiply, [1.5, 100], 17]; + yield 'PostgreSQL: mul 7*-1.5 max100' => [FractionalEngine::PostgreSQL, 7, OperatorType::Multiply, [-1.5, 100], -11]; + yield 'PostgreSQL: mul 3*1.25 max100' => [FractionalEngine::PostgreSQL, 3, OperatorType::Multiply, [1.25, 100], 4]; + yield 'PostgreSQL: div 9/2.5 min0' => [FractionalEngine::PostgreSQL, 9, OperatorType::Divide, [2.5, 0], 4]; + yield 'PostgreSQL: div 10/1.5 min0' => [FractionalEngine::PostgreSQL, 10, OperatorType::Divide, [1.5, 0], 7]; + yield 'PostgreSQL: div 5/0.4 min0' => [FractionalEngine::PostgreSQL, 5, OperatorType::Divide, [0.4, 0], 13]; + yield 'PostgreSQL: pow 2^2.5 max100' => [FractionalEngine::PostgreSQL, 2, OperatorType::Power, [2.5, 100], 6]; + yield 'PostgreSQL: pow 10^0.5 max100' => [FractionalEngine::PostgreSQL, 10, OperatorType::Power, [0.5, 100], 3]; + yield 'PostgreSQL: mod -10%2.75' => [FractionalEngine::PostgreSQL, -10, OperatorType::Modulo, [2.75], -2]; + yield 'PostgreSQL: mod 10%2.5' => [FractionalEngine::PostgreSQL, 10, OperatorType::Modulo, [2.5], 0]; + yield 'PostgreSQL: mod 11%2.5' => [FractionalEngine::PostgreSQL, 11, OperatorType::Modulo, [2.5], 1]; + yield 'PostgreSQL: mod 7%-2.5' => [FractionalEngine::PostgreSQL, 7, OperatorType::Modulo, [-2.5], 2]; + yield 'PostgreSQL: pow 4^-0.5' => [FractionalEngine::PostgreSQL, 4, OperatorType::Power, [-0.5], 0]; + yield 'PostgreSQL: pow 4^-0.5 max100' => [FractionalEngine::PostgreSQL, 4, OperatorType::Power, [-0.5, 100], 0]; + yield 'PostgreSQL: pow 16^-0.25' => [FractionalEngine::PostgreSQL, 16, OperatorType::Power, [-0.25], 0]; + yield 'PostgreSQL: div 5/2 min0.5' => [FractionalEngine::PostgreSQL, 5, OperatorType::Divide, [2, 0.5], 3]; + yield 'PostgreSQL: div 7/2 min0.5' => [FractionalEngine::PostgreSQL, 7, OperatorType::Divide, [2, 0.5], 4]; + yield 'PostgreSQL: div -5/2 min-10.5' => [FractionalEngine::PostgreSQL, -5, OperatorType::Divide, [2, -10.5], -3]; + yield 'PostgreSQL: div 5/2 min2.6' => [FractionalEngine::PostgreSQL, 5, OperatorType::Divide, [2, 2.6], 5]; + yield 'PostgreSQL: mod 11%3.5' => [FractionalEngine::PostgreSQL, 11, OperatorType::Modulo, [3.5], 1]; + yield 'PostgreSQL: mod 13%3.5' => [FractionalEngine::PostgreSQL, 13, OperatorType::Modulo, [3.5], 3]; + yield 'PostgreSQL: mod -11%3.5' => [FractionalEngine::PostgreSQL, -11, OperatorType::Modulo, [3.5], -1]; + yield 'PostgreSQL: inc 10+2 max10.5' => [FractionalEngine::PostgreSQL, 10, OperatorType::Increment, [2, 10.5], 10]; + yield 'PostgreSQL: mul 5*3 max15.5' => [FractionalEngine::PostgreSQL, 5, OperatorType::Multiply, [3, 15.5], 15]; + yield 'MongoDB: inc 11+1.5' => [FractionalEngine::MongoDB, 11, OperatorType::Increment, [1.5], 12]; + yield 'MongoDB: inc 10+1.5' => [FractionalEngine::MongoDB, 10, OperatorType::Increment, [1.5], 11]; + yield 'MongoDB: inc 10+1.25' => [FractionalEngine::MongoDB, 10, OperatorType::Increment, [1.25], 11]; + yield 'MongoDB: inc 10+1.75' => [FractionalEngine::MongoDB, 10, OperatorType::Increment, [1.75], 11]; + yield 'MongoDB: inc -11+0.5' => [FractionalEngine::MongoDB, -11, OperatorType::Increment, [0.5], -10]; + yield 'MongoDB: inc -12+0.5' => [FractionalEngine::MongoDB, -12, OperatorType::Increment, [0.5], -11]; + yield 'MongoDB: dec 11-0.5' => [FractionalEngine::MongoDB, 11, OperatorType::Decrement, [0.5], 10]; + yield 'MongoDB: dec 12-0.5' => [FractionalEngine::MongoDB, 12, OperatorType::Decrement, [0.5], 11]; + yield 'MongoDB: dec -10-0.5' => [FractionalEngine::MongoDB, -10, OperatorType::Decrement, [0.5], -10]; + yield 'MongoDB: inc 10+2 max13.9' => [FractionalEngine::MongoDB, 10, OperatorType::Increment, [2, 13.9], 12]; + yield 'MongoDB: inc 12+2 max13.9' => [FractionalEngine::MongoDB, 12, OperatorType::Increment, [2, 13.9], 12]; + yield 'MongoDB: inc 12+1.5 max13.9' => [FractionalEngine::MongoDB, 12, OperatorType::Increment, [1.5, 13.9], 13]; + yield 'MongoDB: inc 12+1.5 max13.5' => [FractionalEngine::MongoDB, 12, OperatorType::Increment, [1.5, 13.5], 13]; + yield 'MongoDB: dec 12-1 min11.1' => [FractionalEngine::MongoDB, 12, OperatorType::Decrement, [1, 11.1], 12]; + yield 'MongoDB: dec 13-1 min11.1' => [FractionalEngine::MongoDB, 13, OperatorType::Decrement, [1, 11.1], 12]; + yield 'MongoDB: dec 12-0.5 min11.5' => [FractionalEngine::MongoDB, 12, OperatorType::Decrement, [0.5, 11.5], 11]; + yield 'MongoDB: mul 11*1.5' => [FractionalEngine::MongoDB, 11, OperatorType::Multiply, [1.5], 16]; + yield 'MongoDB: mul 5*2.5' => [FractionalEngine::MongoDB, 5, OperatorType::Multiply, [2.5], 12]; + yield 'MongoDB: mul 7*1.5' => [FractionalEngine::MongoDB, 7, OperatorType::Multiply, [1.5], 10]; + yield 'MongoDB: mul 7*-1.5' => [FractionalEngine::MongoDB, 7, OperatorType::Multiply, [-1.5], -10]; + yield 'MongoDB: mul 3*1.25' => [FractionalEngine::MongoDB, 3, OperatorType::Multiply, [1.25], 3]; + yield 'MongoDB: mul 12*2 max25.5' => [FractionalEngine::MongoDB, 12, OperatorType::Multiply, [2, 25.5], 24]; + yield 'MongoDB: mul 13*2 max25.5' => [FractionalEngine::MongoDB, 13, OperatorType::Multiply, [2, 25.5], 13]; + yield 'MongoDB: mul 10*1.5 max15' => [FractionalEngine::MongoDB, 10, OperatorType::Multiply, [1.5, 15], 15]; + yield 'MongoDB: div 10/1.5' => [FractionalEngine::MongoDB, 10, OperatorType::Divide, [1.5], 6]; + yield 'MongoDB: div 9/2.5' => [FractionalEngine::MongoDB, 9, OperatorType::Divide, [2.5], 3]; + yield 'MongoDB: div 10/2 min5.5' => [FractionalEngine::MongoDB, 10, OperatorType::Divide, [2, 5.5], 10]; + yield 'MongoDB: div 10/0.5' => [FractionalEngine::MongoDB, 10, OperatorType::Divide, [0.5], 20]; + yield 'MongoDB: mod 10%2.75' => [FractionalEngine::MongoDB, 10, OperatorType::Modulo, [2.75], 1]; + yield 'MongoDB: mod 11%4.5' => [FractionalEngine::MongoDB, 11, OperatorType::Modulo, [4.5], 2]; + yield 'MongoDB: pow 16^0.5' => [FractionalEngine::MongoDB, 16, OperatorType::Power, [0.5], 4]; + yield 'MongoDB: pow 10^0.5' => [FractionalEngine::MongoDB, 10, OperatorType::Power, [0.5], 3]; + yield 'MongoDB: pow 2^2.5' => [FractionalEngine::MongoDB, 2, OperatorType::Power, [2.5], 5]; + yield 'MongoDB: pow 4^1.5' => [FractionalEngine::MongoDB, 4, OperatorType::Power, [1.5], 8]; + yield 'MongoDB: pow 3^2 max8.5' => [FractionalEngine::MongoDB, 3, OperatorType::Power, [2, 8.5], 3]; + yield 'MongoDB: pow 2^3 max8.5' => [FractionalEngine::MongoDB, 2, OperatorType::Power, [3, 8.5], 8]; + yield 'MongoDB: inc 11+1.5 max20' => [FractionalEngine::MongoDB, 11, OperatorType::Increment, [1.5, 20], 12]; + yield 'MongoDB: inc 10+1.5 max20' => [FractionalEngine::MongoDB, 10, OperatorType::Increment, [1.5, 20], 11]; + yield 'MongoDB: inc -12+0.5 max20' => [FractionalEngine::MongoDB, -12, OperatorType::Increment, [0.5, 20], -11]; + yield 'MongoDB: inc 10+1.25 max20' => [FractionalEngine::MongoDB, 10, OperatorType::Increment, [1.25, 20], 11]; + yield 'MongoDB: dec 11-0.5 min0' => [FractionalEngine::MongoDB, 11, OperatorType::Decrement, [0.5, 0], 10]; + yield 'MongoDB: dec 12-0.5 min0' => [FractionalEngine::MongoDB, 12, OperatorType::Decrement, [0.5, 0], 11]; + yield 'MongoDB: dec -10-0.5 min-100' => [FractionalEngine::MongoDB, -10, OperatorType::Decrement, [0.5, -100], -10]; + yield 'MongoDB: mul 5*2.5 max100' => [FractionalEngine::MongoDB, 5, OperatorType::Multiply, [2.5, 100], 12]; + yield 'MongoDB: mul 11*1.5 max100' => [FractionalEngine::MongoDB, 11, OperatorType::Multiply, [1.5, 100], 16]; + yield 'MongoDB: mul 7*-1.5 max100' => [FractionalEngine::MongoDB, 7, OperatorType::Multiply, [-1.5, 100], -10]; + yield 'MongoDB: mul 3*1.25 max100' => [FractionalEngine::MongoDB, 3, OperatorType::Multiply, [1.25, 100], 3]; + yield 'MongoDB: div 9/2.5 min0' => [FractionalEngine::MongoDB, 9, OperatorType::Divide, [2.5, 0], 3]; + yield 'MongoDB: div 10/1.5 min0' => [FractionalEngine::MongoDB, 10, OperatorType::Divide, [1.5, 0], 6]; + yield 'MongoDB: div 5/0.4 min0' => [FractionalEngine::MongoDB, 5, OperatorType::Divide, [0.4, 0], 12]; + yield 'MongoDB: pow 2^2.5 max100' => [FractionalEngine::MongoDB, 2, OperatorType::Power, [2.5, 100], 5]; + yield 'MongoDB: pow 10^0.5 max100' => [FractionalEngine::MongoDB, 10, OperatorType::Power, [0.5, 100], 3]; + yield 'MongoDB: mod -10%2.75' => [FractionalEngine::MongoDB, -10, OperatorType::Modulo, [2.75], -1]; + yield 'MongoDB: mod 10%2.5' => [FractionalEngine::MongoDB, 10, OperatorType::Modulo, [2.5], 0]; + yield 'MongoDB: mod 11%2.5' => [FractionalEngine::MongoDB, 11, OperatorType::Modulo, [2.5], 1]; + yield 'MongoDB: mod 7%-2.5' => [FractionalEngine::MongoDB, 7, OperatorType::Modulo, [-2.5], 2]; + yield 'MongoDB: pow 4^-0.5' => [FractionalEngine::MongoDB, 4, OperatorType::Power, [-0.5], 0]; + yield 'MongoDB: pow 4^-0.5 max100' => [FractionalEngine::MongoDB, 4, OperatorType::Power, [-0.5, 100], 0]; + yield 'MongoDB: pow 16^-0.25' => [FractionalEngine::MongoDB, 16, OperatorType::Power, [-0.25], 0]; + yield 'MongoDB: div 5/2 min0.5' => [FractionalEngine::MongoDB, 5, OperatorType::Divide, [2, 0.5], 2]; + yield 'MongoDB: div 7/2 min0.5' => [FractionalEngine::MongoDB, 7, OperatorType::Divide, [2, 0.5], 3]; + yield 'MongoDB: div -5/2 min-10.5' => [FractionalEngine::MongoDB, -5, OperatorType::Divide, [2, -10.5], -2]; + yield 'MongoDB: div 5/2 min2.6' => [FractionalEngine::MongoDB, 5, OperatorType::Divide, [2, 2.6], 5]; + yield 'MongoDB: mod 11%3.5' => [FractionalEngine::MongoDB, 11, OperatorType::Modulo, [3.5], 0]; + yield 'MongoDB: mod 13%3.5' => [FractionalEngine::MongoDB, 13, OperatorType::Modulo, [3.5], 2]; + yield 'MongoDB: mod -11%3.5' => [FractionalEngine::MongoDB, -11, OperatorType::Modulo, [3.5], 0]; + yield 'MongoDB: inc 10+2 max10.5' => [FractionalEngine::MongoDB, 10, OperatorType::Increment, [2, 10.5], 10]; + yield 'MongoDB: mul 5*3 max15.5' => [FractionalEngine::MongoDB, 5, OperatorType::Multiply, [3, 15.5], 15]; + } + + /** + * Fractional counter changes: the engine, whether it increases, the start, the change, the bound and the value + * stored, or null where the engine refused the change. MariaDB rounded half to even. PostgreSQL bound the + * change next to an integer column and refused any fraction. MongoDB stored the double and every read cut it + * toward zero; these rows cut each change on its own, which matches it while one change is applied at a time. + * + * @return iterable + */ + public static function counters(): iterable + { + $changes = [ + 'increase rounds a half down to even' => [true, 5, 1.5, null, [6, 6]], + 'increase rounds a half up to even' => [true, 6, 1.5, null, [8, 7]], + 'increase rounds below a half down' => [true, 5, 1.4, null, [6, 6]], + 'increase rounds above a half up' => [true, 5, 1.6, null, [7, 6]], + 'increase rounds a negative half to even' => [true, -10, 2.5, null, [-8, -7]], + 'increase by a half that rounds back to the value' => [true, 4, 0.5, null, [4, 4]], + 'increase by a half that rounds up' => [true, 5, 0.5, null, [6, 5]], + 'increase within a fractional maximum' => [true, 5, 1.5, 6.5, [6, 6]], + 'increase within a whole maximum' => [true, 5, 1.5, 7, [6, 6]], + 'increase a big integer' => [true, 5, 2.5, null, [8, 7]], + 'decrease rounds a half up to even' => [false, 5, 1.5, null, [4, 3]], + 'decrease rounds a half down to even' => [false, 6, 1.5, null, [4, 4]], + 'decrease by less than a half keeps the value' => [false, 5, 0.4, null, [5, 4]], + 'decrease within a fractional minimum' => [false, 5, 1.5, 3.5, [4, 3]], + 'increase beyond float precision rounds half to even' => [true, 9007199254740993, 1.5, null, [9007199254740994, 9007199254740994]], + 'increase beyond float precision rounds up past the half' => [true, 9007199254740993, 1.75, null, [9007199254740995, 9007199254740994]], + 'increase beyond float precision rounds down below the half' => [true, 9007199254740993, 1.25, null, [9007199254740994, 9007199254740994]], + 'decrease beyond float precision rounds half to even' => [false, 9007199254740995, 1.5, null, [9007199254740994, 9007199254740993]], + 'decrease beyond float precision rounds down past the half' => [false, 9007199254740995, 1.75, null, [9007199254740993, 9007199254740993]], + 'decrease beyond float precision rounds up below the half' => [false, 9007199254740995, 1.25, null, [9007199254740994, 9007199254740993]], + ]; + + foreach ($changes as $name => [$increase, $start, $value, $bound, [$mariaDB, $mongoDB]]) { + yield "MariaDB: {$name}" => [FractionalEngine::MariaDB, $increase, $start, $value, $bound, $mariaDB]; + yield "PostgreSQL: {$name}" => [FractionalEngine::PostgreSQL, $increase, $start, $value, $bound, null]; + yield "MongoDB: {$name}" => [FractionalEngine::MongoDB, $increase, $start, $value, $bound, $mongoDB]; + } + + foreach (FractionalEngine::cases() as $engine) { + yield "{$engine->name}: a whole float change is stored whole" => [$engine, true, 8, 2.0, null, 10]; + } + } +} From f5c1f2a95bab3483cc6e54a46b46aac0255b58af Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 16:48:54 +1300 Subject: [PATCH 05/55] feat(database): add placeholder replacement and grant check toggles appwrite generated IDs for nested 'unique()' documents and checked the roles a caller grants on related documents app-side, because the library stores 'unique()' literally (a deliberate 8.0 decision) and never compares granted roles with held ones. Both become opt-in library toggles so the default stays 8.0.3's. Id::PLACEHOLDER and Id::isPlaceholder() name the placeholder. The toggles follow the setPreserveDates idiom: State\Value-backed so they are scoped to the calling coroutine, carried by Snapshot (trailing defaulted parameters keep existing constructions valid) and restored by withSnapshot(), and forwarded by Mirror to its source and destination like the other toggles. Co-Authored-By: Claude Opus 5.5 --- src/Database/Database.php | 96 ++++++++++++++++++++++++++++++++- src/Database/Id.php | 10 ++++ src/Database/Mirror.php | 34 ++++++++++++ src/Database/State/Snapshot.php | 4 ++ 4 files changed, 143 insertions(+), 1 deletion(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index aa4f3fd5d..c45e3eb79 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -266,6 +266,12 @@ class Database /** @var Value|null */ private ?Value $ignoringDuplicates = null; + /** @var Value|null */ + private ?Value $placeholderReplacement = null; + + /** @var Value|null */ + private ?Value $grantChecking = null; + protected ?Relationships $relationshipHook = null; protected bool $dropUnknownAttributes = false; @@ -1150,6 +1156,74 @@ public function withPreserveSequence(bool $preserve, callable $callback): mixed return $this->sequencePreservation()->with($preserve, $callback); } + /** + * Set whether a document whose ID is Id::PLACEHOLDER gets a generated ID, as one whose ID is empty does. This + * covers the documents given to createDocument(), createDocuments(), upsertDocument() and upsertDocuments(), and + * every related document they or updateDocument() are given as a document, as deep as the write relates them. A + * related document given as the string Id::PLACEHOLDER stays a reference to the document stored under that ID. + * + * @return $this + */ + public function setReplacePlaceholders(bool $replace): static + { + $this->placeholderReplacement()->set($replace); + + return $this; + } + + public function isReplacingPlaceholders(): bool + { + return $this->placeholderReplacement()->get(); + } + + /** + * Run the callback with placeholder replacement on or off for the calling coroutine and the coroutines it starts. + * + * @template T + * + * @param callable(): T $callback + * @return T + */ + public function withReplacePlaceholders(bool $replace, callable $callback): mixed + { + return $this->placeholderReplacement()->with($replace, $callback); + } + + /** + * Set whether, while authorization is enabled, a related document given as a document to createDocument(), + * createDocuments(), updateDocument(), upsertDocument(), upsertDocuments(), authorizeCreate() or authorizeUpdate() + * may only add permissions for roles the caller holds, as deep as the write relates it. Permissions it already + * has may be sent back, and permissions it inherits from its parent are never checked. Malformed permissions + * throw a structure exception and an unheld role an authorization exception, before anything is written. + * + * @return $this + */ + public function setCheckGrants(bool $check): static + { + $this->grantChecking()->set($check); + + return $this; + } + + public function isCheckingGrants(): bool + { + return $this->grantChecking()->get(); + } + + /** + * Run the callback with the grant check on related documents on or off for the calling coroutine and the + * coroutines it starts. + * + * @template T + * + * @param callable(): T $callback + * @return T + */ + public function withCheckGrants(bool $check, callable $callback): mixed + { + return $this->grantChecking()->with($check, $callback); + } + /** * Set the migration mode flag, which relaxes certain constraints during data migrations. * @@ -1555,6 +1629,8 @@ public function snapshot(): Snapshot preserveSequence: $this->sequencePreservation()->get(), ignoreDuplicates: $this->ignoringDuplicates()->get(), requestTimestamp: $this->requestTimestamp()->get(), + replacePlaceholders: $this->placeholderReplacement()->get(), + checkGrants: $this->grantChecking()->get(), ); } @@ -1595,7 +1671,9 @@ public function withSnapshot(Snapshot $snapshot, callable $callback): mixed */ private function withToggles(Snapshot $snapshot, callable $callback): mixed { - $timestamped = fn (): mixed => $this->requestTimestamp()->with($snapshot->requestTimestamp, $callback); + $checked = fn (): mixed => $this->grantChecking()->with($snapshot->checkGrants, $callback); + $replaced = fn (): mixed => $this->placeholderReplacement()->with($snapshot->replacePlaceholders, $checked); + $timestamped = fn (): mixed => $this->requestTimestamp()->with($snapshot->requestTimestamp, $replaced); $deduplicated = fn (): mixed => $this->ignoringDuplicates()->with($snapshot->ignoreDuplicates, $timestamped); $sequenced = fn (): mixed => $this->sequencePreservation()->with($snapshot->preserveSequence, $deduplicated); $dated = fn (): mixed => $this->datePreservation()->with($snapshot->preserveDates, $sequenced); @@ -1674,6 +1752,22 @@ private function ignoringDuplicates(): Value return $this->ignoringDuplicates ??= new Value(false); } + /** + * @return Value + */ + private function placeholderReplacement(): Value + { + return $this->placeholderReplacement ??= new Value(false); + } + + /** + * @return Value + */ + private function grantChecking(): Value + { + return $this->grantChecking ??= new Value(false); + } + protected function isIgnoringDuplicates(): bool { return $this->ignoringDuplicates()->get(); diff --git a/src/Database/Id.php b/src/Database/Id.php index ed63cecef..da34edd0c 100644 --- a/src/Database/Id.php +++ b/src/Database/Id.php @@ -10,6 +10,11 @@ */ class Id { + /** + * The ID a caller sends to ask for a generated one, replaced only where the database replaces placeholders. + */ + public const string PLACEHOLDER = 'unique()'; + /** * Create a new unique ID using uniqid with optional random padding. * @@ -44,4 +49,9 @@ public static function custom(string $id): string { return $id; } + + public static function isPlaceholder(string $id): bool + { + return $id === self::PLACEHOLDER; + } } diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php index b34e0d1f9..ff029a3fd 100644 --- a/src/Database/Mirror.php +++ b/src/Database/Mirror.php @@ -482,6 +482,22 @@ public function setPreserveSequence(bool $preserve): static return parent::setPreserveSequence($preserve); } + #[\Override] + public function setReplacePlaceholders(bool $replace): static + { + $this->delegate(__FUNCTION__, \func_get_args()); + + return parent::setReplacePlaceholders($replace); + } + + #[\Override] + public function setCheckGrants(bool $check): static + { + $this->delegate(__FUNCTION__, \func_get_args()); + + return parent::setCheckGrants($check); + } + #[\Override] public function setValidation(bool $validation): static { @@ -530,6 +546,24 @@ public function withPreserveSequence(bool $preserve, callable $callback): mixed return parent::withPreserveSequence($preserve, fn (): mixed => $this->source->withPreserveSequence($preserve, $scoped)); } + #[\Override] + public function withReplacePlaceholders(bool $replace, callable $callback): mixed + { + $destination = $this->destination; + $scoped = $destination === null ? $callback : fn (): mixed => $destination->withReplacePlaceholders($replace, $callback); + + return parent::withReplacePlaceholders($replace, fn (): mixed => $this->source->withReplacePlaceholders($replace, $scoped)); + } + + #[\Override] + public function withCheckGrants(bool $check, callable $callback): mixed + { + $destination = $this->destination; + $scoped = $destination === null ? $callback : fn (): mixed => $destination->withCheckGrants($check, $callback); + + return parent::withCheckGrants($check, fn (): mixed => $this->source->withCheckGrants($check, $scoped)); + } + #[\Override] public function skipRelationships(callable $callback): mixed { diff --git a/src/Database/State/Snapshot.php b/src/Database/State/Snapshot.php index 7e644abb1..5bd5130fc 100644 --- a/src/Database/State/Snapshot.php +++ b/src/Database/State/Snapshot.php @@ -27,6 +27,8 @@ * @param bool $preserveSequence Whether writes keep the sequences they are given * @param bool $ignoreDuplicates Whether creating a document that exists is skipped instead of failing * @param DateTime|null $requestTimestamp The time an update conflicts after + * @param bool $replacePlaceholders Whether a related document sent with an Id::PLACEHOLDER ID gets a generated one + * @param bool $checkGrants Whether a related document may only be given permissions for roles the caller holds */ public function __construct( public bool $authorization, @@ -44,6 +46,8 @@ public function __construct( public bool $preserveSequence, public bool $ignoreDuplicates, public ?DateTime $requestTimestamp, + public bool $replacePlaceholders = false, + public bool $checkGrants = false, ) { } } From 5da298624f3eebd27fdb1e3908db7277afe1502a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 16:48:55 +1300 Subject: [PATCH 06/55] feat(relationships): prepare nested documents before a write One recursive pre-pass, Relationships::prepareNested(), walks the related documents a write is given as documents, as deep as relateAttributes() relates them, before anything is written. With placeholders replaced, an empty or 'unique()' ID gets a generated one up front, so links, junctions and the prepared-create path all see the final ID; a string 'unique()' stays a reference to the stored document. With grants checked and authorization enabled, permissions a related document adds over its current ones (none when new, else the staged view or the stored document) must name roles the caller holds, matching appwrite's RelatedPermissions text, and malformed ones are a structure error. It runs before the parent passes its permissions down, so inherited permissions are never checked. createDocument, createDocuments, updateDocument and upsertDocument(s) run it at the root (all documents before the first is related, so a refusal writes nothing); nested writes skip it because the root already prepared their subtree, which also keeps documents past the write depth unchecked. authorizeCreate/authorizeUpdate run it on a copy with the staged view, so a generated ID is judged new and needs create. The root is changed in place (only its relationship values, and only once the walk succeeds) because Mirror replicates the caller's documents after the source wrote them; the caller's nested documents are copied, never changed. With both toggles off it returns before any walk. Co-Authored-By: Claude Opus 5.5 --- src/Database/Hook/Relationships.php | 188 ++++++ src/Database/Trait/Documents.php | 80 ++- .../Relationships/NestedPreparationTest.php | 567 ++++++++++++++++++ 3 files changed, 831 insertions(+), 4 deletions(-) create mode 100644 tests/unit/Relationships/NestedPreparationTest.php diff --git a/src/Database/Hook/Relationships.php b/src/Database/Hook/Relationships.php index 1ce899a0c..ae1325e7d 100644 --- a/src/Database/Hook/Relationships.php +++ b/src/Database/Hook/Relationships.php @@ -20,6 +20,7 @@ use Utopia\Database\Exception\Relationship as RelationshipException; use Utopia\Database\Exception\Restricted as RestrictedException; use Utopia\Database\Exception\Structure as StructureException; +use Utopia\Database\Id; use Utopia\Database\Operator; use Utopia\Database\OperatorType; use Utopia\Database\Permission; @@ -33,6 +34,7 @@ use Utopia\Database\State\Snapshot; use Utopia\Database\State\Value; use Utopia\Database\Validator\Authorization\Input; +use Utopia\Database\Validator\Permissions as PermissionsValidator; use Utopia\Query\Hook; use Utopia\Query\Method; @@ -3165,6 +3167,192 @@ private function authorizeLink(Document $collection, Document $related): void } } + /** + * Prepare the related documents $document holds as documents for a write of it to $collection, as deep as the + * write relates them, replacing its relationship values with prepared copies once every one is prepared; the + * related documents given are not changed. Where the database replaces placeholders, each whose ID is empty or + * Id::PLACEHOLDER gets a generated ID, so the write creates it. Where the database checks grants and + * authorization is enabled, each that carries permissions may only add permissions for roles the caller holds: + * a new one has none yet, any other has those of the document $resolve(collection, ID) gives, or of the stored + * one when $resolve gives null or is null. A write nested in another relationship write is left as given, as the + * outermost write prepared it. + * + * @param (Closure(string, string): ?Document)|null $resolve + * + * @throws AuthorizationException When a related document adds a permission for a role the caller does not hold + * @throws RelationshipException When a related document given as an array is not a valid document + * @throws StructureException When a related document's permissions are malformed + * + * @internal + */ + public function prepareNested(Document $collection, Document $document, ?Closure $resolve = null): Document + { + $replace = $this->database->isReplacingPlaceholders(); + $check = $this->database->isCheckingGrants() && $this->database->getAuthorization()->getStatus(); + if ((! $replace && ! $check) || isset($this->writeStacks[$this->coroutine()])) { + return $document; + } + + foreach ($this->prepareNestedValues($collection, $document, [], $replace, $check, $resolve) as $key => $value) { + $document->setAttribute($key, $value); + } + + return $document; + } + + /** + * The relationship values of $document that hold related documents as documents, with copies of those + * documents prepared. + * + * @param list $writeStack The collections of the writes relating $document, innermost last + * @param (Closure(string, string): ?Document)|null $resolve + * @return array + * + * @throws AuthorizationException + * @throws RelationshipException + * @throws StructureException + */ + private function prepareNestedValues(Document $collection, Document $document, array $writeStack, bool $replace, bool $check, ?Closure $resolve): array + { + $depth = \count($writeStack); + $prepared = []; + + foreach (self::relationships($collection) as $attribute) { + $relationship = $attribute->relationship; + if ($relationship === null) { + continue; + } + + $value = $document->getAttribute($attribute->key); + $list = \is_array($value) && \array_is_list($value); + $values = $list ? $value : [$value]; + /** @var array> $related */ + $related = \array_filter($values, self::isDocumentValue(...)); + if ($related === []) { + continue; + } + + $relatedCollection = $this->database->getCollection($relationship->relatedCollection); + if ($depth >= Database::RELATION_MAX_DEPTH - 1 && $writeStack[$depth - 1] !== $relatedCollection->getId()) { + continue; + } + + foreach ($related as $index => $nested) { + $values[$index] = $this->prepareNestedDocument( + $relatedCollection, + $nested instanceof Document ? clone $nested : self::toDocument($nested), + [...$writeStack, $collection->getId()], + $replace, + $check, + $resolve, + ); + } + + $prepared[$attribute->key] = $list ? $values : $values[0]; + } + + return $prepared; + } + + /** + * @param list $writeStack + * @param (Closure(string, string): ?Document)|null $resolve + * + * @throws AuthorizationException + * @throws RelationshipException + * @throws StructureException + */ + private function prepareNestedDocument(Document $collection, Document $document, array $writeStack, bool $replace, bool $check, ?Closure $resolve): Document + { + $id = $document->getId(); + $new = $id === ''; + if ($replace && ($new || Id::isPlaceholder($id))) { + $document->setAttribute(Document::ID, Id::unique()); + $new = true; + } + + if ($check) { + $this->authorizeGrants($collection, $document, $new, $resolve); + } + + foreach ($this->prepareNestedValues($collection, $document, $writeStack, $replace, $check, $resolve) as $key => $value) { + $document->setAttribute($key, $value); + } + + return $document; + } + + /** + * Refuse permissions $document adds over its current ones for a role the caller does not hold. + * + * @param (Closure(string, string): ?Document)|null $resolve + * + * @throws AuthorizationException + * @throws StructureException + */ + private function authorizeGrants(Document $collection, Document $document, bool $new, ?Closure $resolve): void + { + $permissions = $document->getAttribute(Document::PERMISSIONS); + if ($permissions === null) { + return; + } + + $validator = new PermissionsValidator(); + if (! $validator->isValid($permissions)) { + throw new StructureException($validator->getDescription()); + } + + /** @var list $permissions */ + $current = $new ? [] : (Permission::aggregate($this->grantedView($collection, $document->getId(), $resolve)->getPermissions()) ?? []); + $authorization = $this->database->getAuthorization(); + + foreach (\array_diff(Permission::aggregate($permissions) ?? [], $current) as $granted) { + $permission = Permission::parse($granted); + $role = new Role($permission->getRole(), $permission->getIdentifier(), $permission->getDimension()); + if (! $authorization->hasRole($role->toString())) { + throw new AuthorizationException('Permissions must be one of: ('.\implode(', ', $authorization->getRoles()).')'); + } + } + } + + /** + * The related document $id as a write would find it, whether or not the caller can read it: the document + * $resolve gives for it, else the stored one. + * + * @param (Closure(string, string): ?Document)|null $resolve + */ + private function grantedView(Document $collection, string $id, ?Closure $resolve): Document + { + $staged = $resolve === null ? null : $resolve($collection->getId(), $id); + + return $staged ?? $this->database->getAuthorization()->skip(fn (): Document => $this->database->skipRelationships( + fn (): Document => $this->database->getDocument($collection->getId(), $id, [Query::select([Document::PERMISSIONS])]) + )); + } + + private static function isDocumentValue(mixed $value): bool + { + return $value instanceof Document || (\is_array($value) && ! \array_is_list($value)); + } + + /** + * @param Document|array $value + * + * @throws RelationshipException + */ + private static function toDocument(Document|array $value): Document + { + if ($value instanceof Document) { + return $value; + } + + try { + return new Document($value); + } catch (StructureException $error) { + throw new RelationshipException('Invalid relationship value. '.$error->getMessage()); + } + } + /** * Check, without writing, that the caller may relate the related documents $document carries the way writing it * relates them, as deep as a write relates them: create on the related collection for a new document, update diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 16e43b7f3..390826849 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -1072,7 +1072,7 @@ public function authorizeCreate(string $collection, Document $document, ?callabl } if ($this->relationshipHook?->isEnabled()) { - $this->relationshipHook->authorize($definition, $document, resolve: $resolve); + $this->relationshipHook->authorize($definition, $this->prepareNestedCopy($definition, $document, $resolve), resolve: $resolve); } } @@ -1105,7 +1105,7 @@ public function authorizeUpdate(string $collection, Document $old, Document $doc } if ($this->relationshipHook?->isEnabled()) { - $this->relationshipHook->authorize($definition, $document, $old, $resolve); + $this->relationshipHook->authorize($definition, $this->prepareNestedCopy($definition, $document, $resolve), $old, $resolve); } } @@ -1124,6 +1124,7 @@ public function createDocument(string $collection, Document $document): Document $collection = $this->silent(fn () => $this->getCollection($collection)); + $document = $this->prepareNested($collection, $document); $document = $this->prepareDocument($collection, $document); /** @var array}> $copies */ @@ -1232,6 +1233,58 @@ public function createPrepared(array $documents): void }); } + /** + * Whether a document given $id is created under a generated ID. + */ + private function generatesId(string $id): bool + { + return empty($id) || (Id::isPlaceholder($id) && $this->placeholderReplacement()->get()); + } + + private function preparesNested(): bool + { + return $this->placeholderReplacement()->get() || $this->grantChecking()->get(); + } + + /** + * The document with its related documents prepared by the relationship hook, when placeholders are replaced or + * grants checked. + * + * @param (Closure(string, string): ?Document)|null $resolve + * + * @throws AuthorizationException + * @throws RelationshipException + * @throws StructureException + */ + private function prepareNested(Document $collection, Document $document, ?Closure $resolve = null): Document + { + $hook = $this->relationshipHook; + if ($hook === null || ! $this->preparesNested() || ! $hook->isEnabled()) { + return $document; + } + + return $this->silent(fn (): Document => $hook->prepareNested($collection, $document, $resolve)); + } + + /** + * A copy of $document as prepareNested() prepares it, leaving $document unchanged; $document itself when + * neither placeholders are replaced nor grants checked. + * + * @param (callable(string, string): ?Document)|null $resolve + * + * @throws AuthorizationException + * @throws RelationshipException + * @throws StructureException + */ + private function prepareNestedCopy(Document $collection, Document $document, ?callable $resolve): Document + { + if (! $this->preparesNested()) { + return $document; + } + + return $this->prepareNested($collection, clone $document, $resolve === null ? null : $resolve(...)); + } + /** * @throws DatabaseException */ @@ -1275,7 +1328,7 @@ private function prepareDocument(Document $collection, Document $document): Docu $id = $document->getId(); $document - ->setAttribute(Document::ID, empty($id) ? Id::unique() : $id) + ->setAttribute(Document::ID, $this->generatesId($id) ? Id::unique() : $id) ->setAttribute(Document::COLLECTION, $collection->getId()) ->setAttribute(Document::CREATED_AT, ($createdAt === null || ! $this->datePreservation()->get()) ? $time : $createdAt) ->setAttribute(Document::UPDATED_AT, ($updatedAt === null || ! $this->datePreservation()->get()) ? $time : $updatedAt); @@ -1371,6 +1424,12 @@ public function createDocuments( $modified = 0; $hasRelationships = self::relationshipAttributes($collection) !== []; + if ($hasRelationships && $this->preparesNested()) { + foreach ($documents as $index => $document) { + $documents[$index] = $this->prepareNested($collection, $document); + } + } + // Hoisted: validator only depends on the collection + adapter properties, // both stable for this call. Allocating once and reusing across all // documents avoids per-document construction and (with the in-class @@ -1387,7 +1446,7 @@ public function createDocuments( $updatedAt = $document->getUpdatedAt(); $document - ->setAttribute(Document::ID, empty($document->getId()) ? Id::unique() : $document->getId()) + ->setAttribute(Document::ID, $this->generatesId($document->getId()) ? Id::unique() : $document->getId()) ->setAttribute(Document::COLLECTION, $collection->getId()) ->setAttribute(Document::CREATED_AT, ($createdAt === null || ! $this->datePreservation()->get()) ? $time : $createdAt) ->setAttribute(Document::UPDATED_AT, ($updatedAt === null || ! $this->datePreservation()->get()) ? $time : $updatedAt); @@ -1495,6 +1554,7 @@ public function updateDocument(string $collection, string $id, Document $documen } $collection = $this->silent(fn () => $this->getCollection($collection)); + $document = $this->prepareNested($collection, $document); $newUpdatedAt = $document->getUpdatedAt(); $hasOperators = false; $cacheTarget = $collection->getId() === self::METADATA @@ -2160,6 +2220,18 @@ private function upsert( $operatorIds = []; $seenIds = []; $hasRelationships = self::relationshipAttributes($collection) !== []; + $replacing = $this->placeholderReplacement()->get(); + $preparesNested = $hasRelationships && $this->preparesNested(); + if ($replacing || $preparesNested) { + foreach ($documents as $key => $document) { + if ($replacing && Id::isPlaceholder($document->getId())) { + $document->setAttribute(Document::ID, Id::unique()); + } + if ($preparesNested) { + $documents[$key] = $this->prepareNested($collection, $document); + } + } + } $internalKeys = \array_flip(\array_map( static fn (Attribute $attribute): string => $attribute->key, self::internalAttributesFor(true) diff --git a/tests/unit/Relationships/NestedPreparationTest.php b/tests/unit/Relationships/NestedPreparationTest.php new file mode 100644 index 000000000..d4685c489 --- /dev/null +++ b/tests/unit/Relationships/NestedPreparationTest.php @@ -0,0 +1,567 @@ + + */ + public static function types(): iterable + { + foreach (self::adapters() as $adapterName => [$adapter]) { + foreach (RelationshipType::cases() as $type) { + yield "{$adapterName}, {$type->value}" => [$adapter, $type]; + } + } + } + + /** + * @return iterable + */ + public static function adapters(): iterable + { + yield 'memory' => [static fn (): Adapter => new Memory()]; + yield 'sqlite' => [static fn (): Adapter => new SQLite(new PDO('sqlite::memory:'))]; + } + + #[DataProvider('types')] + public function testPlaceholdersGetGeneratedIdsAtEveryDepth(Closure $adapter, RelationshipType $type): void + { + $database = $this->database($adapter(), $type)->setReplacePlaceholders(true); + $grandchild = new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'grandchild']); + $child = new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'child', self::NEXT => self::value($type, $grandchild)]); + $root = new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'root', self::NEXT => self::value($type, $child)]); + + $created = $database->createDocument(self::FIRST, $root); + + $rootId = $this->assertGeneratedId($database, self::FIRST, 'root'); + $childId = $this->assertGeneratedId($database, self::SECOND, 'child'); + $grandchildId = $this->assertGeneratedId($database, self::THIRD, 'grandchild'); + $this->assertSame($rootId, $created->getId()); + $this->assertSame([$childId], self::ids($database->getDocument(self::FIRST, $rootId)->getAttribute(self::NEXT))); + $this->assertSame([$grandchildId], self::ids($database->getDocument(self::SECOND, $childId)->getAttribute(self::NEXT))); + $this->assertSame(Id::PLACEHOLDER, $child->getId(), 'The caller\'s nested document is not changed'); + $this->assertSame(Id::PLACEHOLDER, $grandchild->getId(), 'The caller\'s nested document is not changed'); + } + + #[DataProvider('adapters')] + public function testAPlaceholderThreeLevelsDownGetsAGeneratedId(Closure $adapter): void + { + $database = $this->database($adapter(), RelationshipType::ManyToMany)->setReplacePlaceholders(true); + + $database->createDocument(self::FIRST, new Document([ + Document::ID => 'root', + self::NEXT => [new Document([ + Document::ID => Id::PLACEHOLDER, + 'name' => 'child', + self::NEXT => [new Document([ + Document::ID => Id::PLACEHOLDER, + 'name' => 'grandchild', + self::PREVIOUS => [new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'great-grandchild'])], + ])], + ])], + ])); + + $this->assertGeneratedId($database, self::SECOND, 'child'); + $this->assertGeneratedId($database, self::SECOND, 'great-grandchild'); + $this->assertGeneratedId($database, self::THIRD, 'grandchild'); + } + + #[DataProvider('types')] + public function testAnEmptyIdIsGeneratedAndAnExplicitIdIsKept(Closure $adapter, RelationshipType $type): void + { + $database = $this->database($adapter(), $type)->setReplacePlaceholders(true); + + $database->createDocument(self::FIRST, new Document([ + Document::ID => 'root', + self::NEXT => self::value($type, new Document([ + Document::ID => 'chosen', + 'name' => 'child', + self::NEXT => self::value($type, new Document(['name' => 'grandchild'])), + ])), + ])); + + $this->assertSame('child', $database->getDocument(self::SECOND, 'chosen')->getAttribute('name')); + $this->assertGeneratedId($database, self::THIRD, 'grandchild'); + } + + #[DataProvider('types')] + public function testAStringPlaceholderStaysAReference(Closure $adapter, RelationshipType $type): void + { + $database = $this->database($adapter(), $type); + $database->createDocument(self::SECOND, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'literal'])); + $database->setReplacePlaceholders(true); + + $database->createDocument(self::FIRST, new Document([Document::ID => 'linked', self::NEXT => self::value($type, Id::PLACEHOLDER)])); + + $this->assertSame([Id::PLACEHOLDER], self::ids($database->getDocument(self::FIRST, 'linked')->getAttribute(self::NEXT))); + $this->assertSame(['literal'], self::names($database, self::SECOND), 'Linking by the string creates nothing'); + } + + #[DataProvider('types')] + public function testAStringPlaceholderWithoutALiteralDocumentLinksNothing(Closure $adapter, RelationshipType $type): void + { + $database = $this->database($adapter(), $type)->setReplacePlaceholders(true); + + $database->createDocument(self::FIRST, new Document([Document::ID => 'linked', self::NEXT => self::value($type, Id::PLACEHOLDER)])); + + $this->assertSame([], self::ids($database->getDocument(self::FIRST, 'linked')->getAttribute(self::NEXT))); + $this->assertSame([], self::names($database, self::SECOND)); + } + + #[DataProvider('types')] + public function testEveryWriteGeneratesNewIds(Closure $adapter, RelationshipType $type): void + { + $database = $this->database($adapter(), $type)->setReplacePlaceholders(true); + $root = static fn (string $id): Document => new Document([ + Document::ID => $id, + self::NEXT => self::value($type, new Document([Document::ID => Id::PLACEHOLDER, 'name' => $id])), + ]); + + $database->createDocument(self::FIRST, $root('one')); + $database->createDocuments(self::FIRST, [$root('two'), $root('three')]); + + $one = $this->assertGeneratedId($database, self::SECOND, 'one'); + $two = $this->assertGeneratedId($database, self::SECOND, 'two'); + $three = $this->assertGeneratedId($database, self::SECOND, 'three'); + $this->assertCount(3, \array_unique([$one, $two, $three])); + $this->assertSame([$two], self::ids($database->getDocument(self::FIRST, 'two')->getAttribute(self::NEXT))); + } + + #[DataProvider('adapters')] + public function testTopLevelPlaceholdersAreGenerated(Closure $adapter): void + { + $database = $this->database($adapter(), RelationshipType::ManyToOne)->setReplacePlaceholders(true); + + $database->createDocument(self::SECOND, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'single'])); + $database->createDocuments(self::SECOND, [new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'batched'])]); + if ($database->getAdapter()->hasFeature(Upserts::class)) { + $upserted = $database->upsertDocument(self::SECOND, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'upserted'])); + $database->upsertDocuments(self::SECOND, [new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'upserted in bulk'])]); + $this->assertSame($upserted->getId(), $this->assertGeneratedId($database, self::SECOND, 'upserted')); + $this->assertGeneratedId($database, self::SECOND, 'upserted in bulk'); + } + + $this->assertGeneratedId($database, self::SECOND, 'single'); + $this->assertGeneratedId($database, self::SECOND, 'batched'); + } + + #[DataProvider('types')] + public function testAnUpdateGeneratesIdsForNestedPlaceholders(Closure $adapter, RelationshipType $type): void + { + $database = $this->database($adapter(), $type); + $database->createDocument(self::FIRST, new Document([Document::ID => 'root', 'name' => 'root'])); + $database->setReplacePlaceholders(true); + + $database->updateDocument(self::FIRST, 'root', new Document([ + self::NEXT => self::value($type, new Document([ + Document::ID => Id::PLACEHOLDER, + 'name' => 'child', + self::NEXT => self::value($type, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'grandchild'])), + ])), + ])); + + $childId = $this->assertGeneratedId($database, self::SECOND, 'child'); + $this->assertGeneratedId($database, self::THIRD, 'grandchild'); + $this->assertSame([$childId], self::ids($database->getDocument(self::FIRST, 'root')->getAttribute(self::NEXT))); + } + + #[DataProvider('types')] + public function testAnUpsertGeneratesIdsForNestedPlaceholders(Closure $adapter, RelationshipType $type): void + { + $database = $this->database($adapter(), $type)->setReplacePlaceholders(true); + if (! $database->getAdapter()->hasFeature(Upserts::class)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $database->upsertDocument(self::FIRST, new Document([ + Document::ID => 'single', + self::NEXT => self::value($type, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'single child'])), + ])); + $database->upsertDocuments(self::FIRST, [new Document([ + Document::ID => 'bulk', + self::NEXT => self::value($type, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'bulk child'])), + ])]); + + $this->assertSame([$this->assertGeneratedId($database, self::SECOND, 'single child')], self::ids($database->getDocument(self::FIRST, 'single')->getAttribute(self::NEXT))); + $this->assertSame([$this->assertGeneratedId($database, self::SECOND, 'bulk child')], self::ids($database->getDocument(self::FIRST, 'bulk')->getAttribute(self::NEXT))); + } + + #[DataProvider('types')] + public function testAnUnheldRoleIsRefusedAtEachDepth(Closure $adapter, RelationshipType $type): void + { + $database = $this->database($adapter(), $type)->setCheckGrants(true); + $unheld = [Permission::read(Role::user('u2'))]; + + $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $database->createDocument(self::FIRST, new Document([ + Document::ID => 'depth one', + self::NEXT => self::value($type, new Document(['name' => 'child', Document::PERMISSIONS => $unheld])), + ])))); + $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $database->createDocument(self::FIRST, new Document([ + Document::ID => 'depth two', + self::NEXT => self::value($type, new Document([ + 'name' => 'child', + self::NEXT => self::value($type, new Document(['name' => 'grandchild', Document::PERMISSIONS => $unheld])), + ])), + ])))); + + $this->assertWroteNothing($database); + } + + #[DataProvider('adapters')] + public function testAnUnheldRoleOnANewToManyItemIsRefused(Closure $adapter): void + { + foreach ([RelationshipType::OneToMany, RelationshipType::ManyToMany] as $type) { + $database = $this->database($adapter(), $type); + $database->createDocument(self::FIRST, new Document([Document::ID => 'root', self::NEXT => [new Document([Document::ID => 'held', 'name' => 'held'])]])); + $database->setCheckGrants(true); + + $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $database->updateDocument(self::FIRST, 'root', new Document([ + self::NEXT => ['held', new Document(['name' => 'new', Document::PERMISSIONS => [Permission::update(Role::team('admin'))]])], + ])))); + $this->assertSame(['held'], self::names($database, self::SECOND), $type->value); + $this->assertSame(['held'], self::ids($database->getDocument(self::FIRST, 'root')->getAttribute(self::NEXT)), $type->value); + } + } + + #[DataProvider('types')] + public function testHeldUnchangedAndClearedPermissionsAreAllowed(Closure $adapter, RelationshipType $type): void + { + $database = $this->database($adapter(), $type); + $foreign = [Permission::read(Role::user('u2'))]; + $database->createDocument(self::SECOND, new Document([Document::ID => 'stored', 'name' => 'stored', Document::PERMISSIONS => [...$foreign, Permission::update(Role::any())]])); + $database->setCheckGrants(true); + + $held = [Permission::read(Role::user(self::USER)), Permission::update(Role::users()), Permission::write(Role::any())]; + $database->createDocument(self::FIRST, new Document([Document::ID => 'held', self::NEXT => self::value($type, new Document(['name' => 'held', Document::PERMISSIONS => $held]))])); + $database->createDocument(self::FIRST, new Document([Document::ID => 'unchanged', self::NEXT => self::value($type, new Document([Document::ID => 'stored', 'name' => 'unchanged', Document::PERMISSIONS => $foreign]))])); + $database->createDocument(self::FIRST, new Document([Document::ID => 'cleared', self::NEXT => self::value($type, new Document(['name' => 'cleared', Document::PERMISSIONS => []]))])); + + $this->assertSame($held, $this->related($database, 'held')->getPermissions()); + $this->assertSame('unchanged', $database->getDocument(self::SECOND, 'stored')->getAttribute('name')); + $this->assertSame([], $this->related($database, 'cleared')->getPermissions()); + } + + #[DataProvider('types')] + public function testAbsentOrNullPermissionsInheritTheParentsUnchecked(Closure $adapter, RelationshipType $type): void + { + $database = $this->database($adapter(), $type)->setCheckGrants(true); + $inherited = [Permission::read(Role::user('u2'))]; + $nulled = new Document(['name' => 'nulled']); + $nulled->setAttribute(Document::PERMISSIONS, null); + + $database->createDocument(self::FIRST, new Document([Document::ID => 'absent', Document::PERMISSIONS => $inherited, self::NEXT => self::value($type, new Document(['name' => 'absent']))])); + $database->createDocument(self::FIRST, new Document([Document::ID => 'nulled', Document::PERMISSIONS => $inherited, self::NEXT => self::value($type, $nulled)])); + + $this->assertSame($inherited, $this->related($database, 'absent')->getPermissions()); + $this->assertSame($inherited, $this->related($database, 'nulled')->getPermissions()); + } + + #[DataProvider('adapters')] + public function testMalformedPermissionsAreAStructureError(Closure $adapter): void + { + $database = $this->database($adapter(), RelationshipType::ManyToOne); + $create = fn () => $database->createDocument(self::FIRST, new Document([Document::ID => 'root', self::NEXT => new Document(['name' => 'child', Document::PERMISSIONS => ['*']])])); + + $unchecked = $this->failure($create); + $this->assertNotInstanceOf(StructureException::class, $unchecked); + $this->assertInstanceOf(DatabaseException::class, $unchecked); + + $database->setCheckGrants(true); + $checked = $this->failure($create); + $this->assertInstanceOf(StructureException::class, $checked); + $this->assertSame(self::MALFORMED, $checked->getMessage()); + $this->assertWroteNothing($database); + } + + #[DataProvider('types')] + public function testDisabledAuthorizationOrTheToggleOffAllowsAnyRole(Closure $adapter, RelationshipType $type): void + { + $database = $this->database($adapter(), $type); + $unheld = [Permission::read(Role::user('u2'))]; + $root = fn (string $id): Document => new Document([Document::ID => $id, self::NEXT => self::value($type, new Document(['name' => $id, Document::PERMISSIONS => $unheld]))]); + + $database->createDocument(self::FIRST, $root('toggle off')); + $database->setCheckGrants(true); + $this->authorization->skip(fn () => $database->createDocument(self::FIRST, $root('authorization disabled'))); + $database->withCheckGrants(false, fn () => $database->createDocument(self::FIRST, $root('scoped off'))); + + $this->assertSame($unheld, $this->related($database, 'toggle off')->getPermissions()); + $this->assertSame($unheld, $this->related($database, 'authorization disabled')->getPermissions()); + $this->assertSame($unheld, $this->related($database, 'scoped off')->getPermissions()); + $this->assertTrue($database->isCheckingGrants()); + } + + #[DataProvider('types')] + public function testAPlaceholderWithAnUnheldRoleWritesNothingAndWithAHeldRoleIsCreated(Closure $adapter, RelationshipType $type): void + { + $database = $this->database($adapter(), $type)->setReplacePlaceholders(true)->setCheckGrants(true); + $root = fn (string $role): Document => new Document([ + Document::ID => 'root', + self::NEXT => self::value($type, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'child', Document::PERMISSIONS => [Permission::read(Role::parse($role))]])), + ]); + + $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $database->createDocument(self::FIRST, $root('user:u2')))); + $this->assertWroteNothing($database); + + $database->createDocument(self::FIRST, $root('user:'.self::USER)); + $childId = $this->assertGeneratedId($database, self::SECOND, 'child'); + $this->assertSame([Permission::read(Role::user(self::USER))], $database->getDocument(self::SECOND, $childId)->getPermissions()); + } + + #[DataProvider('adapters')] + public function testTheTogglesAreScopedAndOffByDefault(Closure $adapter): void + { + $database = $this->database($adapter(), RelationshipType::ManyToOne); + + $this->assertFalse($database->isReplacingPlaceholders()); + $this->assertFalse($database->isCheckingGrants()); + $this->assertSame([true, true], $database->withReplacePlaceholders(true, fn (): array => $database->withCheckGrants(true, fn (): array => [ + $database->isReplacingPlaceholders(), + $database->isCheckingGrants(), + ]))); + $this->assertFalse($database->isReplacingPlaceholders()); + $this->assertFalse($database->isCheckingGrants()); + } + + public function testASnapshotCarriesBothTogglesIntoAChildCoroutine(): void + { + if (! \extension_loaded('swoole')) { + $this->markTestSkipped('ext-swoole is required for coroutine-scoped toggles'); + } + + $database = $this->database(new Memory(), RelationshipType::ManyToOne); + $outcomes = null; + $hookFlags = Runtime::getHookFlags(); + + try { + run(function () use ($database, &$outcomes): void { + $released = new Channel(1); + $results = new Channel(1); + + $database->withReplacePlaceholders(true, fn () => $database->withCheckGrants(true, function () use ($database, $released, $results): void { + $snapshot = $database->snapshot(); + + Coroutine::create(function () use ($database, $snapshot, $released, $results): void { + $released->pop(); + $results->push([ + 'unscoped' => $this->refusal(fn () => $database->createDocument(self::FIRST, $this->placeholderChild('unscoped', 'user:u2'))), + 'refused' => $database->withSnapshot($snapshot, fn (): ?string => $this->refusal(fn () => $database->createDocument(self::FIRST, $this->placeholderChild('refused', 'user:u2')))), + 'generated' => $database->withSnapshot($snapshot, fn (): ?string => $this->refusal(fn () => $database->createDocument(self::FIRST, $this->placeholderChild('generated', 'user:'.self::USER)))), + ]); + }); + })); + + $released->push(true); + $outcomes = $results->pop(5); + }); + } finally { + Runtime::setHookFlags($hookFlags); + } + + $this->assertSame(['unscoped' => null, 'refused' => self::UNHELD_REFUSAL, 'generated' => null], $outcomes); + $this->assertSame([Id::PLACEHOLDER], self::ids($database->getDocument(self::FIRST, 'unscoped')->getAttribute(self::NEXT)), 'Outside the snapshot both toggles are off'); + $this->assertTrue($database->getDocument(self::FIRST, 'refused')->isEmpty()); + $generated = self::ids($database->getDocument(self::FIRST, 'generated')->getAttribute(self::NEXT)); + $this->assertCount(1, $generated); + $this->assertNotSame(Id::PLACEHOLDER, $generated[0]); + } + + private function placeholderChild(string $id, string $role): Document + { + return new Document([ + Document::ID => $id, + self::NEXT => new Document([Document::ID => Id::PLACEHOLDER, 'name' => $id, Document::PERMISSIONS => [Permission::read(Role::parse($role))]]), + ]); + } + + private function database(Adapter $adapter, RelationshipType $type): Database + { + $this->authorization = new Authorization(); + $this->authorization->addRole(Role::users()->toString()); + $this->authorization->addRole(Role::user(self::USER)->toString()); + + $database = (new Database($adapter, new Cache(new None()))) + ->setAuthorization($this->authorization) + ->setDatabase('nested_preparation') + ->setNamespace('nested_preparation_'.\uniqid()); + $database->addHook(new Permissions()); + $database->addHook(new Relationships()); + $database->create(); + + foreach ([self::FIRST, self::SECOND, self::THIRD] as $collection) { + $database->createCollection(Collection::create( + $collection, + attributes: [Attribute::string('name', size: 64)], + permissions: [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any()), Permission::delete(Role::any())], + documentSecurity: false, + )); + } + + $database->createRelationship(self::FIRST, self::relationship($type, self::SECOND)); + $database->createRelationship(self::SECOND, self::relationship($type, self::THIRD)); + + return $database; + } + + private static function relationship(RelationshipType $type, string $related): Relationship + { + return match ($type) { + RelationshipType::OneToOne => Relationship::oneToOne($related, self::NEXT, onDelete: RelationshipDeleteAction::SetNull), + RelationshipType::ManyToOne => Relationship::manyToOne($related, self::NEXT, onDelete: RelationshipDeleteAction::SetNull), + RelationshipType::OneToMany => Relationship::oneToMany($related, self::NEXT, twoWay: true, twoWayKey: self::PREVIOUS, onDelete: RelationshipDeleteAction::SetNull), + RelationshipType::ManyToMany => Relationship::manyToMany($related, self::NEXT, twoWay: true, twoWayKey: self::PREVIOUS, onDelete: RelationshipDeleteAction::SetNull), + }; + } + + /** + * @return Document|string|list + */ + private static function value(RelationshipType $type, Document|string $related): Document|string|array + { + return \in_array($type, [RelationshipType::OneToMany, RelationshipType::ManyToMany], true) ? [$related] : $related; + } + + /** + * The generated ID of the only document named $name in $collection. + */ + private function assertGeneratedId(Database $database, string $collection, string $name): string + { + $matches = \array_values(\array_filter( + $this->authorization->skip(fn (): array => $database->find($collection)), + static fn (Document $document): bool => $document->getAttribute('name') === $name, + )); + + $this->assertCount(1, $matches, "One document named {$name} in {$collection}"); + $id = $matches[0]->getId(); + $this->assertNotSame('', $id); + $this->assertNotSame(Id::PLACEHOLDER, $id); + $this->assertTrue($this->authorization->skip(fn (): bool => $database->getDocument($collection, Id::PLACEHOLDER)->isEmpty()), "Nothing is stored under the placeholder in {$collection}"); + + return $id; + } + + /** + * The first related document of the parent $id. + */ + private function related(Database $database, string $id): Document + { + $related = $this->authorization->skip(fn (): Document => $database->getDocument(self::FIRST, $id))->getAttribute(self::NEXT); + $related = \is_array($related) ? $related[0] : $related; + $this->assertInstanceOf(Document::class, $related); + + return $this->authorization->skip(fn (): Document => $database->getDocument(self::SECOND, $related->getId())); + } + + private function assertWroteNothing(Database $database): void + { + foreach ([self::FIRST, self::SECOND, self::THIRD] as $collection) { + $this->assertSame([], self::names($database, $collection), "Nothing is written to {$collection}"); + } + } + + /** + * @return list + */ + private static function names(Database $database, string $collection): array + { + return $database->getAuthorization()->skip(fn (): array => \array_map( + static fn (Document $document): string => (string) $document->getAttribute('name', $document->getId()), + $database->find($collection), + )); + } + + /** + * @return list + */ + private static function ids(mixed $value): array + { + $values = \is_array($value) ? $value : ($value === null ? [] : [$value]); + + $ids = \array_map(static fn (mixed $item): string => $item instanceof Document ? $item->getId() : (string) $item, $values); + + return \array_values(\array_filter($ids, static fn (string $id): bool => $id !== '')); + } + + private function refusal(callable $write): ?string + { + try { + $write(); + } catch (AuthorizationException $error) { + return $error->getMessage(); + } + + return null; + } + + private function failure(callable $write): ?Throwable + { + try { + $write(); + } catch (Throwable $error) { + return $error; + } + + return null; + } +} From 46bcd86f74db23b20b976ab5022eccb5f67500f6 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 16:49:57 +1300 Subject: [PATCH 07/55] test(relationships): cover the nested pre-pass in the staging pre-checks authorizeCreate/authorizeUpdate must decide what the write decides once the staged writes ran: an unheld grant is refused without writing, a staged view's permissions count as current, a document staged for deletion is new so every role it carries is a new grant, and a generated placeholder ID is new and needs create where the literal document only needed update. Co-Authored-By: Claude Opus 5.5 --- .../unit/Relationships/NestedPreCheckTest.php | 257 ++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 tests/unit/Relationships/NestedPreCheckTest.php diff --git a/tests/unit/Relationships/NestedPreCheckTest.php b/tests/unit/Relationships/NestedPreCheckTest.php new file mode 100644 index 000000000..b2c21d532 --- /dev/null +++ b/tests/unit/Relationships/NestedPreCheckTest.php @@ -0,0 +1,257 @@ + + */ + public static function types(): iterable + { + $adapters = [ + 'memory' => static fn (): Adapter => new Memory(), + 'sqlite' => static fn (): Adapter => new SQLite(new PDO('sqlite::memory:')), + ]; + + foreach ($adapters as $adapterName => $adapter) { + foreach (RelationshipType::cases() as $type) { + yield "{$adapterName}, {$type->value}" => [$adapter, $type]; + } + } + } + + /** + * @param Closure(): Adapter $adapter + */ + #[DataProvider('types')] + public function testAnUnheldGrantIsRefusedAndNothingIsWritten(Closure $adapter, RelationshipType $type): void + { + $this->seed($adapter(), $type, [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], null); + $this->database->setCheckGrants(true); + $related = new Document(['name' => 'new', Document::PERMISSIONS => [Permission::read(Role::user('u2'))]]); + $old = $this->storedParent(); + + $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $this->database->authorizeCreate('parents', $this->parent($type, $related)))); + $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $this->database->authorizeUpdate('parents', $old, new Document([self::RELATED => self::value($type, $related)])))); + $this->assertSame([], $this->relatedNames()); + $this->assertNull($this->refusal(fn () => $this->database->withCheckGrants(false, fn () => $this->database->authorizeCreate('parents', $this->parent($type, $related)))), 'With the toggle off the grant is not checked'); + $this->assertNull($this->refusal(fn () => $this->authorization->skip(fn () => $this->database->authorizeCreate('parents', $this->parent($type, $related)))), 'With authorization disabled the grant is not checked'); + } + + /** + * @param Closure(): Adapter $adapter + */ + #[DataProvider('types')] + public function testAStagedViewsPermissionsCountAsCurrent(Closure $adapter, RelationshipType $type): void + { + $foreign = [Permission::read(Role::user('u2'))]; + $this->seed($adapter(), $type, [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], []); + $this->database->setCheckGrants(true); + $staged = fn (string $collection, string $id): ?Document => $collection === self::RELATED && $id === self::STORED + ? new Document([Document::ID => self::STORED, Document::COLLECTION => self::RELATED, 'name' => 'Stored', Document::PERMISSIONS => $foreign]) + : null; + $parent = $this->parent($type, new Document([Document::ID => self::STORED, 'name' => 'Changed', Document::PERMISSIONS => $foreign])); + + $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $this->database->authorizeCreate('parents', $parent)), 'Judged as stored, the role is granted anew'); + $this->assertNull($this->refusal(fn () => $this->database->authorizeCreate('parents', $parent, $staged)), 'Judged as staged, the role is sent back unchanged'); + $this->assertNull($this->refusal(fn () => $this->database->authorizeUpdate('parents', $this->storedParent(), new Document([self::RELATED => $parent->getAttribute(self::RELATED)]), $staged))); + } + + /** + * @param Closure(): Adapter $adapter + */ + #[DataProvider('types')] + public function testADocumentStagedForDeletionIsNew(Closure $adapter, RelationshipType $type): void + { + $foreign = [Permission::read(Role::user('u2'))]; + $this->seed($adapter(), $type, [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], $foreign); + $this->database->setCheckGrants(true); + $deleted = static fn (string $collection, string $id): ?Document => $collection === self::RELATED && $id === self::STORED ? new Document() : null; + $parent = $this->parent($type, new Document([Document::ID => self::STORED, 'name' => 'Recreated', Document::PERMISSIONS => $foreign])); + + $this->assertNull($this->refusal(fn () => $this->database->authorizeCreate('parents', $parent)), 'Judged as stored, the role is sent back unchanged'); + $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $this->database->authorizeCreate('parents', $parent, $deleted)), 'Judged as staged, the document is new and every role is granted anew'); + } + + /** + * @param Closure(): Adapter $adapter + */ + #[DataProvider('types')] + public function testAPlaceholderIsJudgedNewWithItsGrantsChecked(Closure $adapter, RelationshipType $type): void + { + $this->seed($adapter(), $type, [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], null); + $this->database->setReplacePlaceholders(true)->setCheckGrants(true); + $placeholder = static fn (string $role): Document => new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'new', Document::PERMISSIONS => [Permission::read(Role::parse($role))]]); + $parent = $this->parent($type, $placeholder('user:u2')); + + $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $this->database->authorizeCreate('parents', $parent))); + $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $this->database->authorizeUpdate('parents', $this->storedParent(), new Document([self::RELATED => $parent->getAttribute(self::RELATED)])))); + $this->assertNull($this->refusal(fn () => $this->database->authorizeCreate('parents', $this->parent($type, $placeholder('user:'.self::USER))))); + $this->assertSame([], $this->relatedNames()); + $this->assertSame([Id::PLACEHOLDER], self::ids($parent->getAttribute(self::RELATED)), 'The document checked is not changed'); + } + + /** + * @param Closure(): Adapter $adapter + */ + #[DataProvider('types')] + public function testAPlaceholderNeedsCreateWhereALiteralDocumentNeedsUpdate(Closure $adapter, RelationshipType $type): void + { + $this->seed($adapter(), $type, [Permission::read(Role::any()), Permission::update(Role::any())], null); + $this->authorization->skip(fn () => $this->database->createDocument(self::RELATED, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'Literal']))); + $parent = $this->parent($type, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'Changed'])); + + $this->assertNull($this->refusal(fn () => $this->database->authorizeCreate('parents', $parent)), 'The literal document is linked and changed, which needs update'); + $this->assertSame(self::NO_CREATE, $this->refusal(fn () => $this->database->withReplacePlaceholders(true, fn () => $this->database->authorizeCreate('parents', $parent))), 'A generated ID is new, which needs create'); + } + + /** + * @param list $collectionPermissions + * @param list|null $storedPermissions The related document's permissions; null stores none + */ + private function seed(Adapter $adapter, RelationshipType $type, array $collectionPermissions, ?array $storedPermissions): void + { + $this->authorization = new Authorization(); + $this->authorization->addRole(Role::users()->toString()); + $this->authorization->addRole(Role::user(self::USER)->toString()); + $this->database = (new Database($adapter, new Cache(new None()))) + ->setAuthorization($this->authorization) + ->setDatabase('nested_pre_check') + ->setNamespace('nested_pre_check_'.\uniqid()); + $this->database->addHook(new Permissions()); + $this->database->addHook(new Relationships()); + + $this->authorization->skip(function () use ($type, $collectionPermissions, $storedPermissions): void { + $this->database->create(); + $this->database->createCollection(Collection::create( + 'parents', + attributes: [Attribute::string('name', size: 64)], + permissions: [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], + )); + $this->database->createCollection(Collection::create( + self::RELATED, + attributes: [Attribute::string('name', size: 64)], + permissions: $collectionPermissions, + documentSecurity: false, + )); + $this->database->createRelationship('parents', match ($type) { + RelationshipType::OneToOne => Relationship::oneToOne(self::RELATED, self::RELATED, onDelete: RelationshipDeleteAction::SetNull), + RelationshipType::ManyToOne => Relationship::manyToOne(self::RELATED, self::RELATED, onDelete: RelationshipDeleteAction::SetNull), + RelationshipType::OneToMany => Relationship::oneToMany(self::RELATED, self::RELATED, twoWay: true, twoWayKey: 'back', onDelete: RelationshipDeleteAction::SetNull), + RelationshipType::ManyToMany => Relationship::manyToMany(self::RELATED, self::RELATED, twoWay: true, twoWayKey: 'back', onDelete: RelationshipDeleteAction::SetNull), + }); + if ($storedPermissions !== null) { + $this->database->createDocument(self::RELATED, new Document([Document::ID => self::STORED, 'name' => 'Stored', Document::PERMISSIONS => $storedPermissions])); + } + }); + } + + private function parent(RelationshipType $type, Document $related): Document + { + return new Document([ + Document::ID => 'parent', + Document::PERMISSIONS => [Permission::read(Role::any())], + self::RELATED => self::value($type, $related), + ]); + } + + private function storedParent(): Document + { + $stored = $this->authorization->skip(fn (): Document => $this->database->getDocument('parents', 'parent')); + if (! $stored->isEmpty()) { + return $stored; + } + + return $this->authorization->skip(fn (): Document => $this->database->createDocument('parents', new Document([ + Document::ID => 'parent', + Document::PERMISSIONS => [Permission::read(Role::any()), Permission::update(Role::any())], + ]))); + } + + /** + * @return Document|list + */ + private static function value(RelationshipType $type, Document $related): Document|array + { + return \in_array($type, [RelationshipType::OneToMany, RelationshipType::ManyToMany], true) ? [$related] : $related; + } + + /** + * @return list + */ + private function relatedNames(): array + { + $names = $this->authorization->skip(fn (): array => \array_map( + static fn (Document $document): string => (string) $document->getAttribute('name'), + $this->database->find(self::RELATED), + )); + + return \array_values(\array_filter($names, static fn (string $name): bool => $name !== 'Stored')); + } + + /** + * @return list + */ + private static function ids(mixed $value): array + { + $values = \is_array($value) ? $value : [$value]; + + return \array_values(\array_map(static fn (mixed $item): string => $item instanceof Document ? $item->getId() : (string) $item, $values)); + } + + private function refusal(callable $check): ?string + { + try { + $check(); + } catch (AuthorizationException $error) { + return $error->getMessage(); + } + + return null; + } +} From c9c921b2eee26b23131b3b7b063c319cbb8a0a31 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 16:52:41 +1300 Subject: [PATCH 08/55] test(relationships): cover nested preparation on every relationship adapter Placeholder IDs on create, update and upsert, a string 'unique()' kept as a reference, the grant check on create and update (held, privileged and unchecked writes allowed), and the placeholder plus unheld role case in both the write and authorizeCreate(), on every adapter with relationships, Mirror included, where the toggles are forwarded to source and destination. Co-Authored-By: Claude Opus 5.5 --- .../e2e/Adapter/Scopes/RelationshipTests.php | 2 + .../Relationships/NestedPreparationTests.php | 259 ++++++++++++++++++ 2 files changed, 261 insertions(+) create mode 100644 tests/e2e/Adapter/Scopes/Relationships/NestedPreparationTests.php diff --git a/tests/e2e/Adapter/Scopes/RelationshipTests.php b/tests/e2e/Adapter/Scopes/RelationshipTests.php index 713de7981..9ae2be121 100644 --- a/tests/e2e/Adapter/Scopes/RelationshipTests.php +++ b/tests/e2e/Adapter/Scopes/RelationshipTests.php @@ -9,6 +9,7 @@ use Tests\E2E\Adapter\Scopes\Relationships\ChildSideTests; use Tests\E2E\Adapter\Scopes\Relationships\ManyToManyTests; use Tests\E2E\Adapter\Scopes\Relationships\ManyToOneTests; +use Tests\E2E\Adapter\Scopes\Relationships\NestedPreparationTests; use Tests\E2E\Adapter\Scopes\Relationships\OneToManyTests; use Tests\E2E\Adapter\Scopes\Relationships\OneToOneTests; use Tests\E2E\Adapter\Support\EventRecorder; @@ -42,6 +43,7 @@ trait RelationshipTests use ChildSideTests; use ManyToManyTests; use ManyToOneTests; + use NestedPreparationTests; use OneToManyTests; use OneToOneTests; diff --git a/tests/e2e/Adapter/Scopes/Relationships/NestedPreparationTests.php b/tests/e2e/Adapter/Scopes/Relationships/NestedPreparationTests.php new file mode 100644 index 000000000..29e606b28 --- /dev/null +++ b/tests/e2e/Adapter/Scopes/Relationships/NestedPreparationTests.php @@ -0,0 +1,259 @@ +getDatabase(); + if (! $database->getAdapter()->hasFeature(Feature\Relationships::class)) { + $this->expectNotToPerformAssertions(); + + return; + } + + [$parents, $children, $owners] = $this->createNestedPreparationCollections($database); + + $created = $database->withReplacePlaceholders(true, fn (): Document => $database->createDocument($parents, new Document([ + Document::ID => Id::PLACEHOLDER, + 'name' => 'parent', + 'children' => [ + new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'placeholder child']), + new Document(['name' => 'empty child']), + new Document([Document::ID => 'explicit', 'name' => 'explicit child']), + ], + 'owner' => new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'owner']), + ]))); + + $this->assertNotSame(Id::PLACEHOLDER, $created->getId()); + $stored = $database->getDocument($parents, $created->getId()); + $this->assertSame('parent', $stored->getAttribute('name')); + $childIds = $this->nestedPreparationIds($stored->getAttribute('children')); + $this->assertCount(3, $childIds); + $this->assertContains('explicit', $childIds); + $this->assertNotContains(Id::PLACEHOLDER, $childIds); + $owner = $stored->getAttribute('owner'); + $this->assertInstanceOf(Document::class, $owner); + $this->assertNotSame(Id::PLACEHOLDER, $owner->getId()); + $this->assertTrue($database->getDocument($children, Id::PLACEHOLDER)->isEmpty()); + $this->assertTrue($database->getDocument($owners, Id::PLACEHOLDER)->isEmpty()); + $this->assertFalse($database->isReplacingPlaceholders()); + } + + public function testNestedPreparationGeneratesPlaceholderIdsOnUpdateAndUpsert(): void + { + $database = $this->getDatabase(); + if (! $database->getAdapter()->hasFeature(Feature\Relationships::class)) { + $this->expectNotToPerformAssertions(); + + return; + } + + [$parents, $children] = $this->createNestedPreparationCollections($database); + $database->createDocument($parents, new Document([Document::ID => 'updated', 'name' => 'updated'])); + + $database->withReplacePlaceholders(true, function () use ($database, $parents): void { + $database->updateDocument($parents, 'updated', new Document([ + 'children' => [new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'updated child'])], + ])); + + if ($database->getAdapter()->hasFeature(Feature\Upserts::class)) { + $database->upsertDocument($parents, new Document([ + Document::ID => 'upserted', + 'name' => 'upserted', + 'children' => [new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'upserted child'])], + ])); + } + }); + + $updated = $this->nestedPreparationIds($database->getDocument($parents, 'updated')->getAttribute('children')); + $this->assertCount(1, $updated); + $this->assertNotSame(Id::PLACEHOLDER, $updated[0]); + $this->assertSame('updated child', $database->getDocument($children, $updated[0])->getAttribute('name')); + $this->assertTrue($database->getDocument($children, Id::PLACEHOLDER)->isEmpty()); + + if ($database->getAdapter()->hasFeature(Feature\Upserts::class)) { + $upserted = $this->nestedPreparationIds($database->getDocument($parents, 'upserted')->getAttribute('children')); + $this->assertCount(1, $upserted); + $this->assertNotSame(Id::PLACEHOLDER, $upserted[0]); + $this->assertNotSame($updated[0], $upserted[0]); + } + } + + public function testNestedPreparationKeepsAStringPlaceholderAsAReference(): void + { + $database = $this->getDatabase(); + if (! $database->getAdapter()->hasFeature(Feature\Relationships::class)) { + $this->expectNotToPerformAssertions(); + + return; + } + + [$parents, , $owners] = $this->createNestedPreparationCollections($database); + $database->createDocument($owners, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'literal'])); + + $database->withReplacePlaceholders(true, fn (): Document => $database->createDocument($parents, new Document([ + Document::ID => 'linked', + 'name' => 'linked', + 'owner' => Id::PLACEHOLDER, + ]))); + + $owner = $database->getDocument($parents, 'linked')->getAttribute('owner'); + $this->assertInstanceOf(Document::class, $owner); + $this->assertSame(Id::PLACEHOLDER, $owner->getId()); + $this->assertSame('literal', $owner->getAttribute('name')); + } + + public function testNestedPreparationChecksGrants(): void + { + $database = $this->getDatabase(); + if (! $database->getAdapter()->hasFeature(Feature\Relationships::class)) { + $this->expectNotToPerformAssertions(); + + return; + } + + [$parents, $children] = $this->createNestedPreparationCollections($database); + $unheld = [Permission::read(Role::user('nestedPreparationUnheld'))]; + $refusal = $this->nestedPreparationRefusal(fn () => $database->withCheckGrants(true, fn (): Document => $database->createDocument($parents, $this->nestedPreparationParent('refused', $unheld)))); + $this->assertSame($this->nestedPreparationUnheldRefusal($database), $refusal); + $this->assertTrue($database->getDocument($parents, 'refused')->isEmpty()); + $this->assertSame([], $database->find($children)); + + $database->withCheckGrants(true, fn (): Document => $database->createDocument($parents, $this->nestedPreparationParent('held', [Permission::read(Role::any())]))); + $database->withCheckGrants(true, fn (): Document => $database->getAuthorization()->skip(fn (): Document => $database->createDocument($parents, $this->nestedPreparationParent('privileged', $unheld)))); + $database->createDocument($parents, $this->nestedPreparationParent('unchecked', $unheld)); + + $this->assertCount(1, $this->nestedPreparationIds($database->getDocument($parents, 'held')->getAttribute('children'))); + $this->assertFalse($database->getAuthorization()->skip(fn (): Document => $database->getDocument($parents, 'privileged'))->isEmpty()); + $this->assertFalse($database->getAuthorization()->skip(fn (): Document => $database->getDocument($parents, 'unchecked'))->isEmpty()); + + $database->createDocument($parents, new Document([Document::ID => 'update target', 'name' => 'update target'])); + $refusal = $this->nestedPreparationRefusal(fn () => $database->withCheckGrants(true, fn (): Document => $database->updateDocument($parents, 'update target', new Document([ + 'children' => [new Document(['name' => 'update refused', Document::PERMISSIONS => $unheld])], + ])))); + $this->assertSame($this->nestedPreparationUnheldRefusal($database), $refusal); + $this->assertSame([], $this->nestedPreparationIds($database->getDocument($parents, 'update target')->getAttribute('children'))); + } + + public function testNestedPreparationRefusesAnUnheldPlaceholderInWritesAndPreChecks(): void + { + $database = $this->getDatabase(); + if (! $database->getAdapter()->hasFeature(Feature\Relationships::class)) { + $this->expectNotToPerformAssertions(); + + return; + } + + [$parents, $children] = $this->createNestedPreparationCollections($database); + $parent = static fn (string $id, string $role): Document => new Document([ + Document::ID => $id, + 'name' => $id, + 'children' => [new Document([Document::ID => Id::PLACEHOLDER, 'name' => $id, Document::PERMISSIONS => [Permission::read(Role::parse($role))]])], + ]); + $prepared = static fn (callable $callback): mixed => $database->withReplacePlaceholders(true, fn (): mixed => $database->withCheckGrants(true, $callback)); + + $this->assertSame($this->nestedPreparationUnheldRefusal($database), $this->nestedPreparationRefusal(fn () => $prepared(fn () => $database->authorizeCreate($parents, $parent('checked', 'user:nestedPreparationUnheld'))))); + $this->assertSame($this->nestedPreparationUnheldRefusal($database), $this->nestedPreparationRefusal(fn () => $prepared(fn (): Document => $database->createDocument($parents, $parent('refused', 'user:nestedPreparationUnheld'))))); + $this->assertTrue($database->getDocument($parents, 'refused')->isEmpty()); + $this->assertSame([], $database->find($children)); + + $this->assertNull($this->nestedPreparationRefusal(fn () => $prepared(fn () => $database->authorizeCreate($parents, $parent('created', 'any'))))); + $prepared(fn (): Document => $database->createDocument($parents, $parent('created', 'any'))); + $created = $this->nestedPreparationIds($database->getDocument($parents, 'created')->getAttribute('children')); + $this->assertCount(1, $created); + $this->assertNotSame(Id::PLACEHOLDER, $created[0]); + $this->assertTrue($database->getDocument($children, Id::PLACEHOLDER)->isEmpty()); + } + + /** + * @return array{string, string, string} The parent, child and owner collection IDs + */ + private function createNestedPreparationCollections(Database $database): array + { + $parents = Id::unique(); + $children = Id::unique(); + $owners = Id::unique(); + $permissions = [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ]; + + foreach ([$parents, $children, $owners] as $collection) { + $database->createCollection(Collection::create( + id: $collection, + attributes: [Attribute::string(key: 'name', size: 64)], + permissions: $permissions, + documentSecurity: false, + )); + } + + $database->createRelationship($parents, Relationship::oneToMany( + relatedCollection: $children, + key: 'children', + twoWay: true, + twoWayKey: 'parent', + onDelete: RelationshipDeleteAction::SetNull, + )); + $database->createRelationship($parents, Relationship::manyToOne( + relatedCollection: $owners, + key: 'owner', + onDelete: RelationshipDeleteAction::SetNull, + )); + + return [$parents, $children, $owners]; + } + + /** + * @param list $permissions The permissions of the parent's new child + */ + private function nestedPreparationParent(string $id, array $permissions): Document + { + return new Document([ + Document::ID => $id, + 'name' => $id, + 'children' => [new Document(['name' => $id, Document::PERMISSIONS => $permissions])], + ]); + } + + /** + * @return list + */ + private function nestedPreparationIds(mixed $value): array + { + $values = \is_array($value) ? $value : ($value === null ? [] : [$value]); + + return \array_values(\array_map(static fn (mixed $item): string => $item instanceof Document ? $item->getId() : (\is_string($item) ? $item : ''), $values)); + } + + private function nestedPreparationUnheldRefusal(Database $database): string + { + return 'Permissions must be one of: ('.\implode(', ', $database->getAuthorization()->getRoles()).')'; + } + + private function nestedPreparationRefusal(callable $write): ?string + { + try { + $write(); + } catch (AuthorizationException $error) { + return $error->getMessage(); + } + + return null; + } +} From c0d3a338345f7ba0725bc8d40a8732ff3c56faae Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 16:53:50 +1300 Subject: [PATCH 09/55] refactor(relationships): narrow the nested pre-pass array conversion Related documents given as documents are cloned before preparing, so the conversion helper only ever sees arrays. Co-Authored-By: Claude Opus 5.5 --- src/Database/Hook/Relationships.php | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/Database/Hook/Relationships.php b/src/Database/Hook/Relationships.php index ae1325e7d..f94fc4c95 100644 --- a/src/Database/Hook/Relationships.php +++ b/src/Database/Hook/Relationships.php @@ -3302,11 +3302,10 @@ private function authorizeGrants(Document $collection, Document $document, bool throw new StructureException($validator->getDescription()); } - /** @var list $permissions */ $current = $new ? [] : (Permission::aggregate($this->grantedView($collection, $document->getId(), $resolve)->getPermissions()) ?? []); $authorization = $this->database->getAuthorization(); - foreach (\array_diff(Permission::aggregate($permissions) ?? [], $current) as $granted) { + foreach (\array_diff(Permission::aggregate($document->getPermissions()) ?? [], $current) as $granted) { $permission = Permission::parse($granted); $role = new Role($permission->getRole(), $permission->getIdentifier(), $permission->getDimension()); if (! $authorization->hasRole($role->toString())) { @@ -3336,16 +3335,12 @@ private static function isDocumentValue(mixed $value): bool } /** - * @param Document|array $value + * @param array $value * * @throws RelationshipException */ - private static function toDocument(Document|array $value): Document + private static function toDocument(array $value): Document { - if ($value instanceof Document) { - return $value; - } - try { return new Document($value); } catch (StructureException $error) { From e5cbc14df13f14ecc7199beb03a3f196ff9f1594 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 16:53:54 +1300 Subject: [PATCH 10/55] test(relationships): type the nested preparation test helpers PHPStan runs at max level over tests: the adapter factories need their closure signature and stored values are narrowed instead of cast from mixed. Co-Authored-By: Claude Opus 5.5 --- .../unit/Relationships/NestedPreCheckTest.php | 4 +- .../Relationships/NestedPreparationTest.php | 63 +++++++++++++++++-- 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/tests/unit/Relationships/NestedPreCheckTest.php b/tests/unit/Relationships/NestedPreCheckTest.php index b2c21d532..0efb6dee8 100644 --- a/tests/unit/Relationships/NestedPreCheckTest.php +++ b/tests/unit/Relationships/NestedPreCheckTest.php @@ -227,7 +227,7 @@ private static function value(RelationshipType $type, Document $related): Docume private function relatedNames(): array { $names = $this->authorization->skip(fn (): array => \array_map( - static fn (Document $document): string => (string) $document->getAttribute('name'), + static fn (Document $document): string => \is_string($name = $document->getAttribute('name')) ? $name : '', $this->database->find(self::RELATED), )); @@ -241,7 +241,7 @@ private static function ids(mixed $value): array { $values = \is_array($value) ? $value : [$value]; - return \array_values(\array_map(static fn (mixed $item): string => $item instanceof Document ? $item->getId() : (string) $item, $values)); + return \array_values(\array_map(static fn (mixed $item): string => $item instanceof Document ? $item->getId() : (\is_string($item) ? $item : ''), $values)); } private function refusal(callable $check): ?string diff --git a/tests/unit/Relationships/NestedPreparationTest.php b/tests/unit/Relationships/NestedPreparationTest.php index d4685c489..ae0874677 100644 --- a/tests/unit/Relationships/NestedPreparationTest.php +++ b/tests/unit/Relationships/NestedPreparationTest.php @@ -82,6 +82,9 @@ public static function adapters(): iterable yield 'sqlite' => [static fn (): Adapter => new SQLite(new PDO('sqlite::memory:'))]; } + /** + * @param Closure(): Adapter $adapter + */ #[DataProvider('types')] public function testPlaceholdersGetGeneratedIdsAtEveryDepth(Closure $adapter, RelationshipType $type): void { @@ -102,6 +105,9 @@ public function testPlaceholdersGetGeneratedIdsAtEveryDepth(Closure $adapter, Re $this->assertSame(Id::PLACEHOLDER, $grandchild->getId(), 'The caller\'s nested document is not changed'); } + /** + * @param Closure(): Adapter $adapter + */ #[DataProvider('adapters')] public function testAPlaceholderThreeLevelsDownGetsAGeneratedId(Closure $adapter): void { @@ -125,6 +131,9 @@ public function testAPlaceholderThreeLevelsDownGetsAGeneratedId(Closure $adapter $this->assertGeneratedId($database, self::THIRD, 'grandchild'); } + /** + * @param Closure(): Adapter $adapter + */ #[DataProvider('types')] public function testAnEmptyIdIsGeneratedAndAnExplicitIdIsKept(Closure $adapter, RelationshipType $type): void { @@ -143,6 +152,9 @@ public function testAnEmptyIdIsGeneratedAndAnExplicitIdIsKept(Closure $adapter, $this->assertGeneratedId($database, self::THIRD, 'grandchild'); } + /** + * @param Closure(): Adapter $adapter + */ #[DataProvider('types')] public function testAStringPlaceholderStaysAReference(Closure $adapter, RelationshipType $type): void { @@ -156,6 +168,9 @@ public function testAStringPlaceholderStaysAReference(Closure $adapter, Relation $this->assertSame(['literal'], self::names($database, self::SECOND), 'Linking by the string creates nothing'); } + /** + * @param Closure(): Adapter $adapter + */ #[DataProvider('types')] public function testAStringPlaceholderWithoutALiteralDocumentLinksNothing(Closure $adapter, RelationshipType $type): void { @@ -167,6 +182,9 @@ public function testAStringPlaceholderWithoutALiteralDocumentLinksNothing(Closur $this->assertSame([], self::names($database, self::SECOND)); } + /** + * @param Closure(): Adapter $adapter + */ #[DataProvider('types')] public function testEveryWriteGeneratesNewIds(Closure $adapter, RelationshipType $type): void { @@ -186,6 +204,9 @@ public function testEveryWriteGeneratesNewIds(Closure $adapter, RelationshipType $this->assertSame([$two], self::ids($database->getDocument(self::FIRST, 'two')->getAttribute(self::NEXT))); } + /** + * @param Closure(): Adapter $adapter + */ #[DataProvider('adapters')] public function testTopLevelPlaceholdersAreGenerated(Closure $adapter): void { @@ -204,6 +225,9 @@ public function testTopLevelPlaceholdersAreGenerated(Closure $adapter): void $this->assertGeneratedId($database, self::SECOND, 'batched'); } + /** + * @param Closure(): Adapter $adapter + */ #[DataProvider('types')] public function testAnUpdateGeneratesIdsForNestedPlaceholders(Closure $adapter, RelationshipType $type): void { @@ -224,6 +248,9 @@ public function testAnUpdateGeneratesIdsForNestedPlaceholders(Closure $adapter, $this->assertSame([$childId], self::ids($database->getDocument(self::FIRST, 'root')->getAttribute(self::NEXT))); } + /** + * @param Closure(): Adapter $adapter + */ #[DataProvider('types')] public function testAnUpsertGeneratesIdsForNestedPlaceholders(Closure $adapter, RelationshipType $type): void { @@ -247,6 +274,9 @@ public function testAnUpsertGeneratesIdsForNestedPlaceholders(Closure $adapter, $this->assertSame([$this->assertGeneratedId($database, self::SECOND, 'bulk child')], self::ids($database->getDocument(self::FIRST, 'bulk')->getAttribute(self::NEXT))); } + /** + * @param Closure(): Adapter $adapter + */ #[DataProvider('types')] public function testAnUnheldRoleIsRefusedAtEachDepth(Closure $adapter, RelationshipType $type): void { @@ -268,6 +298,9 @@ public function testAnUnheldRoleIsRefusedAtEachDepth(Closure $adapter, Relations $this->assertWroteNothing($database); } + /** + * @param Closure(): Adapter $adapter + */ #[DataProvider('adapters')] public function testAnUnheldRoleOnANewToManyItemIsRefused(Closure $adapter): void { @@ -284,6 +317,9 @@ public function testAnUnheldRoleOnANewToManyItemIsRefused(Closure $adapter): voi } } + /** + * @param Closure(): Adapter $adapter + */ #[DataProvider('types')] public function testHeldUnchangedAndClearedPermissionsAreAllowed(Closure $adapter, RelationshipType $type): void { @@ -302,6 +338,9 @@ public function testHeldUnchangedAndClearedPermissionsAreAllowed(Closure $adapte $this->assertSame([], $this->related($database, 'cleared')->getPermissions()); } + /** + * @param Closure(): Adapter $adapter + */ #[DataProvider('types')] public function testAbsentOrNullPermissionsInheritTheParentsUnchecked(Closure $adapter, RelationshipType $type): void { @@ -317,6 +356,9 @@ public function testAbsentOrNullPermissionsInheritTheParentsUnchecked(Closure $a $this->assertSame($inherited, $this->related($database, 'nulled')->getPermissions()); } + /** + * @param Closure(): Adapter $adapter + */ #[DataProvider('adapters')] public function testMalformedPermissionsAreAStructureError(Closure $adapter): void { @@ -334,6 +376,9 @@ public function testMalformedPermissionsAreAStructureError(Closure $adapter): vo $this->assertWroteNothing($database); } + /** + * @param Closure(): Adapter $adapter + */ #[DataProvider('types')] public function testDisabledAuthorizationOrTheToggleOffAllowsAnyRole(Closure $adapter, RelationshipType $type): void { @@ -352,6 +397,9 @@ public function testDisabledAuthorizationOrTheToggleOffAllowsAnyRole(Closure $ad $this->assertTrue($database->isCheckingGrants()); } + /** + * @param Closure(): Adapter $adapter + */ #[DataProvider('types')] public function testAPlaceholderWithAnUnheldRoleWritesNothingAndWithAHeldRoleIsCreated(Closure $adapter, RelationshipType $type): void { @@ -369,6 +417,9 @@ public function testAPlaceholderWithAnUnheldRoleWritesNothingAndWithAHeldRoleIsC $this->assertSame([Permission::read(Role::user(self::USER))], $database->getDocument(self::SECOND, $childId)->getPermissions()); } + /** + * @param Closure(): Adapter $adapter + */ #[DataProvider('adapters')] public function testTheTogglesAreScopedAndOffByDefault(Closure $adapter): void { @@ -525,10 +576,14 @@ private function assertWroteNothing(Database $database): void */ private static function names(Database $database, string $collection): array { - return $database->getAuthorization()->skip(fn (): array => \array_map( - static fn (Document $document): string => (string) $document->getAttribute('name', $document->getId()), + return $database->getAuthorization()->skip(fn (): array => \array_values(\array_map( + static function (Document $document): string { + $name = $document->getAttribute('name'); + + return \is_string($name) ? $name : $document->getId(); + }, $database->find($collection), - )); + ))); } /** @@ -538,7 +593,7 @@ private static function ids(mixed $value): array { $values = \is_array($value) ? $value : ($value === null ? [] : [$value]); - $ids = \array_map(static fn (mixed $item): string => $item instanceof Document ? $item->getId() : (string) $item, $values); + $ids = \array_map(static fn (mixed $item): string => $item instanceof Document ? $item->getId() : (\is_string($item) ? $item : ''), $values); return \array_values(\array_filter($ids, static fn (string $id): bool => $id !== '')); } From 3f94ca7c9576de130aadcb16f7d147376271de37 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:02:16 +1300 Subject: [PATCH 11/55] test(adapter): prove hasFeature answers spatial support behind every proxy appwrite wrapped Feature\Spatial in a Support class because it believed hasFeature() behind a pool was unreliable. 8.0.3 already answers it correctly, so this pins that: every adapter, a Pool, a ReadWritePool, a Database profile and a Mirror (whose destination answers the opposite way) all give the held adapter's answer, the spatial axis order capability plays no part, and every other Feature and Capability a consumer reads through a pool (Relationships, Schemaless, DefinedAttributes, Operators, Joins, ...) matches the adapter as well. Consumers can ask hasFeature(Feature\Spatial::class) directly. Co-Authored-By: Claude Opus 5.5 --- tests/unit/Adapter/SpatialFeatureTest.php | 281 ++++++++++++++++++++++ 1 file changed, 281 insertions(+) create mode 100644 tests/unit/Adapter/SpatialFeatureTest.php diff --git a/tests/unit/Adapter/SpatialFeatureTest.php b/tests/unit/Adapter/SpatialFeatureTest.php new file mode 100644 index 000000000..a912be172 --- /dev/null +++ b/tests/unit/Adapter/SpatialFeatureTest.php @@ -0,0 +1,281 @@ + + */ + public static function adapters(): iterable + { + yield 'MariaDB' => [new MariaDB(new \stdClass()), true]; + yield 'MySQL' => [new MySQL(new \stdClass()), true]; + yield 'Postgres' => [new Postgres(new \stdClass()), true]; + yield 'SQLite' => [new SQLite(new PDO('sqlite::memory:')), false]; + yield 'MongoDB' => [self::mongo(), false]; + yield 'Memory' => [new Memory(), false]; + yield 'Redis' => [self::redis(), false]; + yield 'spatial adapter without the spatial axis order capability' => [ + new class (new \stdClass()) extends MySQL { + /** + * @return array + */ + #[\Override] + public function capabilities(): array + { + return \array_values(\array_filter( + parent::capabilities(), + static fn (Capability $capability): bool => $capability !== Capability::SpatialAxisOrder, + )); + } + }, + true, + ]; + yield 'adapter with the spatial axis order capability but no spatial types' => [ + new class () extends Memory { + /** + * @return array + */ + #[\Override] + public function capabilities(): array + { + return [...parent::capabilities(), Capability::SpatialAxisOrder]; + } + }, + false, + ]; + } + + /** + * @return iterable + */ + public static function everyAdapter(): iterable + { + foreach (self::adapters() as $name => [$adapter]) { + yield $name => [$adapter]; + } + } + + /** + * @return iterable + */ + public static function nonSpatialAdapters(): iterable + { + yield 'SQLite' => [new SQLite(new PDO('sqlite::memory:'))]; + yield 'Memory' => [new Memory()]; + } + + #[DataProvider('adapters')] + public function testTheAdapterSaysWhetherItStoresGeometries(Adapter $adapter, bool $spatial): void + { + $this->assertSame($spatial, $adapter->hasFeature(Feature\Spatial::class)); + } + + #[DataProvider('adapters')] + public function testAPoolAnswersForTheAdapterItHolds(Adapter $adapter, bool $spatial): void + { + $pool = self::pool($adapter); + + $this->assertSame($spatial, $pool->hasFeature(Feature\Spatial::class)); + $this->assertSame($spatial, $pool->hasFeature(Feature\Spatial::class), 'A remembered answer must match the first'); + $this->assertSame($spatial, self::pool($adapter)->hasFeature(Feature\Spatial::class), 'Another handle over a fresh pool must answer alike'); + } + + #[DataProvider('adapters')] + public function testAReadWritePoolAnswersForTheAdapterItHolds(Adapter $adapter, bool $spatial): void + { + $pool = self::readWritePool($adapter)->setSticky(false); + + $this->assertSame($spatial, $pool->hasFeature(Feature\Spatial::class)); + $this->assertSame($spatial, $pool->hasFeature(Feature\Spatial::class), 'A remembered answer must match the first'); + } + + #[DataProvider('adapters')] + public function testADatabaseOverAPoolProfilesTheAdapterItHolds(Adapter $adapter, bool $spatial): void + { + $database = self::database(self::pool($adapter)); + + $this->assertSame($spatial, $database->getAdapter()->hasFeature(Feature\Spatial::class)); + $this->assertSame($spatial, $database->profile()->hasFeature(Feature\Spatial::class)); + } + + #[DataProvider('adapters')] + public function testAMirrorAnswersForItsSourceWhateverItsDestinationStores(Adapter $adapter, bool $spatial): void + { + $destination = $spatial ? new Memory() : new MariaDB(new \stdClass()); + $this->assertSame(! $spatial, $destination->hasFeature(Feature\Spatial::class)); + + $mirrors = [ + 'direct source' => new Mirror(self::database($adapter), self::database($destination)), + 'pooled source' => new Mirror(self::database(self::pool($adapter)), self::database(self::pool($destination))), + ]; + + foreach ($mirrors as $name => $mirror) { + $this->assertSame($spatial, $mirror->getAdapter()->hasFeature(Feature\Spatial::class), $name); + $this->assertSame($spatial, $mirror->profile()->hasFeature(Feature\Spatial::class), $name); + } + } + + /** + * Every Feature and Capability a consumer checks on a pooled adapter, such as Relationships, Schemaless, + * DefinedAttributes, Operators, Joins or IndexSpatialNull, is the answer of the adapter the pool holds. + * Timeouts is left out: a pool holds timeouts itself whichever adapter it holds. + */ + #[DataProvider('everyAdapter')] + public function testEveryFeatureAndCapabilityThroughAPoolIsTheAdapters(Adapter $adapter): void + { + $proxies = [ + 'pool' => self::pool($adapter), + 'read/write pool' => self::readWritePool($adapter)->setSticky(false), + ]; + + foreach ($proxies as $name => $proxy) { + $profile = self::database($proxy)->profile(); + + foreach (self::features() as $feature) { + $this->assertSame($adapter->hasFeature($feature), $proxy->hasFeature($feature), "{$name}: {$feature}"); + $this->assertSame($adapter->hasFeature($feature), $profile->hasFeature($feature), "{$name} profile: {$feature}"); + } + + foreach (Capability::cases() as $capability) { + $this->assertSame($adapter->supports($capability), $proxy->supports($capability), "{$name}: {$capability->name}"); + $this->assertSame($adapter->supports($capability), $profile->supports($capability), "{$name} profile: {$capability->name}"); + } + } + } + + #[DataProvider('nonSpatialAdapters')] + public function testAPooledAdapterWithoutSpatialTypesRefusesAPointAttribute(Adapter $adapter): void + { + $database = self::database(self::pool($adapter)); + $database->setDatabase('spatial_feature')->setNamespace('spatial_feature_'.\uniqid()); + $database->create(); + $database->createCollection(Collection::create(id: 'places')); + + $this->assertFalse($database->getAdapter()->hasFeature(Feature\Spatial::class)); + + $this->expectException(DatabaseException::class); + $this->expectExceptionMessage('Spatial attributes are not supported'); + $database->createAttribute('places', Attribute::point(key: 'location', required: true)); + } + + public function testAMirrorOverASourceWithoutSpatialTypesRefusesAPointAttribute(): void + { + $source = self::database(new Memory()); + $mirror = new Mirror($source, self::database(new MariaDB(new \stdClass()))); + $mirror->setDatabase('spatial_feature')->setNamespace('spatial_feature_'.\uniqid()); + $source->create(); + $source->createCollection(Collection::create(id: 'places')); + + $this->assertFalse($mirror->getAdapter()->hasFeature(Feature\Spatial::class)); + + $this->expectException(DatabaseException::class); + $this->expectExceptionMessage('Spatial attributes are not supported'); + $mirror->createAttribute('places', Attribute::point(key: 'location', required: true)); + } + + /** + * @return list + */ + private static function features(): array + { + $directory = \dirname((string) (new ReflectionClass(Adapter::class))->getFileName()).'/Adapter/Feature'; + $features = []; + + foreach (\glob($directory.'/*.php') ?: [] as $file) { + /** @var class-string $feature */ + $feature = 'Utopia\\Database\\Adapter\\Feature\\'.\basename($file, '.php'); + if ($feature !== Feature\Timeouts::class) { + $features[] = $feature; + } + } + + return $features; + } + + private static function mongo(): Mongo + { + return new class () extends Mongo { + public function __construct() + { + } + }; + } + + private static function redis(): Redis + { + return new class () extends Redis { + public function __construct() + { + } + }; + } + + /** + * @return UtopiaPool + */ + private static function connections(Adapter $adapter): UtopiaPool + { + /** @var UtopiaPool $connections */ + $connections = new UtopiaPool(new Stack(), 'spatial_feature', 1, static fn (): Adapter => $adapter, 1.0); + + return $connections; + } + + private static function pool(Adapter $adapter): Pool + { + $pool = new Pool(self::connections($adapter)); + $pool->setAuthorization(new Authorization()); + + return $pool; + } + + private static function readWritePool(Adapter $adapter): ReadWritePool + { + $pool = new ReadWritePool(self::connections($adapter), self::connections($adapter)); + $pool->setAuthorization(new Authorization()); + + return $pool; + } + + private static function database(Adapter $adapter): Database + { + $database = new Database($adapter, new Cache(new None())); + $database->setAuthorization(new Authorization()); + + return $database; + } +} From 35ca85b9f993dadf8f30ff6f390b45f33a29a7dc Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:02:16 +1300 Subject: [PATCH 12/55] test(e2e): tie the spatial feature to creating a point attribute On every adapter job (including Pool, Mirror, shared tables and schemaless), the adapter's Feature\Spatial answer and the profile's must match whether a point attribute can actually be created, so a consumer gating spatial types on hasFeature() never offers one the engine refuses or hides one it stores. Co-Authored-By: Claude Opus 5.5 --- tests/e2e/Adapter/Base.php | 2 ++ tests/e2e/Adapter/Scopes/FeatureTests.php | 42 +++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 tests/e2e/Adapter/Scopes/FeatureTests.php diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 429c3a9a4..01e9e7d47 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -10,6 +10,7 @@ use Tests\E2E\Adapter\Scopes\CustomDocumentTypeTests; use Tests\E2E\Adapter\Scopes\DatabaseTests; use Tests\E2E\Adapter\Scopes\DocumentTests; +use Tests\E2E\Adapter\Scopes\FeatureTests; use Tests\E2E\Adapter\Scopes\GeneralTests; use Tests\E2E\Adapter\Scopes\IndexTests; use Tests\E2E\Adapter\Scopes\JoinComboTests; @@ -49,6 +50,7 @@ abstract class Base extends TestCase use CustomDocumentTypeTests; use DatabaseTests; use DocumentTests; + use FeatureTests; use GeneralTests; use IndexTests; use JoinComboTests; diff --git a/tests/e2e/Adapter/Scopes/FeatureTests.php b/tests/e2e/Adapter/Scopes/FeatureTests.php new file mode 100644 index 000000000..528849301 --- /dev/null +++ b/tests/e2e/Adapter/Scopes/FeatureTests.php @@ -0,0 +1,42 @@ +getDatabase(); + $spatial = $database->getAdapter()->hasFeature(Feature\Spatial::class); + + $this->assertSame($spatial, $database->profile()->hasFeature(Feature\Spatial::class)); + + $collection = 'feature_spatial'; + $database->createCollection(Collection::create(id: $collection)); + + try { + if (! $spatial) { + $this->expectException(DatabaseException::class); + $this->expectExceptionMessage('Spatial attributes are not supported'); + } + + $database->createAttribute($collection, Attribute::point(key: 'location', required: true)); + + $types = []; + foreach ($database->getCollection($collection)->attributes() as $attribute) { + $types[$attribute->key] = $attribute->type; + } + $this->assertSame(ColumnType::Point, $types['location'] ?? null); + } finally { + $database->deleteCollection($collection); + } + } +} From 56c359b1001d24c53960cf97e327d3b3ba98a732 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:08:02 +1300 Subject: [PATCH 13/55] feat(documents): read documents and aggregate rows through read() find() refuses aggregate and groupBy queries by design (DEC-40), so a caller serving one list endpoint had to inspect the queries itself and pick find() or aggregate(), which appwrite did app-side in Listing. read() makes that routing a library API: aggregate or groupBy queries return aggregate()'s rows wrapped as documents, everything else returns find(). Aggregates read under the read permission only, so another permission is refused rather than silently read as read. find() keeps refusing aggregates. ParsedQuery::isAggregation() names the check fetch() and read() share. Co-Authored-By: Claude Opus 5.5 --- src/Database/ParsedQuery.php | 8 ++ src/Database/Trait/Documents.php | 34 +++++- tests/e2e/Adapter/Base.php | 2 + tests/e2e/Adapter/Scopes/ReadTests.php | 159 +++++++++++++++++++++++++ tests/unit/Documents/ReadTest.php | 152 +++++++++++++++++++++++ 5 files changed, 353 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/Adapter/Scopes/ReadTests.php create mode 100644 tests/unit/Documents/ReadTest.php diff --git a/src/Database/ParsedQuery.php b/src/Database/ParsedQuery.php index 3c4fd1ba7..d20b98e0e 100644 --- a/src/Database/ParsedQuery.php +++ b/src/Database/ParsedQuery.php @@ -43,4 +43,12 @@ public function __construct( public array $orderTypes = [], ) { } + + /** + * Whether the queries read one row per group, through an aggregate or a groupBy, instead of documents. + */ + public function isAggregation(): bool + { + return $this->aggregations !== [] || $this->groupBy !== []; + } } diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 79c08a931..d25631e54 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -3870,6 +3870,36 @@ public function find(string $collection, array $queries = [], PermissionType $fo return $this->fetch($this->silent(fn () => $this->getCollection($collection)), $queries, $forPermission, false); } + /** + * Read a list the way its queries ask: the documents find() reads, or, when the queries aggregate or group, the + * rows aggregate() reads, each as a document. + * + * @param array $queries + * @param PermissionType $forPermission The permission documents are read under; aggregate rows read under the read permission only + * @return list + * + * @throws DatabaseException + * @throws QueryException When aggregate or groupBy queries are read under another permission than read + * @throws TimeoutException + */ + public function read(string $collection, array $queries = [], PermissionType $forPermission = PermissionType::Read): array + { + $this->checkQueryTypes($queries); + + if (! Query::groupByType($queries)->isAggregation()) { + return \array_values($this->find($collection, $queries, $forPermission)); + } + + if ($forPermission !== PermissionType::Read) { + throw new QueryException('Aggregate and groupBy queries read under the read permission only, not '.$forPermission->value); + } + + return \array_map( + static fn (array $row): Document => new Document($row), + $this->aggregate($collection, $queries), + ); + } + /** * @param array $queries * @param bool $aggregate Whether the queries must aggregate (aggregate()) or must not (find()) @@ -3921,7 +3951,7 @@ private function fetch(Collection $collection, array $queries, PermissionType $f $cursor = $parsed->cursor; $cursorDirection = $parsed->cursorDirection ?? CursorDirection::After; - $isAggregation = ! empty($aggregations) || ! empty($groupByAttrs); + $isAggregation = $parsed->isAggregation(); if ($isAggregation !== $aggregate) { throw new QueryException($aggregate @@ -4753,7 +4783,7 @@ private function nextPageCheck(string $collection, array $queries): Closure $parsed = Query::groupByType($queries); $joins = $parsed->joins; $distinct = $parsed->distinct; - if (($joins === [] && ! $distinct) || $parsed->aggregations !== [] || $parsed->groupBy !== []) { + if (($joins === [] && ! $distinct) || $parsed->isAggregation()) { return static function (Document $cursor): void { }; } diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 429c3a9a4..d374bb8a2 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -19,6 +19,7 @@ use Tests\E2E\Adapter\Scopes\ObjectAttributeTests; use Tests\E2E\Adapter\Scopes\OperatorTests; use Tests\E2E\Adapter\Scopes\PermissionTests; +use Tests\E2E\Adapter\Scopes\ReadTests; use Tests\E2E\Adapter\Scopes\RelationshipTests; use Tests\E2E\Adapter\Scopes\SchemalessTests; use Tests\E2E\Adapter\Scopes\SchemaReconciliationTests; @@ -58,6 +59,7 @@ abstract class Base extends TestCase use ObjectAttributeTests; use OperatorTests; use PermissionTests; + use ReadTests; use RelationshipTests; use SchemaReconciliationTests; use SchemalessTests; diff --git a/tests/e2e/Adapter/Scopes/ReadTests.php b/tests/e2e/Adapter/Scopes/ReadTests.php new file mode 100644 index 000000000..bbaffb952 --- /dev/null +++ b/tests/e2e/Adapter/Scopes/ReadTests.php @@ -0,0 +1,159 @@ +getDatabase(); + $this->seedReadPosts($database); + + try { + $queries = [Query::greaterThan('views', 1), Query::orderDesc('views')]; + $read = $database->read(self::READ_POSTS, $queries); + + $this->assertSame(['third', 'second'], \array_map(static fn (Document $document): string => $document->getId(), $read)); + $this->assertSame( + \array_map(static fn (Document $document): array => $document->getArrayCopy(), $database->find(self::READ_POSTS, $queries)), + \array_map(static fn (Document $document): array => $document->getArrayCopy(), $read), + ); + } finally { + $this->deleteReadPosts($database); + } + } + + public function testReadUnderAnotherPermissionReadsTheDocumentsHeldUnderIt(): void + { + $database = $this->getDatabase(); + $this->seedReadPosts($database); + + try { + $this->assertSame( + ['first', 'third'], + \array_map(static fn (Document $document): string => $document->getId(), $database->read(self::READ_POSTS, [Query::orderAsc('views')], PermissionType::Update)), + ); + } finally { + $this->deleteReadPosts($database); + } + } + + public function testReadReturnsAggregateRowsAsDocuments(): void + { + $database = $this->getDatabase(); + if (! $database->getAdapter()->supports(Capability::Aggregations)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $this->seedReadPosts($database); + + try { + $totals = $database->read(self::READ_POSTS, [Query::count('*', 'posts'), Query::sum('views', 'views')]); + $this->assertCount(1, $totals); + $this->assertInstanceOf(Document::class, $totals[0]); + $this->assertSame([3, 6], [(int) $totals[0]->getAttribute('posts'), (int) $totals[0]->getAttribute('views')]); + + $groups = $database->read(self::READ_POSTS, [Query::groupBy(['category']), Query::count('*', 'posts'), Query::orderAsc('category')]); + $this->assertSame( + [['news', 2], ['sport', 1]], + \array_map(static fn (Document $row): array => [$row->getAttribute('category'), (int) $row->getAttribute('posts')], $groups), + ); + } finally { + $this->deleteReadPosts($database); + } + } + + public function testReadRefusesAnAggregationUnderAnotherPermissionThanRead(): void + { + $database = $this->getDatabase(); + $this->seedReadPosts($database); + + try { + $database->read(self::READ_POSTS, [Query::count()], PermissionType::Update); + $this->fail('An aggregation under the update permission must be refused'); + } catch (QueryException $exception) { + $this->assertStringContainsString('read under the read permission only', $exception->getMessage()); + } finally { + $this->deleteReadPosts($database); + } + } + + public function testFindStillRefusesAnAggregation(): void + { + $database = $this->getDatabase(); + $this->seedReadPosts($database); + + try { + $database->find(self::READ_POSTS, [Query::count()]); + $this->fail('find() must refuse an aggregate query'); + } catch (QueryException $exception) { + $this->assertInstanceOf(QueryException::class, $exception); + if ($database->getAdapter()->supports(Capability::Aggregations)) { + $this->assertSame('find() does not run aggregate or groupBy queries: use aggregate()', $exception->getMessage()); + } + } finally { + $this->deleteReadPosts($database); + } + } + + /** + * Posts first (news, 1 view), second (news, 2 views) and third (sport, 3 views), all readable, first and third + * updatable by any caller. + */ + private function seedReadPosts(Database $database): void + { + $this->deleteReadPosts($database); + + $database->createCollection(Collection::create( + id: self::READ_POSTS, + attributes: [ + Attribute::string(key: 'category', size: 16), + Attribute::integer(key: 'views'), + ], + permissions: [Permission::create(Role::any())], + documentSecurity: true, + )); + + foreach ([ + 'first' => ['news', 1, true], + 'second' => ['news', 2, false], + 'third' => ['sport', 3, true], + ] as $id => [$category, $views, $updatable]) { + $database->createDocument(self::READ_POSTS, new Document([ + '$id' => $id, + '$permissions' => [ + Permission::read(Role::any()), + ...($updatable ? [Permission::update(Role::any())] : []), + ], + 'category' => $category, + 'views' => $views, + ])); + } + } + + private function deleteReadPosts(Database $database): void + { + if ($database->collectionExists(self::READ_POSTS)) { + $database->deleteCollection(self::READ_POSTS); + } + } +} diff --git a/tests/unit/Documents/ReadTest.php b/tests/unit/Documents/ReadTest.php new file mode 100644 index 000000000..6290c5454 --- /dev/null +++ b/tests/unit/Documents/ReadTest.php @@ -0,0 +1,152 @@ +addRole(Role::user('editor')->toString()); + + $this->database = new Database(new SQLite(new PDO('sqlite::memory:')), new Cache(new None())); + $this->database + ->setAuthorization($authorization) + ->setDatabase('read') + ->setNamespace('read_'.\uniqid()); + $this->database->addHook(new Permissions()); + $this->database->create(); + + $this->database->createCollection(Collection::create( + id: self::COLLECTION, + attributes: [ + Attribute::string(key: 'category', size: 16), + Attribute::integer(key: 'views'), + ], + permissions: [Permission::create(Role::any())], + documentSecurity: true, + )); + + foreach ([ + 'first' => ['news', 1, true], + 'second' => ['news', 2, false], + 'third' => ['sport', 3, true], + ] as $id => [$category, $views, $editable]) { + $this->database->createDocument(self::COLLECTION, new Document([ + Document::ID => $id, + Document::PERMISSIONS => [ + Permission::read(Role::any()), + ...($editable ? [Permission::update(Role::user('editor'))] : []), + ], + 'category' => $category, + 'views' => $views, + ])); + } + } + + public function testDocumentQueriesReadWhatFindReads(): void + { + $queries = [Query::greaterThan('views', 1), Query::orderDesc('views')]; + + $read = $this->database->read(self::COLLECTION, $queries); + + $this->assertSame(['third', 'second'], \array_map(static fn (Document $document): string => $document->getId(), $read)); + $this->assertSame( + \array_map(static fn (Document $document): array => $document->getArrayCopy(), $this->database->find(self::COLLECTION, $queries)), + \array_map(static fn (Document $document): array => $document->getArrayCopy(), $read), + ); + } + + public function testNoQueriesReadEveryDocument(): void + { + $this->assertSame(['first', 'second', 'third'], \array_map(static fn (Document $document): string => $document->getId(), $this->database->read(self::COLLECTION))); + } + + public function testAggregateQueriesReadTheAggregateRowsAsDocuments(): void + { + $queries = [Query::count('*', 'total'), Query::sum('views')]; + + $read = $this->database->read(self::COLLECTION, $queries); + + $this->assertContainsOnlyInstancesOf(Document::class, $read); + $this->assertSame([['total' => 3, 'sum_views' => 6]], \array_map(static fn (Document $row): array => $row->getArrayCopy(), $read)); + $this->assertSame($this->database->aggregate(self::COLLECTION, $queries), \array_map(static fn (Document $row): array => $row->getArrayCopy(), $read)); + } + + public function testGroupByQueriesReadOneDocumentPerGroup(): void + { + $read = $this->database->read(self::COLLECTION, [Query::groupBy(['category']), Query::count('*', 'posts'), Query::orderAsc('category')]); + + $this->assertSame( + [['posts' => 2, 'category' => 'news'], ['posts' => 1, 'category' => 'sport']], + \array_map(static fn (Document $row): array => $row->getArrayCopy(), $read), + ); + } + + public function testThePermissionIsPassedToFind(): void + { + $this->assertSame( + ['first', 'third'], + \array_map(static fn (Document $document): string => $document->getId(), $this->database->read(self::COLLECTION, [], PermissionType::Update)), + ); + } + + public function testAnAggregationUnderAnotherPermissionThanReadIsRefused(): void + { + $this->expectException(QueryException::class); + $this->expectExceptionMessage('Aggregate and groupBy queries read under the read permission only'); + + $this->database->read(self::COLLECTION, [Query::count()], PermissionType::Update); + } + + public function testGroupedQueriesTellAnAggregationFromADocumentList(): void + { + $this->assertFalse(Query::groupByType([Query::equal('category', ['news']), Query::orderAsc('views'), Query::limit(2)])->isAggregation()); + $this->assertFalse(Query::groupByType([])->isAggregation()); + $this->assertTrue(Query::groupByType([Query::count()])->isAggregation()); + $this->assertTrue(Query::groupByType([Query::groupBy(['category'])])->isAggregation(), 'a groupBy reads one row per group without an aggregate'); + } + + public function testFindStillRefusesAggregateQueries(): void + { + $this->expectException(QueryException::class); + $this->expectExceptionMessage('find() does not run aggregate or groupBy queries: use aggregate()'); + + $this->database->find(self::COLLECTION, [Query::count()]); + } + + public function testAQueryOfAnotherTypeIsRefused(): void + { + $this->expectException(QueryException::class); + $this->expectExceptionMessage('Invalid query type: "string"'); + + /** @phpstan-ignore argument.type */ + $this->database->read(self::COLLECTION, ['count()']); + } +} From e88ca1a1322f959fab4b68b8260f666522f7eefb Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:08:02 +1300 Subject: [PATCH 14/55] feat(cursors): resolve a list cursor from a document id Clients page by the ID of the last row they saw, but find() needs the row's order values, and over a join those come from a joined row, not the document. appwrite resolved them app-side in CursorLookup, duplicating the library's join-order rules. resolveCursor() moves that into the library and reuses qualifyJoinedOrders(), addTieBreaks() and joinMatchesAtMostOneRow(), so the cursor carries exactly the values fetch() orders by and cannot disagree with it. The document is read with authorization skipped so a list can page past a document its caller cannot read; joined values come from the rows the caller can read, filtered by the join conditions, the list's filters on that alias and the join's own filters, and ordered so that a page after the document starts behind its last paired row and a page before it ends ahead of its first. Joins on a one-to-many relationship attribute go through rewriteRelationshipJoins() first, like every other read. Co-Authored-By: Claude Opus 5.5 --- src/Database/Database.php | 1 + src/Database/Trait/Cursors.php | 323 +++++++++++++ tests/e2e/Adapter/Base.php | 2 + tests/e2e/Adapter/Scopes/CursorTests.php | 247 ++++++++++ tests/unit/Documents/ResolveCursorTest.php | 530 +++++++++++++++++++++ 5 files changed, 1103 insertions(+) create mode 100644 src/Database/Trait/Cursors.php create mode 100644 tests/e2e/Adapter/Scopes/CursorTests.php create mode 100644 tests/unit/Documents/ResolveCursorTest.php diff --git a/src/Database/Database.php b/src/Database/Database.php index c45e3eb79..aaf5633a1 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -53,6 +53,7 @@ class Database { use Trait\Attributes; use Trait\Collections; + use Trait\Cursors; use Trait\Databases; use Trait\Documents; use Trait\Indexes; diff --git a/src/Database/Trait/Cursors.php b/src/Database/Trait/Cursors.php new file mode 100644 index 000000000..e27ec523c --- /dev/null +++ b/src/Database/Trait/Cursors.php @@ -0,0 +1,323 @@ + '>', + '<=' => '>=', + '>' => '<', + '>=' => '<=', + ]; + + /** + * The cursor a list pages from when it names a document by its ID: the document's values of the list's orders, + * which cursorAfter() or cursorBefore() in the same list reads as the row that ID stands for. + * + * The document is read with authorization skipped, so a list can page past a document its caller cannot read; the + * cursor carries only `$id`, `$sequence`, `$collection` and the values the list orders by. An order on a joined + * attribute takes its value from the joined rows the caller can read that the list pairs with the document: a page + * after the document starts behind the last of them in the list's order, a page before it ends ahead of the first, + * and without such a row the value is null, as it is in the list. + * + * @param array $queries The list's queries; its cursor queries are left out + * @return Document Empty when the collection holds no document with the ID + * + * @throws NotFoundException When the collection does not exist + * @throws QueryException When the queries are invalid + * @throws DatabaseException + */ + public function resolveCursor(string $collection, string $id, CursorDirection $direction, array $queries = []): Document + { + $definition = $this->silent(fn (): Collection => $this->getCollection($collection)); + + $this->checkQueryTypes($queries); + $queries = $this->rewriteRelationshipJoins($definition, \array_values(\array_filter( + $queries, + static fn (Query $query): bool => $query->getMethod() !== Method::CursorAfter && $query->getMethod() !== Method::CursorBefore, + ))); + + $joinedById = $this->resolveJoinedCollections($queries); + if ($this->validation()->get()) { + $this->validateDocumentsQueries($definition, $queries, $joinedById); + } + + $parsed = Query::groupByType($queries); + $joins = $parsed->joins; + + if ($joins !== [] && ! $this->adapter->supports(Capability::Joins)) { + throw new QueryException('Join queries are not supported by this adapter'); + } + + $this->assertJoinCount($joins); + + $document = $this->authorization->skip(fn (): Document => $this->skipRelationships(fn (): Document => $this->readDocument($definition, $id))); + if ($document->isEmpty()) { + return new Document(); + } + + $cursor = new Document([ + Document::ID => $document->getId(), + Document::SEQUENCE => $document->getSequence(), + Document::COLLECTION => $definition->getId(), + ]); + + if (\in_array(OrderDirection::Random, $parsed->orderTypes, true)) { + return $cursor; + } + + $joinedCollections = $this->joinedCollectionsByAlias($joins, $joinedById); + $orderAttributes = $parsed->orderAttributes; + $orderTypes = $parsed->orderTypes; + + if ($joinedCollections !== []) { + [$orderAttributes] = $this->qualifyJoinedOrders($definition, $orderAttributes, $cursor, $joinedCollections); + } + + if (! $parsed->distinct) { + [$orderAttributes, $orderTypes] = $this->addTieBreaks($orderAttributes, $orderTypes, $parsed->filters, $joins, $joinedCollections, true, $parsed->selections); + } + + $joinedOrders = []; + foreach ($orderAttributes as $index => $attribute) { + [$alias, $name] = $this->orderOwner($definition, $attribute, $joinedCollections); + if ($alias === Query::DEFAULT_ALIAS) { + $cursor->setAttribute($attribute, $document->getAttribute($attribute)); + + continue; + } + + $descending = ($orderTypes[$index] ?? OrderDirection::Asc) === OrderDirection::Desc; + $joinedOrders[$alias][] = $descending !== ($direction === CursorDirection::After) + ? Query::orderDesc($name) + : Query::orderAsc($name); + } + + if ($joinedOrders === []) { + return $cursor; + } + + $joinsByAlias = []; + foreach ($joins as $join) { + $joinsByAlias[$join->getAlias()] = $join; + } + + $rows = []; + foreach ($this->pairedJoins($joinsByAlias, $joinedOrders) as $alias => $join) { + $rows[$alias] = $this->pairedRow($join, $alias, $document, $rows, $parsed->filters, $joinedOrders[$alias] ?? []); + } + + foreach ($joinedOrders as $alias => $orders) { + $row = $rows[$alias]; + foreach ($orders as $order) { + $name = $order->getAttribute(); + $cursor->setAttribute($alias.'.'.$name, $row->isEmpty() ? null : $row->getAttribute($name)); + } + } + + return $cursor; + } + + /** + * The join alias an order reads and the attribute it reads there, or the main alias and the order as given. + * + * @param array $joinedCollections + * @return array{0: string, 1: string} + */ + private function orderOwner(Collection $collection, string $attribute, array $joinedCollections): array + { + $dot = \strpos($attribute, '.'); + if ($dot === false || ! isset($joinedCollections[\substr($attribute, 0, $dot)]) || $this->declaresSumAttribute($collection, $attribute)) { + return [Query::DEFAULT_ALIAS, $attribute]; + } + + return [\substr($attribute, 0, $dot), \substr($attribute, $dot + 1)]; + } + + /** + * The ordered joins and every join their conditions reach through, in the order the list declares them. + * + * @param array $joins + * @param array> $orders + * @return array + */ + private function pairedJoins(array $joins, array $orders): array + { + $required = \array_fill_keys(\array_keys($orders), true); + + foreach (\array_reverse($joins, true) as $alias => $join) { + if (! isset($required[$alias])) { + continue; + } + + foreach (self::joinConditions($join) as [$left, , $right]) { + foreach ([self::columnOwner($left, Query::DEFAULT_ALIAS)[0], self::columnOwner($right, $alias)[0]] as $referenced) { + if ($referenced !== $alias && isset($joins[$referenced])) { + $required[$referenced] = true; + } + } + } + } + + return \array_intersect_key($joins, $required); + } + + /** + * The joined row the list pairs with the document first in the given orders, read with the caller's permissions: + * the join's conditions become filters on the joined collection with the values of the document or of the rows + * resolved for earlier joins, next to the list's filters that constrain only this join and the join's own filters. + * + * @param array $rows The rows resolved for earlier joins, by alias + * @param array $filters + * @param list $orders + */ + private function pairedRow(Query $join, string $alias, Document $document, array $rows, array $filters, array $orders): Document + { + $conditions = []; + foreach (self::joinConditions($join) as [$left, $operator, $right]) { + [$leftAlias, $leftAttribute] = self::columnOwner($left, Query::DEFAULT_ALIAS); + [$rightAlias, $rightAttribute] = self::columnOwner($right, $alias); + + if ($rightAlias === $alias && $leftAlias !== $alias) { + $condition = self::comparison($rightAttribute, self::MIRRORED_OPERATORS[$operator] ?? $operator, self::pairedValue($leftAlias, $leftAttribute, $document, $rows)); + } elseif ($leftAlias === $alias && $rightAlias !== $alias) { + $condition = self::comparison($leftAttribute, $operator, self::pairedValue($rightAlias, $rightAttribute, $document, $rows)); + } else { + continue; + } + + if ($condition === null) { + return new Document(); + } + + $conditions[] = $condition; + } + + foreach ([...$filters, ...$join->getJoinOnQueries()] as $filter) { + $condition = self::aliasFilter($filter, $alias); + if ($condition !== null) { + $conditions[] = $condition; + } + } + + $found = $this->skipRelationships(fn (): array => $this->find($join->getAttribute(), [...$conditions, ...$orders, Query::limit(1)])); + + return $found[0] ?? new Document(); + } + + /** + * Each column comparison of a join as its left column, operator and right column. + * + * @return list + */ + private static function joinConditions(Query $join): array + { + $conditions = []; + foreach ($join->getJoinOnQueries() as $on) { + if ($on->getMethod() !== Method::On) { + continue; + } + + [$left, $operator, $right] = \array_pad($on->getValues(), 3, null); + if (\is_string($left) && \is_string($operator) && \is_string($right)) { + $conditions[] = [$left, $operator, $right]; + } + } + + return $conditions; + } + + /** + * The alias a join column belongs to and its attribute, the given alias for a bare column. + * + * @return array{0: string, 1: string} + */ + private static function columnOwner(string $column, string $alias): array + { + $dot = \strpos($column, '.'); + if ($dot === false) { + return [$alias, $column]; + } + + return [\substr($column, 0, $dot), \substr($column, $dot + 1)]; + } + + /** + * @param array $rows + */ + private static function pairedValue(string $alias, string $attribute, Document $document, array $rows): mixed + { + $value = $alias === Query::DEFAULT_ALIAS + ? $document->getAttribute($attribute) + : ($rows[$alias] ?? new Document())->getAttribute($attribute); + + return $value instanceof Document ? $value->getId() : $value; + } + + /** + * A join condition as a filter on the joined column, or null when no row can satisfy it. + * + * @throws QueryException + */ + private static function comparison(string $attribute, string $operator, mixed $value): ?Query + { + if (! \is_scalar($value)) { + return null; + } + + return match ($operator) { + '=' => Query::equal($attribute, [$value]), + '!=', '<>' => Query::notEqual($attribute, $value), + '<' => Query::lessThan($attribute, $value), + '<=' => Query::lessThanEqual($attribute, $value), + '>' => Query::greaterThan($attribute, $value), + '>=' => Query::greaterThanEqual($attribute, $value), + default => throw new QueryException('Invalid join operator: '.$operator), + }; + } + + /** + * A filter on the joined collection when the given filter constrains only this alias, else null. + */ + private static function aliasFilter(BaseQuery $filter, string $alias): ?Query + { + $method = $filter->getMethod(); + + if ($method === Method::And || $method === Method::Or) { + $children = []; + foreach ($filter->getValues() as $child) { + $condition = $child instanceof BaseQuery ? self::aliasFilter($child, $alias) : null; + if ($condition === null) { + return null; + } + $children[] = $condition; + } + + return $children === [] ? null : new Query($method, values: $children); + } + + if (! $method->isFilter() && ! $method->isSpatial() && ! $method->isJson() && $method !== Method::ElemMatch && $method !== Method::ContainsAll) { + return null; + } + + [$owner, $attribute] = self::columnOwner($filter->getAttribute(), Query::DEFAULT_ALIAS); + if ($owner !== $alias) { + return null; + } + + return clone new Query($method, $attribute, $filter->getValues()); + } +} diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index d374bb8a2..730e1fd28 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -7,6 +7,7 @@ use Tests\E2E\Adapter\Scopes\AggregationTests; use Tests\E2E\Adapter\Scopes\AttributeTests; use Tests\E2E\Adapter\Scopes\CollectionTests; +use Tests\E2E\Adapter\Scopes\CursorTests; use Tests\E2E\Adapter\Scopes\CustomDocumentTypeTests; use Tests\E2E\Adapter\Scopes\DatabaseTests; use Tests\E2E\Adapter\Scopes\DocumentTests; @@ -47,6 +48,7 @@ abstract class Base extends TestCase use AggregationTests; use AttributeTests; use CollectionTests; + use CursorTests; use CustomDocumentTypeTests; use DatabaseTests; use DocumentTests; diff --git a/tests/e2e/Adapter/Scopes/CursorTests.php b/tests/e2e/Adapter/Scopes/CursorTests.php new file mode 100644 index 000000000..57e4bcdda --- /dev/null +++ b/tests/e2e/Adapter/Scopes/CursorTests.php @@ -0,0 +1,247 @@ +getDatabase(); + $this->seedCursorCollections($database); + + try { + $this->createCursorDocument($database, self::CURSOR_CUSTOMERS, 'alice', ['name' => 'Alice'], readable: true); + $this->createCursorDocument($database, self::CURSOR_CUSTOMERS, 'bob', ['name' => 'Bob'], readable: false); + $this->createCursorDocument($database, self::CURSOR_CUSTOMERS, 'carol', ['name' => 'Carol'], readable: true); + + $queries = [Query::orderAsc('name')]; + $after = $database->resolveCursor(self::CURSOR_CUSTOMERS, 'bob', CursorDirection::After, $queries); + $before = $database->resolveCursor(self::CURSOR_CUSTOMERS, 'bob', CursorDirection::Before, $queries); + + $this->assertSame('bob', $after->getId()); + $this->assertSame('Bob', $after->getAttribute('name')); + $this->assertSame(['carol'], $this->cursorIds($database->find(self::CURSOR_CUSTOMERS, [...$queries, Query::cursorAfter($after)])), 'a caller who cannot read the cursor document still pages past it'); + $this->assertSame(['alice'], $this->cursorIds($database->find(self::CURSOR_CUSTOMERS, [...$queries, Query::cursorBefore($before)]))); + $this->assertTrue($database->resolveCursor(self::CURSOR_CUSTOMERS, 'nobody', CursorDirection::After, $queries)->isEmpty()); + } finally { + $this->deleteCursorCollections($database); + } + } + + public function testResolveCursorTakesAJoinedOrderFromTheRowsTheCallerCanRead(): void + { + $database = $this->getDatabase(); + if (! $database->getAdapter()->supports(Capability::Joins)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $this->seedCursorCollections($database); + + try { + $this->createCursorDocument($database, self::CURSOR_CUSTOMERS, 'alice', ['name' => 'Alice'], readable: true); + $this->createCursorDocument($database, self::CURSOR_CUSTOMERS, 'bob', ['name' => 'Bob'], readable: false); + $this->createCursorDocument($database, self::CURSOR_CUSTOMERS, 'carol', ['name' => 'Carol'], readable: true); + foreach ([ + ['a-first', 'alice', 10, true, 'paid'], + ['a-hidden', 'alice', 35, false, 'paid'], + ['a-open', 'alice', 60, true, 'open'], + ['a-second', 'alice', 25, true, 'paid'], + ['b-hidden', 'bob', 30, false, 'paid'], + ['b-only', 'bob', 20, true, 'paid'], + ['c-only', 'carol', 15, true, 'paid'], + ] as [$id, $customer, $amount, $readable, $status]) { + $this->createCursorDocument($database, self::CURSOR_ORDERS, $id, ['customerId' => $customer, 'amount' => $amount, 'status' => $status], $readable); + } + + $queries = [ + Query::join(self::CURSOR_ORDERS, 'ord', [Query::on('$id', 'customerId')]), + Query::equal('ord.status', ['paid']), + Query::orderAsc('ord.amount'), + ]; + + $this->assertSame( + [['alice', 'a-first'], ['carol', 'c-only'], ['alice', 'a-second']], + $this->cursorRows($database->find(self::CURSOR_CUSTOMERS, [...$queries, Query::limit(100)])), + ); + $this->assertSame(25, (int) $database->resolveCursor(self::CURSOR_CUSTOMERS, 'alice', CursorDirection::After, $queries)->getAttribute('ord.amount'), 'after alice the list resumes behind its last paid order the caller can read'); + $this->assertSame(10, (int) $database->resolveCursor(self::CURSOR_CUSTOMERS, 'alice', CursorDirection::Before, $queries)->getAttribute('ord.amount')); + $this->assertSame(20, (int) $database->resolveCursor(self::CURSOR_CUSTOMERS, 'bob', CursorDirection::After, $queries)->getAttribute('ord.amount'), 'the joined values of a document the caller cannot read come from the rows it can read'); + + $this->assertSame([], $this->cursorPage($database, $queries, 'alice', CursorDirection::After)); + $this->assertSame([], $this->cursorPage($database, $queries, 'alice', CursorDirection::Before)); + $this->assertSame([['alice', 'a-second']], $this->cursorPage($database, $queries, 'carol', CursorDirection::After)); + $this->assertSame([['alice', 'a-first']], $this->cursorPage($database, $queries, 'carol', CursorDirection::Before)); + $this->assertSame([['alice', 'a-second']], $this->cursorPage($database, $queries, 'bob', CursorDirection::After), 'bob is not listed, and the list resumes behind the order of 20 the caller can read'); + $this->assertSame([['alice', 'a-first'], ['carol', 'c-only']], $this->cursorPage($database, $queries, 'bob', CursorDirection::Before)); + } finally { + $this->deleteCursorCollections($database); + } + } + + public function testResolveCursorOverAJoinOnAOneToManyRelationshipPagesEveryRowOnce(): void + { + $database = $this->getDatabase(); + if (! $database->getAdapter()->supports(Capability::Joins) || ! $database->getAdapter()->hasFeature(Feature\Relationships::class)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $this->seedCursorCollections($database); + + try { + $database->createRelationship(self::CURSOR_CUSTOMERS, Relationship::oneToMany(relatedCollection: self::CURSOR_ORDERS, key: 'orders', twoWayKey: 'customer')); + + foreach (['alice' => 'Alice', 'bob' => 'Bob', 'carol' => 'Carol'] as $id => $name) { + $this->createCursorDocument($database, self::CURSOR_CUSTOMERS, $id, ['name' => $name], readable: true); + } + foreach ([ + ['a-first', 'alice', 10, true], + ['a-second', 'alice', 20, true], + ['b-first', 'bob', 30, true], + ['b-second', 'bob', 40, true], + ['c-hidden', 'carol', 5, false], + ['c-only', 'carol', 50, true], + ] as [$id, $customer, $amount, $readable]) { + $this->createCursorDocument($database, self::CURSOR_ORDERS, $id, ['customer' => $customer, 'amount' => $amount, 'status' => 'paid'], $readable); + } + + $queries = [Query::join(self::CURSOR_ORDERS, 'ord', [Query::on('orders', '$id')]), Query::orderAsc('ord.amount')]; + $all = $this->cursorRows($database->find(self::CURSOR_CUSTOMERS, [...$queries, Query::limit(100)])); + $this->assertSame([['alice', 'a-first'], ['alice', 'a-second'], ['bob', 'b-first'], ['bob', 'b-second'], ['carol', 'c-only']], $all); + + $forward = []; + $cursor = null; + for ($page = 0; $page <= \count($all); $page++) { + $rows = $this->cursorRows($database->find(self::CURSOR_CUSTOMERS, [ + ...$queries, + Query::limit(2), + ...($cursor === null ? [] : [Query::cursorAfter($database->resolveCursor(self::CURSOR_CUSTOMERS, $cursor, CursorDirection::After, $queries))]), + ])); + if ($rows === []) { + break; + } + \array_push($forward, ...$rows); + $cursor = $rows[\count($rows) - 1][0]; + } + $this->assertSame($all, $forward, 'paging forward by the ID of each page\'s last row returns every row once'); + + $backward = [['carol', 'c-only']]; + $cursor = 'carol'; + for ($page = 0; $page <= \count($all); $page++) { + $rows = $this->cursorRows($database->find(self::CURSOR_CUSTOMERS, [ + ...$queries, + Query::limit(2), + Query::cursorBefore($database->resolveCursor(self::CURSOR_CUSTOMERS, $cursor, CursorDirection::Before, $queries)), + ])); + if ($rows === []) { + break; + } + $backward = [...$rows, ...$backward]; + $cursor = $rows[0][0]; + } + $this->assertSame($all, $backward, 'paging backward by the ID of each page\'s first row returns every row once'); + } finally { + $this->deleteCursorCollections($database); + } + } + + private function seedCursorCollections(Database $database): void + { + $this->deleteCursorCollections($database); + + $permissions = [Permission::create(Role::any())]; + $database->createCollection(Collection::create( + id: self::CURSOR_CUSTOMERS, + attributes: [Attribute::string(key: 'name', size: 16)], + permissions: $permissions, + documentSecurity: true, + )); + $database->createCollection(Collection::create( + id: self::CURSOR_ORDERS, + attributes: [ + Attribute::string(key: 'customerId', size: 16), + Attribute::string(key: 'status', size: 16), + Attribute::integer(key: 'amount'), + ], + permissions: $permissions, + documentSecurity: true, + )); + } + + private function deleteCursorCollections(Database $database): void + { + foreach ([self::CURSOR_CUSTOMERS, self::CURSOR_ORDERS] as $collection) { + if ($database->collectionExists($collection)) { + $database->deleteCollection($collection); + } + } + } + + /** + * @param array $attributes + */ + private function createCursorDocument(Database $database, string $collection, string $id, array $attributes, bool $readable): void + { + $database->getAuthorization()->skip(fn (): Document => $database->createDocument($collection, new Document([ + '$id' => $id, + '$permissions' => [Permission::read($readable ? Role::any() : Role::user('cursor-other'))], + ...$attributes, + ]))); + } + + /** + * @param list $queries + * @return list + */ + private function cursorPage(Database $database, array $queries, string $id, CursorDirection $direction): array + { + $cursor = $database->resolveCursor(self::CURSOR_CUSTOMERS, $id, $direction, $queries); + + return $this->cursorRows($database->find(self::CURSOR_CUSTOMERS, [ + ...$queries, + $direction === CursorDirection::After ? Query::cursorAfter($cursor) : Query::cursorBefore($cursor), + ])); + } + + /** + * @param array $documents + * @return list + */ + private function cursorIds(array $documents): array + { + return \array_values(\array_map(static fn (Document $document): string => $document->getId(), $documents)); + } + + /** + * @param array $documents + * @return list + */ + private function cursorRows(array $documents): array + { + return \array_values(\array_map(static fn (Document $document): array => [$document->getId(), $document->getAttribute('ord.$id')], $documents)); + } +} diff --git a/tests/unit/Documents/ResolveCursorTest.php b/tests/unit/Documents/ResolveCursorTest.php new file mode 100644 index 000000000..92d05378a --- /dev/null +++ b/tests/unit/Documents/ResolveCursorTest.php @@ -0,0 +1,530 @@ +addRole(Role::user('reader')->toString()); + + $this->database = new Database(new SQLite(new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION])), new Cache(new None())); + $this->database + ->setAuthorization($authorization) + ->setDatabase('resolve_cursor') + ->setNamespace('resolve_cursor_'.\uniqid()); + $this->database->addHook(new Relationships()); + $this->database->addHook(new Permissions()); + $this->database->create(); + + $this->createCollection(self::CUSTOMERS, [Attribute::string(key: 'name', size: 64)]); + $this->createCollection(self::ORDERS, [ + Attribute::string(key: 'customerId', size: 64), + Attribute::string(key: 'productId', size: 64), + Attribute::string(key: 'status', size: 16), + Attribute::integer(key: 'amount'), + Attribute::datetime(key: 'placedAt'), + ]); + $this->createCollection(self::PRODUCTS, [Attribute::string(key: 'name', size: 64), Attribute::integer(key: 'price')]); + } + + public function testCursorWithoutJoinPagesPastADocumentTheCallerCannotRead(): void + { + $this->customer('alice', readable: true); + $this->customer('bob', readable: false); + $this->customer('carol', readable: true); + + $queries = [Query::orderAsc('name')]; + + $this->assertSame(['carol'], $this->ids($this->page($queries, 'bob', CursorDirection::After)), 'a caller who cannot read the cursor document still pages past it'); + $this->assertSame(['alice'], $this->ids($this->page($queries, 'bob', CursorDirection::Before))); + $this->assertWalksEveryRowOnce($queries, ['alice', 'carol']); + } + + public function testCursorWithJoinPagesByAnOrderTheSelectLeavesOut(): void + { + $this->customer('alice', readable: true); + $this->customer('bob', readable: true); + $this->customer('carol', readable: true); + $this->order('a-only', 'alice', 10, readable: true); + $this->order('b-only', 'bob', 15, readable: true); + $this->order('c-only', 'carol', 20, readable: true); + + $queries = [$this->ordersJoin(), Query::select(['$id']), Query::orderAsc('name')]; + + $this->assertSame(['carol'], $this->ids($this->page($queries, 'bob', CursorDirection::After)), 'the select shapes the listed rows, never the cursor the page starts after'); + $this->assertCursorSplitsTheList($queries); + } + + public function testCursorWithJoinOrderTakesItsValueFromAJoinedRowTheCallerCanRead(): void + { + $this->customer('alice', readable: true); + $this->order('a-hidden', 'alice', 35, readable: false); + $this->order('b-visible', 'alice', 10, readable: true); + + $queries = [$this->ordersJoin(), Query::orderAsc('ord.amount')]; + $cursor = $this->database->resolveCursor(self::CUSTOMERS, 'alice', CursorDirection::After, $queries); + + $this->assertSame('alice', $cursor->getId()); + $this->assertSame(10, $cursor->getAttribute('ord.amount'), 'the page boundary comes from the order the caller can read, not the one the list hides'); + $this->assertCursorSplitsTheList($queries); + } + + public function testCursorWithJoinOrderResolvesJoinedValuesOfADocumentTheCallerCannotRead(): void + { + $this->customer('alice', readable: true); + $this->customer('bob', readable: false); + $this->order('a-visible', 'alice', 25, readable: true); + $this->order('c-hidden', 'bob', 30, readable: false); + $this->order('d-visible', 'bob', 20, readable: true); + + $queries = [$this->ordersJoin(), Query::orderAsc('ord.amount')]; + $cursor = $this->database->resolveCursor(self::CUSTOMERS, 'bob', CursorDirection::After, $queries); + + $this->assertSame('bob', $cursor->getId(), 'a caller who cannot read the cursor document still pages past it'); + $this->assertSame(20, $cursor->getAttribute('ord.amount')); + $this->assertSame([['alice', 'a-visible']], $this->rows($this->page($queries, 'bob', CursorDirection::After))); + } + + public function testCursorWithJoinOrderSkipsJoinedRowsTheListFiltersOut(): void + { + $this->customer('alice', readable: true); + $this->customer('bob', readable: true); + $this->order('a-open', 'alice', 50, readable: true, status: 'open'); + $this->order('b-paid', 'alice', 10, readable: true, status: 'paid'); + $this->order('c-paid', 'bob', 30, readable: true, status: 'paid'); + + $queries = [$this->ordersJoin(), Query::equal('ord.status', ['paid']), Query::orderAsc('ord.amount')]; + $cursor = $this->database->resolveCursor(self::CUSTOMERS, 'alice', CursorDirection::After, $queries); + + $this->assertSame(10, $cursor->getAttribute('ord.amount'), 'the list only pairs the customer with its paid order'); + $this->assertCursorSplitsTheList($queries); + } + + public function testCursorWithJoinOrderSkipsJoinedRowsTheJoinConditionFiltersOut(): void + { + $this->customer('alice', readable: true); + $this->customer('bob', readable: true); + $this->order('a-open', 'alice', 50, readable: true, status: 'open'); + $this->order('b-paid', 'alice', 10, readable: true, status: 'paid'); + $this->order('c-paid', 'bob', 30, readable: true, status: 'paid'); + + $queries = [ + Query::join(self::ORDERS, 'ord', [Query::on('$id', 'customerId'), Query::equal('ord.status', ['paid'])]), + Query::orderAsc('ord.amount'), + ]; + + $this->assertSame(10, $this->database->resolveCursor(self::CUSTOMERS, 'alice', CursorDirection::After, $queries)->getAttribute('ord.amount')); + $this->assertCursorSplitsTheList($queries); + } + + public function testCursorAfterADocumentWithSeveralJoinedRowsStartsBehindItsLastRow(): void + { + $this->customer('alice', readable: true); + $this->customer('bob', readable: true); + $this->order('a-first', 'alice', 10, readable: true); + $this->order('b-second', 'alice', 25, readable: true); + $this->order('c-only', 'bob', 15, readable: true); + + $ascending = [$this->ordersJoin(), Query::orderAsc('ord.amount')]; + $descending = [$this->ordersJoin(), Query::orderDesc('ord.amount')]; + + $this->assertSame(25, $this->database->resolveCursor(self::CUSTOMERS, 'alice', CursorDirection::After, $ascending)->getAttribute('ord.amount')); + $this->assertSame(10, $this->database->resolveCursor(self::CUSTOMERS, 'alice', CursorDirection::After, $descending)->getAttribute('ord.amount')); + $this->assertSame(10, $this->database->resolveCursor(self::CUSTOMERS, 'alice', CursorDirection::Before, $ascending)->getAttribute('ord.amount'), 'the page before a document ends ahead of its first row'); + $this->assertSame([], $this->rows($this->page($ascending, 'alice', CursorDirection::After))); + $this->assertSame([], $this->rows($this->page($ascending, 'alice', CursorDirection::Before))); + $this->assertCursorSplitsTheList($ascending); + $this->assertCursorSplitsTheList($descending); + } + + public function testCursorWithJoinOrderPassesJoinedDatetimesForTheLibraryToEncode(): void + { + $this->customer('alice', readable: true); + $this->customer('bob', readable: true); + $this->order('a-visible', 'alice', 10, readable: true, placedAt: '2024-05-01T10:00:00.000+00:00'); + $this->order('b-visible', 'bob', 10, readable: true, placedAt: '2024-06-01T10:00:00.000+00:00'); + + $queries = [$this->ordersJoin(), Query::orderAsc('ord.placedAt')]; + + $this->assertSame('2024-05-01T10:00:00.000+00:00', $this->database->resolveCursor(self::CUSTOMERS, 'alice', CursorDirection::After, $queries)->getAttribute('ord.placedAt')); + $this->assertSame([['bob', 'b-visible']], $this->rows($this->page($queries, 'alice', CursorDirection::After)), 'the joined datetime is encoded as the cursor document\'s own attributes are'); + } + + public function testCursorWithJoinOrderFollowsAChainOfJoins(): void + { + $this->customer('alice', readable: true); + $this->customer('bob', readable: true); + $this->order('a-visible', 'alice', 10, readable: true, productId: 'lamp'); + $this->order('b-visible', 'bob', 10, readable: true, productId: 'desk'); + $this->product('lamp', 'Lamp'); + $this->product('desk', 'Desk'); + + $queries = [ + $this->ordersJoin(), + Query::join(self::PRODUCTS, 'prod', [Query::on('ord.productId', '$id')]), + Query::orderAsc('prod.name'), + ]; + + $this->assertSame('Lamp', $this->database->resolveCursor(self::CUSTOMERS, 'alice', CursorDirection::After, $queries)->getAttribute('prod.name')); + $this->assertSame(['bob'], $this->ids($this->page($queries, 'alice', CursorDirection::Before))); + $this->assertCursorSplitsTheList($queries); + } + + public function testCursorComparesTheJoinedColumnAsTheJoinConditionDoes(): void + { + $this->customer('alice', readable: true); + $this->customer('bob', readable: true); + $this->order('a-only', 'alice', 20, readable: true); + $this->order('b-only', 'bob', 40, readable: true); + $this->product('lamp', 'Lamp', 10); + $this->product('desk', 'Desk', 30); + + $queries = [ + $this->ordersJoin(), + Query::join(self::PRODUCTS, 'prod', [Query::on('ord.amount', 'price', '>=')]), + Query::orderDesc('prod.name'), + ]; + + $this->assertSame('Lamp', $this->database->resolveCursor(self::CUSTOMERS, 'alice', CursorDirection::After, $queries)->getAttribute('prod.name'), 'an order of 20 pairs only with the product priced at most 20'); + $this->assertSame('Desk', $this->database->resolveCursor(self::CUSTOMERS, 'bob', CursorDirection::After, $queries)->getAttribute('prod.name'), 'the last of the products an order of 40 pairs with'); + $this->assertCursorSplitsTheList($queries); + } + + public function testCursorWithoutAReadableJoinedRowHasNoJoinedOrderValue(): void + { + $this->customer('alice', readable: true); + $this->order('a-hidden', 'alice', 35, readable: false); + + $cursor = $this->database->resolveCursor(self::CUSTOMERS, 'alice', CursorDirection::After, [$this->ordersJoin(), Query::orderAsc('ord.amount')]); + + $this->assertTrue($cursor->offsetExists('ord.amount')); + $this->assertNull($cursor->getAttribute('ord.amount'), 'without a row the caller can read the order value is null, as it is in the list'); + } + + public function testCursorOnAOneToManyJoinPagesBehindTheDocumentsLastJoinedRow(): void + { + $this->customer('alice', readable: true); + $this->customer('bob', readable: true); + $this->order('a-first', 'alice', 10, readable: true); + $this->order('a-second', 'alice', 25, readable: true); + $this->order('b-only', 'bob', 15, readable: true); + + $queries = [$this->ordersJoin()]; + + $this->assertSame([['bob', 'b-only']], $this->rows($this->page($queries, 'alice', CursorDirection::After)), 'the page after a document starts behind every row the list pairs with it'); + $this->assertSame([['alice', 'a-first'], ['alice', 'a-second']], $this->rows($this->page($queries, 'bob', CursorDirection::Before)), 'the page before a document ends ahead of its first row'); + $this->assertWalksEveryRowOnce($queries, ['alice', 'alice', 'bob'], 2); + } + + public function testCursorOrderedByAJoinedAttributeWithoutAReadableRowPagesAsTheListDoes(): void + { + $this->customer('alice', readable: true); + $this->customer('bob', readable: true); + $this->order('a-hidden', 'alice', 35, readable: false); + $this->order('b-only', 'bob', 15, readable: true); + + $queries = [Query::leftJoin(self::ORDERS, 'ord', [Query::on('$id', 'customerId')]), Query::orderAsc('ord.amount')]; + + $this->assertSame([['bob', 'b-only']], $this->rows($this->page($queries, 'alice', CursorDirection::After)), 'a row the join did not match is a page boundary with null joined values'); + $this->assertCursorSplitsTheList($queries); + } + + public function testCursorOrderedByABareJoinedNamePagesAsTheListDoes(): void + { + $this->customer('alice', readable: true); + $this->customer('bob', readable: true); + $this->order('a-first', 'alice', 10, readable: true); + $this->order('a-second', 'alice', 25, readable: true); + $this->order('b-only', 'bob', 15, readable: true); + + $queries = [$this->ordersJoin(), Query::orderAsc('amount')]; + + $this->assertSame([['alice', 'a-second']], $this->rows($this->page($queries, 'bob', CursorDirection::After)), 'a bare order name only the joined collection declares reads the joined row'); + $this->assertSame(15, $this->database->resolveCursor(self::CUSTOMERS, 'bob', CursorDirection::After, $queries)->getAttribute('ord.amount')); + $this->assertCursorSplitsTheList($queries); + } + + public function testCursorOnAJoinOfTheJoinedIdNeedsNoJoinedRow(): void + { + $this->customer('alice', readable: true); + $this->order('a-first', 'alice', 10, readable: true); + $this->order('a-second', 'alice', 25, readable: true); + + $queries = [Query::join(self::CUSTOMERS, 'cus', [Query::on('customerId', '$id')])]; + $cursor = $this->database->resolveCursor(self::ORDERS, 'a-first', CursorDirection::After, $queries); + + $this->assertFalse($cursor->offsetExists('cus.$id'), 'a join on the joined $id pairs at most one row, so no joined value is needed'); + $this->assertSame(['a-second'], $this->ids($this->database->find(self::ORDERS, [...$queries, Query::cursorAfter($cursor)]))); + } + + public function testCursorOverAJoinOnAOneToManyRelationshipPagesEveryRowOnce(): void + { + $this->database->createRelationship(self::CUSTOMERS, Relationship::oneToMany(relatedCollection: self::ORDERS, key: 'orders', twoWayKey: 'customer')); + $this->customer('alice', readable: true); + $this->customer('bob', readable: true); + $this->customer('carol', readable: true); + $this->order('a-first', 'alice', 10, readable: true, customer: 'alice'); + $this->order('a-second', 'alice', 20, readable: true, customer: 'alice'); + $this->order('b-first', 'bob', 30, readable: true, customer: 'bob'); + $this->order('b-second', 'bob', 40, readable: true, customer: 'bob'); + $this->order('c-hidden', 'carol', 5, readable: false, customer: 'carol'); + $this->order('c-only', 'carol', 50, readable: true, customer: 'carol'); + + $queries = [Query::join(self::ORDERS, 'ord', [Query::on('orders', '$id')]), Query::orderAsc('ord.amount')]; + + $this->assertSame(20, $this->database->resolveCursor(self::CUSTOMERS, 'alice', CursorDirection::After, $queries)->getAttribute('ord.amount')); + $this->assertSame(50, $this->database->resolveCursor(self::CUSTOMERS, 'carol', CursorDirection::Before, $queries)->getAttribute('ord.amount'), 'the hidden order never pairs with carol'); + $this->assertWalksEveryRowOnce($queries, ['alice', 'alice', 'bob', 'bob', 'carol'], 2); + $this->assertCursorSplitsTheList($queries); + } + + public function testCursorCarriesOnlyTheValuesTheListPagesBy(): void + { + $this->customer('alice', readable: true); + $this->order('a-first', 'alice', 10, readable: true, status: 'open'); + + $cursor = $this->database->resolveCursor(self::CUSTOMERS, 'alice', CursorDirection::After, [$this->ordersJoin(), Query::orderDesc('name'), Query::orderAsc('ord.status')]); + $sequence = $this->database->getAuthorization()->skip(fn (): Document => $this->database->getDocument(self::CUSTOMERS, 'alice'))->getSequence(); + + $expected = [ + Document::ID => 'alice', + Document::SEQUENCE => $sequence, + Document::COLLECTION => self::CUSTOMERS, + 'name' => 'Alice', + 'ord.status' => 'open', + 'ord.$id' => 'a-first', + ]; + $actual = $cursor->getArrayCopy(); + \ksort($expected); + \ksort($actual); + + $this->assertSame($expected, $actual); + } + + public function testCursorIgnoresTheCursorQueriesOfTheList(): void + { + $this->customer('alice', readable: true); + $this->customer('bob', readable: true); + $alice = $this->database->getDocument(self::CUSTOMERS, 'alice'); + + $cursor = $this->database->resolveCursor(self::CUSTOMERS, 'bob', CursorDirection::Before, [Query::orderAsc('name'), Query::cursorAfter($alice)]); + + $this->assertSame(['alice'], $this->ids($this->database->find(self::CUSTOMERS, [Query::orderAsc('name'), Query::cursorBefore($cursor)]))); + } + + public function testMissingDocumentResolvesToAnEmptyCursor(): void + { + $this->assertTrue($this->database->resolveCursor(self::CUSTOMERS, 'nobody', CursorDirection::After)->isEmpty()); + $this->assertTrue($this->database->resolveCursor(self::CUSTOMERS, 'nobody', CursorDirection::Before, [$this->ordersJoin(), Query::orderAsc('ord.amount')])->isEmpty()); + } + + public function testMissingCollectionIsNotFound(): void + { + $this->expectException(NotFoundException::class); + + $this->database->resolveCursor('missing', 'alice', CursorDirection::After); + } + + public function testInvalidJoinIsRefused(): void + { + $this->customer('alice', readable: true); + + $this->expectException(QueryException::class); + + $this->database->resolveCursor(self::CUSTOMERS, 'alice', CursorDirection::After, [Query::join('missing', 'gone', [Query::on('$id', 'customerId')])]); + } + + /** + * After each listed document the list resumes behind the last row it pairs with that document, before it the list + * ends ahead of its first row. + * + * @param list $queries + */ + private function assertCursorSplitsTheList(array $queries): void + { + $all = $this->rows($this->database->find(self::CUSTOMERS, [...$queries, Query::limit(100)])); + $ids = \array_column($all, 0); + $this->assertNotSame([], $all, 'the list has rows to page through'); + + foreach (\array_unique($ids) as $id) { + $first = (int) \array_search($id, $ids, true); + $last = (int) \array_key_last(\array_filter($ids, static fn (string $listed): bool => $listed === $id)); + + $this->assertSame(\array_slice($all, $last + 1), $this->rows($this->page($queries, $id, CursorDirection::After)), "after {$id}"); + $this->assertSame(\array_slice($all, 0, $first), $this->rows($this->page($queries, $id, CursorDirection::Before)), "before {$id}"); + } + } + + /** + * Pages of $size forward from the start and backward from the end, each cursor resolved from the ID of the page's + * edge row, return every row of the list once, in list order. + * + * @param list $queries + * @param list $expected + */ + private function assertWalksEveryRowOnce(array $queries, array $expected, int $size = 1): void + { + $all = $this->rows($this->database->find(self::CUSTOMERS, [...$queries, Query::limit(100)])); + $this->assertSame($expected, \array_column($all, 0)); + + $forward = []; + $cursor = null; + for ($page = 0; $page <= \count($all); $page++) { + $rows = $this->rows($this->database->find(self::CUSTOMERS, [ + ...$queries, + Query::limit($size), + ...($cursor === null ? [] : [Query::cursorAfter($this->database->resolveCursor(self::CUSTOMERS, $cursor, CursorDirection::After, $queries))]), + ])); + if ($rows === []) { + break; + } + \array_push($forward, ...$rows); + $cursor = $rows[\count($rows) - 1][0]; + } + $this->assertSame($all, $forward, 'paging forward returns every row once'); + + $cursor = $all[\count($all) - 1][0]; + $backward = \array_values(\array_filter($all, static fn (array $row): bool => $row[0] === $cursor)); + for ($page = 0; $page <= \count($all); $page++) { + $rows = $this->rows($this->database->find(self::CUSTOMERS, [ + ...$queries, + Query::limit($size), + Query::cursorBefore($this->database->resolveCursor(self::CUSTOMERS, $cursor, CursorDirection::Before, $queries)), + ])); + if ($rows === []) { + break; + } + $backward = [...$rows, ...$backward]; + $cursor = $rows[0][0]; + } + $this->assertSame($all, $backward, 'paging backward returns every row once'); + } + + /** + * @param list $queries + * @return list + */ + private function page(array $queries, string $id, CursorDirection $direction): array + { + $cursor = $this->database->resolveCursor(self::CUSTOMERS, $id, $direction, $queries); + + return $this->database->find(self::CUSTOMERS, [ + ...$queries, + $direction === CursorDirection::After ? Query::cursorAfter($cursor) : Query::cursorBefore($cursor), + ]); + } + + /** + * @param array $documents + * @return list + */ + private function ids(array $documents): array + { + return \array_values(\array_map(static fn (Document $document): string => $document->getId(), $documents)); + } + + /** + * @param array $documents + * @return list + */ + private function rows(array $documents): array + { + return \array_values(\array_map(static fn (Document $document): array => [$document->getId(), $document->getAttribute('ord.$id')], $documents)); + } + + private function ordersJoin(): Query + { + return Query::join(self::ORDERS, 'ord', [Query::on('$id', 'customerId')]); + } + + private function customer(string $id, bool $readable): void + { + $this->create(self::CUSTOMERS, $id, ['name' => \ucfirst($id)], $readable); + } + + private function order( + string $id, + string $customerId, + int $amount, + bool $readable, + string $status = 'paid', + ?string $placedAt = null, + ?string $productId = null, + ?string $customer = null, + ): void { + $this->create(self::ORDERS, $id, [ + 'customerId' => $customerId, + 'productId' => $productId, + 'status' => $status, + 'amount' => $amount, + 'placedAt' => $placedAt, + ...($customer === null ? [] : ['customer' => $customer]), + ], $readable); + } + + private function product(string $id, string $name, ?int $price = null): void + { + $this->create(self::PRODUCTS, $id, ['name' => $name, 'price' => $price], readable: true); + } + + /** + * @param array $attributes + */ + private function create(string $collection, string $id, array $attributes, bool $readable): void + { + $this->database->getAuthorization()->skip(fn (): Document => $this->database->createDocument($collection, new Document([ + Document::ID => $id, + Document::PERMISSIONS => [Permission::read(Role::user($readable ? 'reader' : 'other'))], + ...$attributes, + ]))); + } + + /** + * @param list $attributes + */ + private function createCollection(string $id, array $attributes): void + { + $this->database->createCollection(Collection::create( + id: $id, + attributes: $attributes, + permissions: [Permission::create(Role::any())], + documentSecurity: true, + )); + } +} From 9db7e45b917655010f4567597df84693afda82f8 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:16:39 +1300 Subject: [PATCH 15/55] feat(validator): accept fractional operands on integers behind a flag A bulk update or upsert under the fractional-integer mode validates its operators before the rows they change are read locked, so the validator must let a finite fractional amount or limit on an integer attribute through and leave its resolution to the caller. The new trailing $fractions flag on Validator\Operator, threaded through Structure and PartialStructure, does exactly that: the fraction must lie within the attribute's range, the stored-value prediction is skipped for it, and every other operand is checked as before. Off by default, so the 8.0.2 refusals are unchanged. Co-Authored-By: Claude Opus 5.5 --- src/Database/Validator/Operator.php | 47 ++++++++-- src/Database/Validator/Structure.php | 4 + .../unit/Validator/OperatorFractionsTest.php | 87 +++++++++++++++++++ 3 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 tests/unit/Validator/OperatorFractionsTest.php diff --git a/src/Database/Validator/Operator.php b/src/Database/Validator/Operator.php index 082895af4..abbdd29e5 100644 --- a/src/Database/Validator/Operator.php +++ b/src/Database/Validator/Operator.php @@ -3,6 +3,7 @@ namespace Utopia\Database\Validator; use Throwable; +use Utopia\Database\Arithmetic\Fractional; use Utopia\Database\Attribute; use Utopia\Database\Collection; use Utopia\Database\Document; @@ -28,11 +29,15 @@ class Operator extends Validator /** * @param Document|null $currentDocument Current document for runtime validation (e.g., array bounds checking) + * @param bool $fractions Whether a numeric operator on an integer attribute may carry a finite fractional + * amount or limit within the attribute's range, which the caller resolves into a whole + * one from the stored value before it is written */ public function __construct( Document $collection, ?Document $currentDocument = null, private readonly bool $supportUnsignedBigInt = true, + private readonly bool $fractions = false, ) { $this->collection = $collection; $this->currentDocument = $currentDocument; @@ -90,9 +95,37 @@ private function isNumericValueInBounds(mixed $value, Attribute $attribute): boo return $numeric >= $bounds->min && $numeric <= $bounds->max; } + private function isFraction(mixed $value, Attribute $attribute): bool + { + return $this->fractions && $attribute->isInteger() && Fractional::isFractional($value); + } + + /** + * @param int|float|string $value A finite fractional number + */ + private function isFractionInBounds(int|float|string $value, Attribute $attribute): bool + { + $bounds = $attribute->bounds(); + + return $bounds !== null && (float) $value >= (float) $bounds->min && (float) $value <= (float) $bounds->max; + } + private function isValidLimit(mixed $limit, Attribute $attribute, DatabaseOperator $operator): bool { $methodName = $operator->getMethod()->value; + $bounds = $attribute->bounds(); + + if ($bounds !== null && $this->isFraction($limit, $attribute)) { + /** @var float|string $limit */ + if ($this->isFractionInBounds($limit, $attribute)) { + return true; + } + + $this->message = "Cannot apply {$methodName} operator: max/min limit must be between {$bounds->min} and {$bounds->max}"; + + return false; + } + $finite = \is_int($limit) || (\is_float($limit) && \is_finite($limit)) || (\is_string($limit) && \is_numeric($limit)); if (! $attribute->isInteger() || ! $finite) { @@ -113,7 +146,6 @@ private function isValidLimit(mixed $limit, Attribute $attribute, DatabaseOperat return false; } - $bounds = $attribute->bounds(); if ($bounds === null) { $this->message = "Cannot apply {$methodName} operator: max/min limit must be numeric, got ".\gettype($limit); @@ -274,16 +306,19 @@ private function validateOperatorForAttribute( return false; } - if (! isset($values[0]) || ! $this->isNumericValueInBounds($values[0], $attribute)) { + $fraction = isset($values[0]) && $this->isFraction($values[0], $attribute); + if (! isset($values[0]) || ! ($fraction ? $this->isFractionInBounds($values[0], $attribute) : $this->isNumericValueInBounds($values[0], $attribute))) { $this->message = "Cannot apply {$methodName} operator: value must be numeric, got ".gettype($operator->getValue()); return false; } $integerType = $attribute->isInteger(); - $operatorValue = $integerType - ? $this->getIntegerValue($values[0]) - : $this->getNumericValue($values[0]); + $operatorValue = match (true) { + $fraction => (float) $values[0], + $integerType => $this->getIntegerValue($values[0]), + default => $this->getNumericValue($values[0]), + }; if ($operatorValue === null) { return false; } @@ -298,7 +333,7 @@ private function validateOperatorForAttribute( return false; } - if ($this->currentDocument !== null && $integerType && ! isset($values[1])) { + if ($this->currentDocument !== null && $integerType && ! $fraction && ! isset($values[1])) { $currentValue = $this->getIntegerValue($this->currentDocument->getAttribute($operator->getAttribute()) ?? 0); if ($currentValue === null || ! $this->isNumericValueInBounds($currentValue, $attribute)) { diff --git a/src/Database/Validator/Structure.php b/src/Database/Validator/Structure.php index c249ce8c6..7bf5ab801 100644 --- a/src/Database/Validator/Structure.php +++ b/src/Database/Validator/Structure.php @@ -62,12 +62,15 @@ class Structure extends Validator * @param list $storedAttributes Attributes whose values are the stored ones, unchanged by the * write: they are not validated again, as the rules may have * tightened since those values were stored. + * @param bool $fractions Whether numeric operators on integer attributes may carry fractional amounts and + * limits, see Validator\Operator */ public function __construct( protected readonly Document $collection, private readonly Profile $profile, private readonly ?Document $currentDocument = null, array $storedAttributes = [], + private readonly bool $fractions = false, ) { $this->storedAttributes = \array_fill_keys($storedAttributes, true); $this->supportForAttributes = $profile->supports(Capability::DefinedAttributes); @@ -265,6 +268,7 @@ protected function checkForInvalidAttributeValues(Document $document, array $str $this->collection, $this->currentDocument, $this->supportUnsignedBigInt, + $this->fractions, ); if (! $operatorValidator->isValid($value)) { $this->message = $operatorValidator->getDescription(); diff --git a/tests/unit/Validator/OperatorFractionsTest.php b/tests/unit/Validator/OperatorFractionsTest.php new file mode 100644 index 000000000..69cbde48d --- /dev/null +++ b/tests/unit/Validator/OperatorFractionsTest.php @@ -0,0 +1,87 @@ + + */ + public static function fractionalOperators(): iterable + { + yield 'fractional increment' => [Operator::increment(1.5)]; + yield 'fractional text decrement' => [Operator::decrement('0.5')]; + yield 'whole increment with a fractional limit' => [Operator::increment(2, 13.9)]; + yield 'fractional increment with a fractional limit' => [Operator::increment(1.5, 13.5)]; + yield 'fractional multiply' => [Operator::multiply(2.5)]; + yield 'fractional divide with a fractional limit' => [Operator::divide(1.5, 5.5)]; + yield 'fractional modulo' => [Operator::modulo(2.75)]; + yield 'fractional power' => [Operator::power(0.5)]; + } + + #[DataProvider('fractionalOperators')] + public function testAFractionalAmountOrLimitOnAnIntegerIsAcceptedWithTheFlag(Operator $operator): void + { + $validator = $this->validator(fractions: true); + + $this->assertTrue($validator->isValid($operator->setAttribute('count')), $validator->getDescription()); + } + + #[DataProvider('fractionalOperators')] + public function testAFractionalAmountOrLimitOnAnIntegerIsRefusedWithoutTheFlag(Operator $operator): void + { + $this->assertFalse($this->validator()->isValid($operator->setAttribute('count'))); + } + + public function testAFractionOutsideTheIntegerRangeIsRefused(): void + { + $validator = $this->validator(fractions: true); + + $this->assertFalse($validator->isValid(Operator::increment(2147483647.5)->setAttribute('count'))); + $this->assertSame('Cannot apply increment operator: value must be numeric, got double', $validator->getDescription()); + + $this->assertFalse($validator->isValid(Operator::increment(1, -2147483648.5)->setAttribute('count'))); + $this->assertSame('Cannot apply increment operator: max/min limit must be between -2147483648 and 2147483647', $validator->getDescription()); + } + + public function testAFractionIsLeftToTheCallerToResolveAgainstTheStoredValue(): void + { + $validator = $this->validator(new Document(['count' => 2147483647]), fractions: true); + + $this->assertTrue($validator->isValid(Operator::increment(1.5)->setAttribute('count'))); + $this->assertFalse($validator->isValid(Operator::increment(1)->setAttribute('count')), 'A whole amount is still checked against the stored value'); + } + + public function testTheFlagLeavesOtherOperandsAsTheyAre(): void + { + $validator = $this->validator(fractions: true); + + $this->assertFalse($validator->isValid(Operator::increment(\INF)->setAttribute('count'))); + $this->assertFalse($validator->isValid(Operator::increment('ten')->setAttribute('count'))); + $this->assertFalse($validator->isValid(Operator::arrayAppend([1.5])->setAttribute('scores'))); + $this->assertTrue($validator->isValid(Operator::increment(1.5, 10.25)->setAttribute('ratio'))); + } + + private function validator(?Document $current = null, bool $fractions = false): OperatorValidator + { + return new OperatorValidator(new Document([ + Document::ID => 'operands', + Document::COLLECTION => Database::METADATA, + 'name' => 'operands', + 'attributes' => [ + Attribute::integer(key: 'count')->toDocument(), + Attribute::integer(key: 'scores', array: true)->toDocument(), + Attribute::double(key: 'ratio')->toDocument(), + ], + 'indexes' => [], + ]), $current, fractions: $fractions); + } +} From 0500eb57355513739e934ed7bc6ce5417f787e80 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:16:53 +1300 Subject: [PATCH 16/55] feat(documents): round fractions on integer attributes as the engine did appwrite rounded fractional counter changes and operator operands on integer columns app-side (Databases/Counter, IntegerOperators) to keep answering as database 7.4.1 did, where the engine stored the result its own way. The library now owns that behind setRoundFractions / isRoundingFractions / withRoundFractions, a State\Value toggle carried by State\Snapshot and forwarded by Mirror. Off by default: every 8.0.2 refusal is unchanged. When on, for single-valued integer and bigint attributes, using the adapter's Arithmetic: - increase/decreaseDocumentAttribute: a whole change floors a fractional maximum and ceils a fractional minimum; a float change is answered from the row read locked in the write's transaction: the exact result is held to the bound as given (Limit before any write), the whole-number change that stores what the engine stored is written (a change of 0 still bumps $updatedAt, purges and fires its events) and the exact float is returned. Bigint stays exact past 2^53; <=0 is still a Type. - updateDocument resolves fractional numeric operators against the locked old document before structure validation and cuts fractional integer-array items toward zero; an operator the engine refused comes back unchanged and the strict validator refuses it as before. - updateDocuments and upsertDocument(s), only when a fractional operator targets an integer column, validate with the fractions flag, then read each row locked inside the batch transaction (under its own tenant), resolve per row, re-validate strictly, and send one adapter update per group of identical resolved operators. A row an upsert creates resolves against 0. The default path reads the toggle once and is otherwise unchanged. D7, PostgreSQL counters on 7.4.1 (source reading of 7.4.1:src/Database/Adapter/Postgres.php increaseDocumentAttribute and 7.4.1:src/Database/Database.php increase/decreaseDocumentAttribute, not a measured run): Database first compared stored +/- value with the bound in PHP and threw LimitException past it; it then bound :val and :max = max - value (:min = min + value) with bindValue as text under emulated prepares into `"attr" = "attr" + :val ... AND "attr" <= :max`. PostgreSQL types an untyped literal next to an integer column as integer, so '1.5' or '8.5' fails with invalid input syntax for type integer. Hence a fractional change was refused with or without a bound, a whole change with a fractional bound was refused too, and in both cases a result past the bound was a LimitException first. Encoded as: Limit check on the exact result, then Type 'Change value must be an integer.' for a fractional change, then Type 'Max/Min must be an integer.' for a non-whole bound; a whole float change with a whole bound is stored. MongoDB stored the double; the per-operation toward-zero result is the documented approximation. Co-Authored-By: Claude Opus 5.5 --- src/Database/Database.php | 51 +- src/Database/Mirror.php | 17 + src/Database/State/Snapshot.php | 2 + src/Database/Trait/Documents.php | 337 +++++- tests/unit/Documents/RoundFractionsTest.php | 1012 +++++++++++++++++++ 5 files changed, 1387 insertions(+), 32 deletions(-) create mode 100644 tests/unit/Documents/RoundFractionsTest.php diff --git a/src/Database/Database.php b/src/Database/Database.php index c45e3eb79..53151abb4 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -272,6 +272,9 @@ class Database /** @var Value|null */ private ?Value $grantChecking = null; + /** @var Value|null */ + private ?Value $fractionRounding = null; + protected ?Relationships $relationshipHook = null; protected bool $dropUnknownAttributes = false; @@ -1224,6 +1227,42 @@ public function withCheckGrants(bool $check, callable $callback): mixed return $this->grantChecking()->with($check, $callback); } + /** + * Set whether a fractional amount or limit on a single-valued integer attribute is stored as the adapter's + * engine stored it on database 7.4.1 (see Adapter::arithmetic()), instead of being refused. This covers + * increaseDocumentAttribute() and decreaseDocumentAttribute(), whose float change returns its exact result, and + * the numeric operators given to updateDocument(), updateDocuments(), upsertDocument() and upsertDocuments(), + * each answered from the stored value it changes, read locked in the write's transaction. Fractional items + * added to an integer array are cut toward zero. What the engine refused stays refused. + * + * @return $this + */ + public function setRoundFractions(bool $round): static + { + $this->fractionRounding()->set($round); + + return $this; + } + + public function isRoundingFractions(): bool + { + return $this->fractionRounding()->get(); + } + + /** + * Run the callback with fractions on integer attributes rounded or refused for the calling coroutine and the + * coroutines it starts. + * + * @template T + * + * @param callable(): T $callback + * @return T + */ + public function withRoundFractions(bool $round, callable $callback): mixed + { + return $this->fractionRounding()->with($round, $callback); + } + /** * Set the migration mode flag, which relaxes certain constraints during data migrations. * @@ -1631,6 +1670,7 @@ public function snapshot(): Snapshot requestTimestamp: $this->requestTimestamp()->get(), replacePlaceholders: $this->placeholderReplacement()->get(), checkGrants: $this->grantChecking()->get(), + roundFractions: $this->fractionRounding()->get(), ); } @@ -1671,7 +1711,8 @@ public function withSnapshot(Snapshot $snapshot, callable $callback): mixed */ private function withToggles(Snapshot $snapshot, callable $callback): mixed { - $checked = fn (): mixed => $this->grantChecking()->with($snapshot->checkGrants, $callback); + $rounded = fn (): mixed => $this->fractionRounding()->with($snapshot->roundFractions, $callback); + $checked = fn (): mixed => $this->grantChecking()->with($snapshot->checkGrants, $rounded); $replaced = fn (): mixed => $this->placeholderReplacement()->with($snapshot->replacePlaceholders, $checked); $timestamped = fn (): mixed => $this->requestTimestamp()->with($snapshot->requestTimestamp, $replaced); $deduplicated = fn (): mixed => $this->ignoringDuplicates()->with($snapshot->ignoreDuplicates, $timestamped); @@ -1768,6 +1809,14 @@ private function grantChecking(): Value return $this->grantChecking ??= new Value(false); } + /** + * @return Value + */ + private function fractionRounding(): Value + { + return $this->fractionRounding ??= new Value(false); + } + protected function isIgnoringDuplicates(): bool { return $this->ignoringDuplicates()->get(); diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php index ff029a3fd..ef8a71a7c 100644 --- a/src/Database/Mirror.php +++ b/src/Database/Mirror.php @@ -498,6 +498,14 @@ public function setCheckGrants(bool $check): static return parent::setCheckGrants($check); } + #[\Override] + public function setRoundFractions(bool $round): static + { + $this->delegate(__FUNCTION__, \func_get_args()); + + return parent::setRoundFractions($round); + } + #[\Override] public function setValidation(bool $validation): static { @@ -564,6 +572,15 @@ public function withCheckGrants(bool $check, callable $callback): mixed return parent::withCheckGrants($check, fn (): mixed => $this->source->withCheckGrants($check, $scoped)); } + #[\Override] + public function withRoundFractions(bool $round, callable $callback): mixed + { + $destination = $this->destination; + $scoped = $destination === null ? $callback : fn (): mixed => $destination->withRoundFractions($round, $callback); + + return parent::withRoundFractions($round, fn (): mixed => $this->source->withRoundFractions($round, $scoped)); + } + #[\Override] public function skipRelationships(callable $callback): mixed { diff --git a/src/Database/State/Snapshot.php b/src/Database/State/Snapshot.php index 5bd5130fc..be301923b 100644 --- a/src/Database/State/Snapshot.php +++ b/src/Database/State/Snapshot.php @@ -29,6 +29,7 @@ * @param DateTime|null $requestTimestamp The time an update conflicts after * @param bool $replacePlaceholders Whether a related document sent with an Id::PLACEHOLDER ID gets a generated one * @param bool $checkGrants Whether a related document may only be given permissions for roles the caller holds + * @param bool $roundFractions Whether a fractional amount or limit on an integer attribute is stored as the engine rounded it */ public function __construct( public bool $authorization, @@ -48,6 +49,7 @@ public function __construct( public ?DateTime $requestTimestamp, public bool $replacePlaceholders = false, public bool $checkGrants = false, + public bool $roundFractions = false, ) { } } diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 79c08a931..fed96d36a 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -12,6 +12,7 @@ use Utopia\Database\Adapter; use Utopia\Database\Adapter\Feature; use Utopia\Database\Adapter\ReadWritePool; +use Utopia\Database\Arithmetic\Fractional; use Utopia\Database\Attribute; use Utopia\Database\Cache\Epoch; use Utopia\Database\Cache\Owners; @@ -282,6 +283,173 @@ private function integerBound(int|float|string $bound, string $name): int|string return BigInt::integralValue($bound) ?? throw new TypeException($name.' must be an integer.'); } + /** + * A bound on an integer attribute as the whole number it allows: with fractions rounded, a fractional maximum + * rounds down and a fractional minimum up. + * + * @throws TypeException + */ + private function integerLimit(int|float|string $bound, bool $maximum, bool $round): int|string + { + if ($round) { + $bound = ($maximum ? Fractional::maximum($bound) : Fractional::minimum($bound)) ?? $bound; + } + + return $this->integerBound($bound, $maximum ? 'Max' : 'Min'); + } + + /** + * With fractions rounded, whether a change on an integer attribute is answered from the stored value it + * changes: a float or decimal text change, which returns its exact result, or a whole change whose bound is + * not a whole number on an engine that refused a fractional counter. + */ + private function isExactChange(int|float|string $value, int|float|string|null $bound): bool + { + if (BigInt::integralValue($value) === null && ! Fractional::isFractional($value)) { + return false; + } + + if (! BigInt::isIntegerValue($value)) { + return true; + } + + return $bound !== null + && \is_numeric($bound) + && BigInt::integralValue($bound) === null + && $this->adapter->arithmetic()->refusesCounterFractions; + } + + /** + * @throws TypeException + */ + private function assertNumericBound(int|float|string $bound, string $name): void + { + if (! \is_numeric($bound) || (\is_float($bound) && \is_nan($bound))) { + throw new TypeException($name.' must be an integer.'); + } + } + + /** + * The whole-number change that stores what the engine stored for $value added to or taken from the stored + * value, with the exact result. The exact result is held to the bound as given, before the engine's own + * refusals, in the order database 7.4.1 checked them. + * + * @return array{int|string, float} + * + * @throws LimitException + * @throws TypeException + */ + private function exactChange(mixed $stored, int|float|string $value, int|float|string|null $bound, bool $increase): array + { + $stored ??= 0; + if (! BigInt::isIntegerValue($stored)) { + throw new TypeException('Attribute value must be an integer.'); + } + + $exact = $increase ? (float) $stored + (float) $value : (float) $stored - (float) $value; + if ($bound !== null && ($increase ? $exact > (float) $bound : $exact < (float) $bound)) { + throw new LimitException('Attribute value exceeds '.($increase ? 'maximum' : 'minimum').' limit: '.$bound); + } + + $arithmetic = $this->adapter->arithmetic(); + $change = (new Fractional($arithmetic))->counter(BigInt::toNative($stored), $value, $increase); + + if ($bound !== null && $arithmetic->refusesCounterFractions && BigInt::integralValue($bound) === null) { + throw new TypeException(($increase ? 'Max' : 'Min').' must be an integer.'); + } + + return [$change, $exact]; + } + + /** + * The keys of the single-valued integer attributes a document changes with a numeric operator whose amount or + * limit is fractional. + * + * @return list + */ + private static function fractionalKeys(Collection $collection, Document $document): array + { + $keys = []; + foreach ($collection->attributes() as $attribute) { + $value = $document->getAttribute($attribute->key); + if ($value instanceof Operator && ! $attribute->array && $attribute->isInteger() && Fractional::needsStored($value)) { + $keys[] = $attribute->key; + } + } + + return $keys; + } + + /** + * Cut each fractional item a document adds to an integer array toward zero, as every engine read it back. + */ + private static function wholeItems(Collection $collection, Document $document): void + { + foreach ($collection->attributes() as $attribute) { + $value = $document->getAttribute($attribute->key); + if ($value instanceof Operator && $attribute->array && $attribute->isInteger()) { + $document->setAttribute($attribute->key, Fractional::whole($value)); + } + } + } + + /** + * The operators under $keys answered from the stored document as the engine answered them; one the engine + * refused comes back unchanged. + * + * @param list $keys + * @return array + */ + private static function resolveFractions(Document $document, array $keys, Document $stored, Fractional $fractional): array + { + $resolved = []; + foreach ($keys as $key) { + $operator = $document->getAttribute($key); + if ($operator instanceof Operator) { + $resolved[$key] = $fractional->operator($operator, $stored->getAttribute($key))->setAttribute($key); + } + } + + return $resolved; + } + + /** + * @param array $operators + * + * @throws StructureException when the engine refused an operator, which still carries its fraction + */ + private function assertResolvedFractions(Collection $collection, array $operators, Document $stored): void + { + if (! $this->validation()->get()) { + return; + } + + $validator = new PartialStructure( + collection: $collection, + profile: $this->profile(), + currentDocument: $stored->isEmpty() ? null : $stored, + ); + if (! $validator->isValid(new Document($operators))) { + throw new StructureException($validator->getDescription()); + } + } + + /** + * The stored document, read locked for the write's transaction under its own tenant, whatever the caller may + * read. + */ + private function readLocked(Collection $collection, Document $document): Document + { + $read = fn (): Document => $this->authorization->skip(fn () => $this->silent( + fn () => $this->readDocument($collection, $document->getId(), forUpdate: true) + )); + $tenant = $document->getTenant(); + + return $this->hasSharedTables() && $this->isTenantPerDocument() && $tenant !== null + ? $this->withTenant($tenant, $read) + : $read(); + } + /** * Cached validator instances keyed by context and a * stable schema/authorization fingerprint. @@ -1559,10 +1727,11 @@ public function updateDocument(string $collection, string $id, Document $documen $document = $this->prepareNested($collection, $document); $newUpdatedAt = $document->getUpdatedAt(); $hasOperators = false; + $round = $this->fractionRounding()->get(); $cacheTarget = $collection->getId() === self::METADATA ? new Document([Document::ID => $id, Document::COLLECTION => self::METADATA]) : $collection->getId(); - $document = $this->withMutation(Event::DocumentUpdate, $cacheTarget, function () use ($collection, $id, $document, $newUpdatedAt, &$hasOperators) { + $document = $this->withMutation(Event::DocumentUpdate, $cacheTarget, function () use ($collection, $id, $document, $newUpdatedAt, $round, &$hasOperators) { $old = $this->authorization->skip(fn () => $this->silent( fn () => $this->readDocument($collection, $id, forUpdate: true) )); @@ -1602,6 +1771,14 @@ public function updateDocument(string $collection, string $id, Document $documen // for a write that leaves the stored document identical. $document = $this->removeUnknownAttributes($collection, $document); + if ($round) { + self::wholeItems($collection, $document); + $keys = self::fractionalKeys($collection, $document); + if ($keys !== []) { + $document->setAttributes(self::resolveFractions($document, $keys, $old, new Fractional($this->adapter->arithmetic()))); + } + } + $shouldUpdate = false; if ($collection->getId() !== self::METADATA) { @@ -1923,6 +2100,13 @@ public function updateDocuments( $updatedAt = $updates->getUpdatedAt(); $updates[Document::UPDATED_AT] = ($updatedAt === null || ! $this->datePreservation()->get()) ? DateTime::now() : $updatedAt; + $fractionalKeys = []; + if ($this->fractionRounding()->get()) { + self::wholeItems($collection, $updates); + $fractionalKeys = self::fractionalKeys($collection, $updates); + } + $fractional = $fractionalKeys === [] ? null : new Fractional($this->adapter->arithmetic()); + $decodedUpdates = clone $updates; $updates = $this->encode( $collection, @@ -1934,6 +2118,7 @@ public function updateDocuments( $validator = new PartialStructure( collection: $collection, profile: $this->profile(), + fractions: $fractional !== null, ); if (! $validator->isValid($updates)) { @@ -1992,8 +2177,9 @@ public function updateDocuments( $cacheTarget = $collection->getId() === self::METADATA ? $batch : $collection->getId(); $found = $batch; - $this->withMutation(Event::DocumentsUpdate, $cacheTarget, function () use ($collection, $updates, $decodedUpdates, $adapterUpdates, &$batch, $found, $currentPermissions) { + $this->withMutation(Event::DocumentsUpdate, $cacheTarget, function () use ($collection, $updates, $decodedUpdates, $adapterUpdates, $adapterData, $fractionalKeys, $fractional, &$batch, $found, $currentPermissions) { $keepsPermissions = []; + $groups = []; foreach ($found as $index => $document) { $skipPermissionsUpdate = true; @@ -2013,9 +2199,19 @@ public function updateDocuments( // keeps the permission rows of every document under that id rewritten. $keepsPermissions[$document->getId()] = $skipPermissionsUpdate && ($keepsPermissions[$document->getId()] ?? true); + $resolved = []; + if ($fractional !== null) { + $stored = $this->readLocked($collection, $document); + $resolved = self::resolveFractions($decodedUpdates, $fractionalKeys, $stored, $fractional); + $this->assertResolvedFractions($collection, $resolved, $stored); + $signature = \serialize(\array_map(static fn (Operator $operator): array => $operator->toArray(), $resolved)); + $groups[$signature]['operators'] = $resolved; + $groups[$signature]['indexes'][] = $index; + } + $updateData = []; foreach ($decodedUpdates->getArrayCopy() as $key => $value) { - $updateData[$key] = $value instanceof Operator ? clone $value : $value; + $updateData[$key] = $resolved[$key] ?? ($value instanceof Operator ? clone $value : $value); } $new = new Document(\array_merge($document->getArrayCopy(), $updateData)); @@ -2041,12 +2237,23 @@ public function updateDocuments( $batch[$index] = $this->castBefore($collection, $encoded); } - $this->adapter->updateDocuments( - $collection, - $adapterUpdates, - $batch, - \array_filter($keepsPermissions), - ); + if ($groups === []) { + $this->adapter->updateDocuments( + $collection, + $adapterUpdates, + $batch, + \array_filter($keepsPermissions), + ); + } + + foreach ($groups as ['operators' => $operators, 'indexes' => $indexes]) { + $this->adapter->updateDocuments( + $collection, + $this->castBefore($collection, new Document([...$adapterData, ...$operators])), + \array_values(\array_intersect_key($batch, \array_flip($indexes))), + \array_filter($keepsPermissions), + ); + } foreach ($batch as $document) { $this->withDocumentTenant( @@ -2239,12 +2446,20 @@ private function upsert( self::internalAttributesFor(true) )); $existing = $this->findDocumentsToUpsert($collection->getId(), $documents); + $round = $this->fractionRounding()->get(); + $fractionalIds = []; foreach ($documents as $key => $document) { $old = $existing[$this->upsertKey($document)] ?? new Document(); $document = $this->removeUnknownAttributes($collection, $document); + $fractions = false; + if ($round) { + self::wholeItems($collection, $document); + $fractions = self::fractionalKeys($collection, $document) !== []; + } + // Extract operators early to avoid comparison issues $documentArray = $document->getArrayCopy(); $extracted = Operator::extractOperators($documentArray); @@ -2372,7 +2587,8 @@ private function upsert( $validator = new Structure( collection: $collection, profile: $this->profile(), - currentDocument: $old->isEmpty() ? null : $old + currentDocument: $old->isEmpty() ? null : $old, + fractions: $fractions, ); if (! $validator->isValid($document)) { @@ -2404,6 +2620,9 @@ private function upsert( if (! empty($operators)) { $operatorIds[$identity] = true; } + if ($fractions) { + $fractionalIds[$identity] = true; + } $old = $this->castBefore($collection, $old); $document = $this->castBefore($collection, $document); @@ -2433,11 +2652,15 @@ private function upsert( $batch = $this->withMutation( Event::DocumentsUpsert, \array_map(static fn (Change $change): Document => $change->new, $chunk), - function () use ($collection, $increase, $chunk): array { + function () use ($collection, $increase, $chunk, $fractionalIds): array { if (! $this->adapterHasFeature(Feature\Upserts::class)) { throw new DatabaseException('Adapter does not support upserts'); } + if ($fractionalIds !== []) { + $this->resolveUpsertFractions($collection, $chunk, $fractionalIds); + } + $adapter = $this->adapter; $batch = $this->authorization->skip(fn () => $adapter->upsertDocuments( $collection, @@ -2512,6 +2735,30 @@ function () use ($collection, $increase, $chunk): array { return [$created, $updated]; } + /** + * Answer the fractional operators of the upserted documents under $identities from their stored versions, read + * locked in the write's transaction; a document not stored yet is answered from zero. + * + * @param array $chunk + * @param array $identities + * + * @throws StructureException when the engine refused an operator + */ + private function resolveUpsertFractions(Collection $collection, array $chunk, array $identities): void + { + $fractional = new Fractional($this->adapter->arithmetic()); + foreach ($chunk as $change) { + if (! isset($identities[$this->getDocumentIdentity($change->new)])) { + continue; + } + + $stored = $this->readLocked($collection, $change->new); + $resolved = self::resolveFractions($change->new, self::fractionalKeys($collection, $change->new), $stored, $fractional); + $this->assertResolvedFractions($collection, $resolved, $stored); + $change->new->setAttributes($this->castBefore($collection, new Document($resolved))->getArrayCopy()); + } + } + /** * Load the stored documents an upsert batch will be compared against, in one * read per tenant instead of one per document. A batch of N documents costs a @@ -2603,18 +2850,27 @@ public function increaseDocumentAttribute( $collection = $this->silent(fn () => $this->getCollection($collection)); $numericAttribute = $this->numericAttribute($collection, $attribute); + $exact = false; if ($this->isDeclaredInteger($numericAttribute ?? $this->declaredAttribute($collection, $attribute))) { - $this->assertIntegerChange($value); - if ($max !== null) { - $max = $this->integerBound($max, 'Max'); + $round = $this->fractionRounding()->get(); + $exact = $round && $this->isExactChange($value, $max); + if ($exact) { + if ($max !== null) { + $this->assertNumericBound($max, 'Max'); + } + } else { + $this->assertIntegerChange($value); + if ($max !== null) { + $max = $this->integerLimit($max, true, $round); + } } } $cacheTarget = $collection->getId() === self::METADATA ? new Document([Document::ID => $id, Document::COLLECTION => self::METADATA]) : $collection->getId(); - $document = $this->withMutation(Event::DocumentIncrease, $cacheTarget, function () use ($collection, $id, $attribute, $value, $max, $numericAttribute) { + $document = $this->withMutation(Event::DocumentIncrease, $cacheTarget, function () use ($collection, $id, $attribute, $value, $max, $numericAttribute, $exact) { /** @var Document $document */ $document = $this->authorization->skip(fn () => $this->silent(fn () => $this->readDocument($collection, $id, forUpdate: true))); // Skip ensures user does not need read permission for this @@ -2634,17 +2890,22 @@ public function increaseDocumentAttribute( } $attributeExists = $document->offsetExists($attribute); - $currentVal = $document->getAttribute($attribute); + $current = $document->getAttribute($attribute); + $returned = null; + if ($exact) { + [$value, $returned] = $this->exactChange($current, $value, $max, true); + $max = null; + } if ($numericAttribute instanceof Attribute) { - $result = $this->getNumericResult($numericAttribute, $currentVal, $value, true); + $result = $this->getNumericResult($numericAttribute, $current, $value, true); } else { if (! $attributeExists) { - $currentVal = 0; + $current = 0; } - if (! \is_int($currentVal) && ! \is_float($currentVal)) { + if (! \is_int($current) && ! \is_float($current)) { throw new TypeException('Attribute value must be numeric.'); } - $result = $currentVal + $this->getNativeNumber($value); + $result = $current + $this->getNativeNumber($value); } $exceedsMaximum = ! \is_null($max) && ( $numericAttribute?->isInteger() === true @@ -2678,7 +2939,7 @@ public function increaseDocumentAttribute( $this->purgeCachedDocumentInternal($collection->getId(), $id); $this->queueDocumentPurge($collection->getId(), $id); - return $document->setAttribute($attribute, $result); + return $document->setAttribute($attribute, $returned ?? $result); }); $listeners = $this->listens(Event::DocumentIncrease); @@ -2727,18 +2988,27 @@ public function decreaseDocumentAttribute( $collection = $this->silent(fn () => $this->getCollection($collection)); $numericAttribute = $this->numericAttribute($collection, $attribute); + $exact = false; if ($this->isDeclaredInteger($numericAttribute ?? $this->declaredAttribute($collection, $attribute))) { - $this->assertIntegerChange($value); - if ($min !== null) { - $min = $this->integerBound($min, 'Min'); + $round = $this->fractionRounding()->get(); + $exact = $round && $this->isExactChange($value, $min); + if ($exact) { + if ($min !== null) { + $this->assertNumericBound($min, 'Min'); + } + } else { + $this->assertIntegerChange($value); + if ($min !== null) { + $min = $this->integerLimit($min, false, $round); + } } } $cacheTarget = $collection->getId() === self::METADATA ? new Document([Document::ID => $id, Document::COLLECTION => self::METADATA]) : $collection->getId(); - $document = $this->withMutation(Event::DocumentDecrease, $cacheTarget, function () use ($collection, $id, $attribute, $value, $min, $numericAttribute) { + $document = $this->withMutation(Event::DocumentDecrease, $cacheTarget, function () use ($collection, $id, $attribute, $value, $min, $numericAttribute, $exact) { /** @var Document $document */ $document = $this->authorization->skip(fn () => $this->silent(fn () => $this->readDocument($collection, $id, forUpdate: true))); // Skip ensures user does not need read permission for this @@ -2758,17 +3028,22 @@ public function decreaseDocumentAttribute( } $attributeExists = $document->offsetExists($attribute); - $currentDecVal = $document->getAttribute($attribute); + $current = $document->getAttribute($attribute); + $returned = null; + if ($exact) { + [$value, $returned] = $this->exactChange($current, $value, $min, false); + $min = null; + } if ($numericAttribute instanceof Attribute) { - $result = $this->getNumericResult($numericAttribute, $currentDecVal, $value, false); + $result = $this->getNumericResult($numericAttribute, $current, $value, false); } else { if (! $attributeExists) { - $currentDecVal = 0; + $current = 0; } - if (! \is_int($currentDecVal) && ! \is_float($currentDecVal)) { + if (! \is_int($current) && ! \is_float($current)) { throw new TypeException('Attribute value must be numeric.'); } - $result = $currentDecVal - $this->getNativeNumber($value); + $result = $current - $this->getNativeNumber($value); } $belowMinimum = ! \is_null($min) && ( $numericAttribute?->isInteger() === true @@ -2802,7 +3077,7 @@ public function decreaseDocumentAttribute( $this->purgeCachedDocumentInternal($collection->getId(), $id); $this->queueDocumentPurge($collection->getId(), $id); - return $document->setAttribute($attribute, $result); + return $document->setAttribute($attribute, $returned ?? $result); }); $listeners = $this->listens(Event::DocumentDecrease); diff --git a/tests/unit/Documents/RoundFractionsTest.php b/tests/unit/Documents/RoundFractionsTest.php new file mode 100644 index 000000000..266fe0265 --- /dev/null +++ b/tests/unit/Documents/RoundFractionsTest.php @@ -0,0 +1,1012 @@ + + */ + public static function integerBounds(): iterable + { + yield 'fractional' => [10.5, 10, 11]; + yield 'negative fractional' => [-2.5, -3, -2]; + yield 'just under a whole number' => [9.999, 9, 10]; + yield 'whole float' => [10.0, 10, 10]; + yield 'integer' => [10, 10, 10]; + yield 'fractional text' => ['10.5', 10, 11]; + yield 'integer text' => ['10', '10', '10']; + yield 'beyond the native integer range' => [1.0e19, '10000000000000000000', '10000000000000000000']; + yield 'fractional text beyond float precision' => ['9007199254740993.5', 9007199254740993, 9007199254740994]; + yield 'negative fractional text beyond float precision' => ['-9007199254740993.5', -9007199254740994, -9007199254740993]; + yield 'negative fractional text' => ['-5.5', -6, -5]; + yield 'negative fraction of one text' => ['-0.5', -1, 0]; + yield 'fraction of one text' => ['0.5', 0, 1]; + yield 'whole decimal text' => ['10.000', 10, 10]; + yield 'leading zeros text' => ['0000000000000000000005.5', 5, 6]; + yield 'fractional text above the native integer range' => ['9223372036854775807.5', \PHP_INT_MAX, '9223372036854775808']; + yield 'fractional text below the native integer range' => ['-9223372036854775808.5', '-9223372036854775809', \PHP_INT_MIN]; + yield 'fractional text at the unsigned limit' => ['18446744073709551615.9', '18446744073709551615', '18446744073709551616']; + yield 'exponent text' => ['1e3', 1000, 1000]; + yield 'capital exponent text' => ['1E3', 1000, 1000]; + yield 'fractional exponent text' => ['1.25e1', 12, 13]; + yield 'negative fractional exponent text' => ['-1.25e1', -13, -12]; + yield 'negative exponent text' => ['1e-3', 0, 1]; + yield 'negative value with a negative exponent text' => ['-1e-3', -1, 0]; + yield 'vanishing exponent text' => ['1e-999999999', 0, 1]; + yield 'negative value with a vanishing exponent text' => ['-1e-999999999', -1, 0]; + yield 'exponent beyond the native integer range text' => ['5e-9223372036854775808', 0, 1]; + yield 'zero exponent text beyond float precision' => ['9007199254740993.5e0', 9007199254740993, 9007199254740994]; + yield 'padded fractional text' => [' 10.5 ', 10, 11]; + yield 'fractional text padded with form feed and vertical tab' => ["\f10.5\v", 10, 11]; + yield 'signed fractional text' => ['+10.5', 10, 11]; + yield 'fraction without a whole part text' => ['.5', 0, 1]; + yield 'whole part without a fraction text' => ['5.', 5, 5]; + yield 'negative zero text' => ['-0.0', 0, 0]; + yield 'zero with a huge exponent text' => ['0e999999999', 0, 0]; + yield 'zero decimal with an exponent text' => ['0.000e50', 0, 0]; + yield 'negative zero with an exponent text' => ['-0e5', 0, 0]; + yield 'no bound' => [null, null, null]; + } + + #[DataProvider('integerBounds')] + public function testAWholeChangeIsHeldAtTheWholeNumberAFractionalBoundRoundsTo(int|float|string|null $bound, int|string|null $maximum, int|string|null $minimum): void + { + $database = $this->database(); + + if ($bound === null || $maximum === null || $minimum === null) { + $this->create($database, ['big' => 5]); + $this->assertSame(6, $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'big', 1, $bound)->getAttribute('big')); + $this->assertSame(5, $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'big', 1, $bound)->getAttribute('big')); + + return; + } + + $this->assertHeldAtMaximum($database, $bound, $maximum); + $this->assertHeldAtMinimum($database, $bound, $minimum); + } + + public function testAnIntegerAttributeRoundsItsBoundsLikeABigInteger(): void + { + $database = $this->database(); + $this->create($database, ['count' => 9], 'up'); + $this->create($database, ['count' => 12], 'down'); + + $this->assertSame(10, $database->increaseDocumentAttribute(self::COLLECTION, 'up', 'count', 1, 10.5)->getAttribute('count')); + $this->assertLimited(fn () => $database->increaseDocumentAttribute(self::COLLECTION, 'up', 'count', 1, 10.5)); + $this->assertSame(11, $database->decreaseDocumentAttribute(self::COLLECTION, 'down', 'count', 1, 10.5)->getAttribute('count')); + $this->assertLimited(fn () => $database->decreaseDocumentAttribute(self::COLLECTION, 'down', 'count', 1, 10.5)); + + $this->assertSame(10, $this->stored($database, 'count', 'up')); + $this->assertSame(11, $this->stored($database, 'count', 'down')); + } + + /** + * @return iterable|null}> + */ + public static function unroundedAttributes(): iterable + { + yield 'double' => ['ratio', null]; + yield 'integer array' => ['scores', TypeException::class]; + yield 'undeclared' => ['missing', NotFoundException::class]; + } + + /** + * @param class-string|null $refusal + */ + #[DataProvider('unroundedAttributes')] + public function testOtherAttributesKeepTheirBoundsAndChangeValues(string $attribute, ?string $refusal): void + { + $database = $this->database(); + $this->create($database, ['ratio' => 5.25]); + + if ($refusal !== null) { + $this->expectException($refusal); + } + + $this->assertSame(6.75, $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, $attribute, 1.5, 6.8)->getAttribute($attribute)); + $this->assertSame(5.25, $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, $attribute, 1.5, 5.2)->getAttribute($attribute)); + $this->assertSame(5.25, $this->stored($database, $attribute)); + } + + public function testBoundsTheLibraryRefusesAnywayArePassedOnUnchanged(): void + { + $database = $this->database(); + $this->create($database, ['count' => 5]); + + $this->assertRefused('Max must be an integer.', fn () => $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 1, \INF)); + $this->assertRefused('Min must be an integer.', fn () => $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 1, 'ten')); + $this->assertSame(5, $this->stored($database, 'count')); + } + + /** + * @return iterable + */ + public static function nonNumericBounds(): iterable + { + yield 'empty' => ['']; + yield 'exponent without digits' => ['1e']; + yield 'hexadecimal' => ['0x1A']; + yield 'digit separators' => ['1_000']; + yield 'two decimal points' => ['1.2.3']; + yield 'two signs' => ['--5']; + } + + #[DataProvider('nonNumericBounds')] + public function testNonNumericTextBoundsArePassedOnUnchanged(string $bound): void + { + $database = $this->database(); + $this->create($database, ['count' => 5]); + + $this->assertRefused('Max must be an integer.', fn () => $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 1, $bound)); + $this->assertRefused('Min must be an integer.', fn () => $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 1, $bound)); + $this->assertRefused('Max must be an integer.', fn () => $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 1.5, $bound)); + $this->assertSame(5, $this->stored($database, 'count')); + } + + /** + * @return iterable + */ + public static function integerChanges(): iterable + { + yield 'integer' => [2, 7]; + yield 'whole float' => [2.0, 7]; + yield 'integer text' => ['2', 7]; + yield 'fractional' => [1.5, 6]; + yield 'fractional text' => ['1.5', 6]; + yield 'infinite' => [\INF, null]; + } + + #[DataProvider('integerChanges')] + public function testAnIntegerAttributeStoresEveryFiniteChangeAsAWholeNumber(int|float|string $value, ?int $stored): void + { + $database = $this->database(); + $this->create($database, ['count' => 5]); + + if ($stored === null) { + $this->assertRefused('Change value must be an integer.', fn () => $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', $value)); + $this->assertSame(5, $this->stored($database, 'count')); + + return; + } + + $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', $value); + + $this->assertSame($stored, $this->stored($database, 'count')); + } + + public function testAWholeChangeValueOnAnIntegerAttributeIsPassedAsAnInteger(): void + { + $database = $this->database(); + $this->create($database, ['count' => 8]); + + $this->assertSame(10, $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 2)->getAttribute('count')); + $this->assertSame(12, $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', '2')->getAttribute('count')); + $this->assertSame(12, $this->stored($database, 'count')); + } + + /** + * @return iterable + */ + public static function metadataAttributes(): iterable + { + yield 'integer' => ['count', 5, null, 6.5, 6]; + yield 'big integer as stored' => ['big', 5, null, 6.5, 6]; + yield 'double' => ['ratio', 5, null, 6.5, 6.5]; + yield 'integer array' => ['scores', null, 'Attribute value must be numeric.', null, null]; + yield 'undeclared' => ['missing', null, null, 1.5, 1.5]; + } + + #[DataProvider('metadataAttributes')] + public function testCollectionMetadataDeclaresTheIntegerAttributesWithoutDefinedAttributes(string $attribute, ?int $start, ?string $refusal, ?float $returned, int|float|null $stored): void + { + $database = $this->database(new class () extends Memory { + #[\Override] + public function capabilities(): array + { + return \array_values(\array_filter( + parent::capabilities(), + static fn (Capability $capability): bool => $capability !== Capability::DefinedAttributes, + )); + } + }); + $this->create($database, $start === null ? [] : [$attribute => $start]); + $increase = fn (): Document => $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, $attribute, 1.5, 10.5); + + if ($refusal !== null) { + $this->assertRefused($refusal, $increase); + + return; + } + + $this->assertSame($returned, $increase()->getAttribute($attribute)); + $this->assertSame($stored, $this->stored($database, $attribute), 'Only a declared single-valued integer stores the change rounded'); + } + + public function testAMissingCollectionIsNotFound(): void + { + $this->expectException(NotFoundException::class); + + $this->database()->increaseDocumentAttribute('missing', self::DOCUMENT, 'count', 1.5, 10.5); + } + + /** + * MariaDB and MySQL on database 7.4.1: the exact result is returned and the column stores it rounded half to + * even. + * + * @return iterable + */ + public static function fractionalChanges(): iterable + { + yield 'increase rounds a half down to even' => [true, 'count', 5, 1.5, null, 6.5, 6]; + yield 'increase rounds a half up to even' => [true, 'count', 6, 1.5, null, 7.5, 8]; + yield 'increase rounds below a half down' => [true, 'count', 5, 1.4, null, 6.4, 6]; + yield 'increase rounds above a half up' => [true, 'count', 5, 1.6, null, 6.6, 7]; + yield 'increase rounds a negative half to even' => [true, 'count', -10, 2.5, null, -7.5, -8]; + yield 'increase by a half that rounds back to the value' => [true, 'count', 4, 0.5, null, 4.5, 4]; + yield 'increase by a half that rounds up' => [true, 'count', 5, 0.5, null, 5.5, 6]; + yield 'increase within a fractional maximum' => [true, 'count', 5, 1.5, 6.5, 6.5, 6]; + yield 'increase within a whole maximum' => [true, 'count', 5, 1.5, 7, 6.5, 6]; + yield 'increase a big integer' => [true, 'big', 5, 2.5, null, 7.5, 8]; + yield 'decrease rounds a half up to even' => [false, 'count', 5, 1.5, null, 3.5, 4]; + yield 'decrease rounds a half down to even' => [false, 'count', 6, 1.5, null, 4.5, 4]; + yield 'decrease by less than a half keeps the value' => [false, 'count', 5, 0.4, null, 4.6, 5]; + yield 'decrease within a fractional minimum' => [false, 'count', 5, 1.5, 3.5, 3.5, 4]; + yield 'a whole float change returns a float' => [true, 'count', 8, 2.0, null, 10.0, 10]; + } + + #[DataProvider('fractionalChanges')] + public function testAFractionalChangeOnAnIntegerMatchesTheEngine(bool $increase, string $attribute, int $start, float $value, int|float|null $bound, float $returned, int $stored): void + { + $database = $this->database(); + $this->create($database, [$attribute => $start]); + + $document = $increase + ? $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, $attribute, $value, $bound) + : $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, $attribute, $value, $bound); + + $this->assertSame($returned, $document->getAttribute($attribute), 'The exact result of a fractional change is returned'); + $this->assertSame($stored, $this->stored($database, $attribute), 'The engine stored the change rounded half to even'); + } + + /** + * @return iterable + */ + public static function fractionalChangesPastTheirBound(): iterable + { + yield 'increase past the maximum' => [true, 6]; + yield 'decrease past the minimum' => [false, 4]; + } + + #[DataProvider('fractionalChangesPastTheirBound')] + public function testAFractionalChangePastItsBoundIsRefusedAndRolledBack(bool $increase, int $bound): void + { + $database = $this->database(); + $this->create($database, ['count' => 5]); + + try { + $increase + ? $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 1.5, $bound) + : $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 1.5, $bound); + $this->fail('A fractional change past its bound must be refused'); + } catch (LimitException $exception) { + $this->assertSame('Attribute value exceeds '.($increase ? 'maximum' : 'minimum').' limit: '.$bound, $exception->getMessage()); + } + + $this->assertSame(5, $this->stored($database, 'count')); + } + + public function testAFractionalChangeOnAMissingDocumentIsNotFound(): void + { + $this->expectException(NotFoundException::class); + + $this->database()->increaseDocumentAttribute(self::COLLECTION, 'missing', 'count', 1.5); + } + + /** + * @return iterable + */ + public static function nonPositiveChanges(): iterable + { + yield 'zero' => [0]; + yield 'negative' => [-1]; + yield 'negative fraction' => [-1.5]; + } + + #[DataProvider('nonPositiveChanges')] + public function testANonPositiveChangeIsATypeError(int|float $value): void + { + $database = $this->database(); + $this->create($database, ['count' => 5]); + + $this->assertRefused('Value must be numeric and greater than 0', fn () => $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', $value)); + $this->assertSame(5, $this->stored($database, 'count')); + } + + public function testAFractionalChangeOnADoubleIsPassedThrough(): void + { + $database = $this->database(); + $this->create($database, ['ratio' => 5.25]); + + $this->assertSame(6.75, $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'ratio', 1.5)->getAttribute('ratio')); + $this->assertSame(6.75, $this->stored($database, 'ratio')); + } + + public function testAFractionalMaximumOnAnIntegerBoundsTheIncrementAtItsWholePart(): void + { + $database = $this->database(); + $this->create($database, ['count' => 8]); + + $this->assertSame(10, $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 2, 10.5)->getAttribute('count')); + $this->assertLimited(fn () => $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 1, 10.5)); + $this->assertSame(10, $this->stored($database, 'count')); + } + + public function testAFractionalMinimumOnAnIntegerBoundsTheDecrementAtItsWholePart(): void + { + $database = $this->database(); + $this->create($database, ['count' => 10]); + + $this->assertSame(3, $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 7, 2.5)->getAttribute('count')); + $this->assertLimited(fn () => $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 1, 2.5)); + $this->assertSame(3, $this->stored($database, 'count')); + } + + public function testWithoutTheModeTheLibraryRefusesTheFractionalBoundItself(): void + { + $database = $this->database(round: false); + $this->create($database, ['count' => 8]); + + $this->assertRefused('Max must be an integer.', fn () => $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 1, 10.5)); + $this->assertSame(8, $this->stored($database, 'count')); + } + + /** + * @return iterable + */ + public static function fractionalChangesBeyondFloatPrecision(): iterable + { + yield 'increase rounds half to even' => [9007199254740993, 1.5, true, 9007199254740994]; + yield 'increase rounds up past the half' => [9007199254740993, 1.75, true, 9007199254740995]; + yield 'increase rounds down below the half' => [9007199254740993, 1.25, true, 9007199254740994]; + yield 'decrease rounds half to even' => [9007199254740995, 1.5, false, 9007199254740994]; + yield 'decrease rounds down past the half' => [9007199254740995, 1.75, false, 9007199254740993]; + yield 'decrease rounds up below the half' => [9007199254740995, 1.25, false, 9007199254740994]; + } + + #[DataProvider('fractionalChangesBeyondFloatPrecision')] + public function testAFractionalChangeOnABigIntegerBeyondFloatPrecisionStoresTheExactRoundedResult(int $before, float $change, bool $increase, int $expected): void + { + $database = $this->database(); + $this->create($database, ['big' => $before]); + + $increase + ? $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'big', $change) + : $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'big', $change); + + $this->assertSame($expected, $this->stored($database, 'big')); + } + + public function testAFractionalTextMaximumBeyondFloatPrecisionBoundsTheIncrementAtItsWholePart(): void + { + $database = $this->database(); + $this->create($database, ['big' => 9007199254740992]); + + $this->assertSame(9007199254740993, $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'big', 1, '9007199254740993.5')->getAttribute('big')); + $this->assertLimited(fn () => $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'big', 1, '9007199254740993.5')); + $this->assertSame(9007199254740993, $this->stored($database, 'big')); + } + + public function testAFractionalTextMinimumBeyondFloatPrecisionBoundsTheDecrementAtItsWholePart(): void + { + $database = $this->database(); + $this->create($database, ['big' => -9007199254740992]); + + $this->assertSame(-9007199254740993, $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'big', 1, '-9007199254740993.5')->getAttribute('big')); + $this->assertLimited(fn () => $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'big', 1, '-9007199254740993.5')); + $this->assertSame(-9007199254740993, $this->stored($database, 'big')); + } + + public function testAMaximumTextBeyondTheIntegerRangeAllowsTheIncrement(): void + { + $database = $this->database(); + $this->create($database, ['big' => 5]); + + $this->assertSame(6, $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'big', 1, '1e400')->getAttribute('big')); + } + + public function testAMaximumTextBelowTheIntegerRangeRefusesTheIncrement(): void + { + $database = $this->database(); + $this->create($database, ['big' => 5]); + + $this->assertLimited(fn () => $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'big', 1, '-1e400')); + $this->assertSame(5, $this->stored($database, 'big')); + } + + public function testAMinimumTextWithAHugeExponentAllowsTheDecrement(): void + { + $database = $this->database(); + $this->create($database, ['big' => 5]); + + $this->assertSame(-5, $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'big', 10, '-1e999999999')->getAttribute('big')); + } + + /** + * @return iterable + */ + public static function counters(): iterable + { + return FractionalResults::counters(); + } + + #[DataProvider('counters')] + public function testAFractionalCounterChangeStoresWhatTheEngineStored(FractionalEngine $engine, bool $increase, int $start, float $value, int|float|null $bound, ?int $stored): void + { + $database = $this->database(self::engine($engine)); + $this->create($database, ['big' => $start]); + $change = fn (): Document => $increase + ? $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'big', $value, $bound) + : $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'big', $value, $bound); + + if ($stored === null) { + $this->assertRefused('Change value must be an integer.', $change); + $this->assertSame($start, $this->stored($database, 'big')); + + return; + } + + $this->assertSame($increase ? $start + $value : $start - $value, $change()->getAttribute('big')); + $this->assertSame($stored, $this->stored($database, 'big')); + } + + /** + * PostgreSQL on 7.4.1 compared the result with a bound in PHP first, then bound the bound next to an integer + * column, which it read as an integer and refused when fractional. + * + * @return iterable + */ + public static function postgresBounds(): iterable + { + yield 'a whole change within a fractional maximum' => [true, 2, 10.5, 'Max must be an integer.']; + yield 'a whole change past a fractional maximum' => [true, 3, 10.5, null]; + yield 'a whole change within a fractional minimum' => [false, 2, 5.5, 'Min must be an integer.']; + yield 'a whole change past a fractional minimum' => [false, 3, 5.5, null]; + yield 'a fractional change past a whole maximum' => [true, 2.5, 10, null]; + yield 'a fractional change past a whole minimum' => [false, 2.5, 6, null]; + yield 'a fractional change within a fractional maximum' => [true, 1.5, 10.5, 'Change value must be an integer.']; + } + + #[DataProvider('postgresBounds')] + public function testPostgresRefusesAFractionalBoundAfterTheLimitCheck(bool $increase, int|float $value, int|float $bound, ?string $refusal): void + { + $database = $this->database(self::engine(FractionalEngine::PostgreSQL)); + $this->create($database, ['count' => 8]); + $change = fn (): Document => $increase + ? $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', $value, $bound) + : $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', $value, $bound); + + $refusal === null ? $this->assertLimited($change) : $this->assertRefused($refusal, $change); + $this->assertSame(8, $this->stored($database, 'count')); + } + + public function testPostgresStoresAWholeChangeUnderAWholeBound(): void + { + $database = $this->database(self::engine(FractionalEngine::PostgreSQL)); + $this->create($database, ['count' => 8]); + + $this->assertSame(10.0, $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 2.0, 10)->getAttribute('count')); + $this->assertSame(7, $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 3, 7)->getAttribute('count')); + $this->assertSame(7, $this->stored($database, 'count')); + } + + public function testAChangeThatRoundsBackToTheValueStillWritesAndFiresItsEvents(): void + { + $database = $this->database(); + $this->create($database, ['count' => 4]); + $before = $database->getDocument(self::COLLECTION, self::DOCUMENT)->getUpdatedAt(); + $recorder = new class () implements Lifecycle { + /** @var list */ + public array $events = []; + + #[\Override] + public function handle(Domain $event): void + { + $this->events[] = $event->event; + } + }; + $database->addHook($recorder); + + $document = $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 0.5); + + $this->assertSame([Event::DocumentPurge, Event::DocumentIncrease], $recorder->events); + $this->assertSame(4.5, $document->getAttribute('count')); + $this->assertSame(4, $this->stored($database, 'count')); + $this->assertNotSame($before, $database->getDocument(self::COLLECTION, self::DOCUMENT)->getUpdatedAt(), 'The write still bumps $updatedAt'); + } + + /** + * @return iterable, int|null}> + */ + public static function operators(): iterable + { + return FractionalResults::operators(); + } + + /** + * @param list $values + */ + #[DataProvider('operators')] + public function testAFractionalOperatorOnUpdateStoresWhatTheEngineStored(FractionalEngine $engine, int $stored, OperatorType $method, array $values, ?int $expected): void + { + $database = $this->database(self::engine($engine)); + $this->create($database, ['count' => $stored]); + $update = fn (): Document => $database->updateDocument(self::COLLECTION, self::DOCUMENT, new Document(['count' => new Operator($method, 'count', $values)])); + + if ($expected === null) { + $this->assertStructureRefused($update); + $this->assertSame($stored, $this->stored($database, 'count')); + + return; + } + + $this->assertSame($expected, $update()->getAttribute('count')); + $this->assertSame($expected, $this->stored($database, 'count')); + } + + public function testFractionalItemsAddedToAnIntegerArrayAreCutTowardZero(): void + { + $database = $this->database(); + $this->create($database, ['scores' => [1]]); + + $database->updateDocument(self::COLLECTION, self::DOCUMENT, new Document(['scores' => Operator::arrayAppend([2.7, -3.5])])); + $database->updateDocument(self::COLLECTION, self::DOCUMENT, new Document(['scores' => Operator::arrayPrepend(['0.9'])])); + $database->updateDocument(self::COLLECTION, self::DOCUMENT, new Document(['scores' => Operator::arrayInsert(1, 4.2)])); + + $this->assertSame([0, 4, 1, 2, -3], $this->stored($database, 'scores')); + } + + public function testABulkUpdateRoundsEachDocumentFromItsOwnStoredValue(): void + { + $database = $this->database(); + $this->create($database, ['count' => 10, 'scores' => []], 'first'); + $this->create($database, ['count' => 11, 'scores' => []], 'second'); + $this->create($database, ['count' => 12, 'scores' => []], 'third'); + $updated = []; + + $count = $database->updateDocuments( + self::COLLECTION, + new Document(['count' => Operator::increment(1.5), 'scores' => Operator::arrayAppend([7.9])]), + onNext: function (Document $document) use (&$updated): void { + $updated[$document->getId()] = $document->getAttribute('count'); + }, + ); + + $this->assertSame(3, $count); + $this->assertSame(['first' => 12, 'second' => 12, 'third' => 14], $updated); + $this->assertSame(12, $this->stored($database, 'count', 'first')); + $this->assertSame(12, $this->stored($database, 'count', 'second')); + $this->assertSame(14, $this->stored($database, 'count', 'third')); + $this->assertSame([7], $this->stored($database, 'scores', 'third')); + } + + public function testAnUpsertAcrossTenantsRoundsEachDocumentFromItsOwnTenantsValue(): void + { + $authorization = new Authorization(); + $authorization->addRole(Role::any()->toString()); + $database = (new Database(self::sqlite(), new Cache(new None()))) + ->setAuthorization($authorization) + ->setDatabase('round_fractions') + ->setNamespace('round_fractions_'.\uniqid()) + ->setSharedTables(true) + ->setTenant(null) + ->setTenantPerDocument(true) + ->setRoundFractions(true); + $database->create(); + $database->createCollection(Collection::create( + id: self::COLLECTION, + attributes: [Attribute::integer(key: 'count')], + permissions: [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], + documentSecurity: false, + )); + foreach ([5 => 10, 6 => 11] as $tenant => $count) { + $database->createDocument(self::COLLECTION, new Document([Document::ID => self::DOCUMENT, Document::TENANT => $tenant, 'count' => $count])); + } + + $database->upsertDocuments(self::COLLECTION, [ + new Document([Document::ID => self::DOCUMENT, Document::TENANT => 5, 'count' => Operator::increment(1.5)]), + new Document([Document::ID => self::DOCUMENT, Document::TENANT => 6, 'count' => Operator::increment(1.5)]), + ]); + + $this->assertSame(12, $database->withTenant(5, fn (): mixed => $this->stored($database, 'count'))); + $this->assertSame(12, $database->withTenant(6, fn (): mixed => $this->stored($database, 'count'))); + } + + public function testABulkUpdateWithAFractionalLimitHoldsEachDocumentAtItsLimit(): void + { + $database = $this->database(); + $this->create($database, ['count' => 10], 'first'); + $this->create($database, ['count' => 12], 'second'); + + $database->updateDocuments(self::COLLECTION, new Document(['count' => Operator::increment(2, 13.9)]), [Query::orderAsc('count')]); + + $this->assertSame(12, $this->stored($database, 'count', 'first')); + $this->assertSame(12, $this->stored($database, 'count', 'second')); + } + + public function testABulkUpdateTheEngineRefusedWritesNothing(): void + { + $database = $this->database(self::engine(FractionalEngine::PostgreSQL)); + $this->create($database, ['count' => 10], 'first'); + $this->create($database, ['count' => 11], 'second'); + + $this->assertStructureRefused(fn () => $database->updateDocuments(self::COLLECTION, new Document(['count' => Operator::increment(1.5)]))); + + $this->assertSame(10, $this->stored($database, 'count', 'first')); + $this->assertSame(11, $this->stored($database, 'count', 'second')); + } + + public function testAnUpsertRoundsAStoredDocumentFromItsValueAndANewOneFromZero(): void + { + $database = $this->database(self::sqlite()); + $this->create($database, ['count' => 10], 'first'); + $this->create($database, ['count' => 11], 'second'); + + $upserted = $database->upsertDocument(self::COLLECTION, new Document(['$id' => 'first', 'count' => Operator::increment(1.5)])); + $count = $database->upsertDocuments(self::COLLECTION, [ + new Document(['$id' => 'second', 'count' => Operator::increment(1.5)]), + new Document(['$id' => 'third', 'count' => Operator::increment(2.5), 'scores' => Operator::arrayAppend([3.9])]), + ]); + + $this->assertSame(2, $count); + $this->assertSame(12, $upserted->getAttribute('count')); + $this->assertSame(12, $this->stored($database, 'count', 'first')); + $this->assertSame(12, $this->stored($database, 'count', 'second')); + $this->assertSame(2, $this->stored($database, 'count', 'third')); + } + + public function testAnUpsertTheEngineRefusedWritesNothing(): void + { + $database = $this->database(self::sqlite(self::arithmetic(FractionalEngine::PostgreSQL))); + $this->create($database, ['count' => 10], 'first'); + + $this->assertStructureRefused(fn () => $database->upsertDocuments(self::COLLECTION, [ + new Document(['$id' => 'second', 'count' => 4]), + new Document(['$id' => 'first', 'count' => Operator::increment(1.5)]), + ])); + + $this->assertSame(10, $this->stored($database, 'count', 'first')); + $this->assertTrue($database->getDocument(self::COLLECTION, 'second')->isEmpty()); + } + + public function testWithoutTheModeEveryFractionIsRefusedAsBefore(): void + { + $database = $this->database(self::sqlite(), round: false); + $this->create($database, ['count' => 10, 'scores' => []]); + + $this->assertRefused('Change value must be an integer.', fn () => $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 1.5)); + $this->assertRefused('Min must be an integer.', fn () => $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 1, 2.5)); + $this->assertRefused('Change value must be an integer.', fn () => $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 2.0)); + $this->assertStructureRefused(fn () => $database->updateDocument(self::COLLECTION, self::DOCUMENT, new Document(['count' => Operator::increment(1.5)]))); + $this->assertStructureRefused(fn () => $database->updateDocument(self::COLLECTION, self::DOCUMENT, new Document(['count' => Operator::increment(1, 10.5)]))); + $this->assertStructureRefused(fn () => $database->updateDocument(self::COLLECTION, self::DOCUMENT, new Document(['scores' => Operator::arrayAppend([1.5])]))); + $this->assertStructureRefused(fn () => $database->updateDocuments(self::COLLECTION, new Document(['count' => Operator::multiply(1.5)]))); + $this->assertStructureRefused(fn () => $database->upsertDocuments(self::COLLECTION, [new Document(['$id' => self::DOCUMENT, 'count' => Operator::increment(1.5)])])); + + $this->assertSame(10, $this->stored($database, 'count')); + $this->assertSame([], $this->stored($database, 'scores')); + } + + public function testTheModeIsScopedAndOffByDefault(): void + { + $database = $this->database(round: false); + + $this->assertFalse($database->isRoundingFractions()); + $this->assertTrue($database->withRoundFractions(true, $database->isRoundingFractions(...))); + $this->assertFalse($database->isRoundingFractions()); + $this->assertTrue($database->setRoundFractions(true)->isRoundingFractions()); + $this->assertFalse($database->withRoundFractions(false, $database->isRoundingFractions(...))); + } + + public function testAMirrorSetsTheModeOnItsSourceAndDestination(): void + { + $source = new Database(new Memory(), new Cache(new None())); + $destination = new Database(new Memory(), new Cache(new None())); + $mirror = new Mirror($source, $destination); + + $this->assertSame([true, true, true], $mirror->withRoundFractions(true, fn (): array => [ + $mirror->isRoundingFractions(), + $source->isRoundingFractions(), + $destination->isRoundingFractions(), + ])); + $this->assertFalse($source->isRoundingFractions()); + + $mirror->setRoundFractions(true); + + $this->assertTrue($mirror->isRoundingFractions()); + $this->assertTrue($source->isRoundingFractions()); + $this->assertTrue($destination->isRoundingFractions()); + } + + public function testASnapshotCarriesTheModeIntoAChildCoroutine(): void + { + if (! \extension_loaded('swoole')) { + $this->markTestSkipped('ext-swoole is required for coroutine-scoped toggles'); + } + + $database = $this->database(round: false); + $this->create($database, ['count' => 5]); + $outcomes = null; + $hookFlags = Runtime::getHookFlags(); + + try { + run(function () use ($database, &$outcomes): void { + $released = new Channel(1); + $results = new Channel(1); + + $database->withRoundFractions(true, function () use ($database, $released, $results): void { + $snapshot = $database->snapshot(); + + Coroutine::create(function () use ($database, $snapshot, $released, $results): void { + $released->pop(); + $results->push([ + 'unscoped' => $this->refusal(fn () => $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 1.5)), + 'scoped' => $database->withSnapshot($snapshot, fn (): ?string => $this->refusal(fn () => $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', 1.5))), + ]); + }); + }); + + $released->push(true); + $outcomes = $results->pop(5); + }); + } finally { + Runtime::setHookFlags($hookFlags); + } + + $this->assertSame(['unscoped' => 'Change value must be an integer.', 'scoped' => null], $outcomes); + $this->assertSame(6, $this->stored($database, 'count'), 'Only the change under the snapshot rounded 5 + 1.5'); + $this->assertFalse($database->isRoundingFractions()); + } + + private function database(?Adapter $adapter = null, bool $round = true): Database + { + $authorization = new Authorization(); + $authorization->addRole(Role::any()->toString()); + + $database = (new Database($adapter ?? new Memory(), new Cache(new None()))) + ->setAuthorization($authorization) + ->setDatabase('round_fractions') + ->setNamespace('round_fractions_'.\uniqid()); + $database->create(); + $database->createCollection(Collection::create( + id: self::COLLECTION, + attributes: [ + Attribute::integer(key: 'count'), + Attribute::bigInteger(key: 'big'), + Attribute::double(key: 'ratio'), + Attribute::integer(key: 'scores', array: true), + ], + permissions: [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + ], + documentSecurity: false, + )); + + return $database->setRoundFractions($round); + } + + private static function engine(FractionalEngine $engine): Memory + { + return new class (self::arithmetic($engine)) extends Memory { + public function __construct(private readonly Arithmetic $declared) + { + parent::__construct(); + } + + #[\Override] + public function arithmetic(): Arithmetic + { + return $this->declared; + } + }; + } + + private static function sqlite(?Arithmetic $arithmetic = null): SQLite + { + return new class (new PDO('sqlite::memory:'), $arithmetic ?? self::arithmetic(FractionalEngine::MariaDB)) extends SQLite { + public function __construct(PDO $pdo, private readonly Arithmetic $declared) + { + parent::__construct($pdo); + } + + #[\Override] + public function arithmetic(): Arithmetic + { + return $this->declared; + } + }; + } + + private static function arithmetic(FractionalEngine $engine): Arithmetic + { + return match ($engine) { + FractionalEngine::MariaDB => (new Memory())->arithmetic(), + FractionalEngine::PostgreSQL => (new Postgres(new stdClass()))->arithmetic(), + FractionalEngine::MongoDB => (new Mongo(new class () extends Client { + public function __construct() + { + } + + #[\Override] + public function connect(): self + { + return $this; + } + + #[\Override] + public function close(): void + { + } + }))->arithmetic(), + }; + } + + /** + * @param array $values + */ + private function create(Database $database, array $values, string $id = self::DOCUMENT): void + { + $database->createDocument(self::COLLECTION, new Document([Document::ID => $id, ...$values])); + } + + private function stored(Database $database, string $attribute, string $id = self::DOCUMENT): mixed + { + return $database->getDocument(self::COLLECTION, $id)->getAttribute($attribute); + } + + private function assertHeldAtMaximum(Database $database, int|float|string $bound, int|string $maximum): void + { + $increase = fn (): Document => $database->increaseDocumentAttribute(self::COLLECTION, 'maximum', 'big', 1, $bound); + + if (BigInt::compare($maximum, \PHP_INT_MAX) > 0) { + $this->create($database, ['big' => 5], 'maximum'); + $this->assertSame(6, $increase()->getAttribute('big')); + + return; + } + + if (BigInt::compare($maximum, \PHP_INT_MIN) <= 0) { + $this->create($database, ['big' => 5], 'maximum'); + $this->assertLimited($increase); + $this->assertSame(5, $this->stored($database, 'big', 'maximum')); + + return; + } + + $this->create($database, ['big' => BigInt::subtract($maximum, 1)], 'maximum'); + $this->assertSame(BigInt::toNative($maximum), $increase()->getAttribute('big')); + $this->assertLimited($increase); + $this->assertSame(BigInt::toNative($maximum), $this->stored($database, 'big', 'maximum')); + } + + private function assertHeldAtMinimum(Database $database, int|float|string $bound, int|string $minimum): void + { + $decrease = fn (): Document => $database->decreaseDocumentAttribute(self::COLLECTION, 'minimum', 'big', 1, $bound); + + if (BigInt::compare($minimum, \PHP_INT_MIN) < 0) { + $this->create($database, ['big' => 5], 'minimum'); + $this->assertSame(4, $decrease()->getAttribute('big')); + + return; + } + + if (BigInt::compare($minimum, \PHP_INT_MAX) >= 0) { + $this->create($database, ['big' => 5], 'minimum'); + $this->assertLimited($decrease); + $this->assertSame(5, $this->stored($database, 'big', 'minimum')); + + return; + } + + $this->create($database, ['big' => BigInt::add($minimum, 1)], 'minimum'); + $this->assertSame(BigInt::toNative($minimum), $decrease()->getAttribute('big')); + $this->assertLimited($decrease); + $this->assertSame(BigInt::toNative($minimum), $this->stored($database, 'big', 'minimum')); + } + + /** + * @param callable(): mixed $call + */ + private function assertLimited(callable $call): void + { + try { + $call(); + $this->fail('A change past its bound must be refused'); + } catch (LimitException) { + $this->addToAssertionCount(1); + } + } + + /** + * @param callable(): mixed $call + */ + private function assertRefused(string $message, callable $call): void + { + $this->assertSame($message, $this->refusal($call)); + } + + /** + * @param callable(): mixed $call + */ + private function assertStructureRefused(callable $call): void + { + try { + $call(); + $this->fail('The write must be refused as an invalid structure'); + } catch (StructureException) { + $this->addToAssertionCount(1); + } + } + + /** + * @param callable(): mixed $call + */ + private function refusal(callable $call): ?string + { + try { + $call(); + } catch (TypeException $exception) { + return $exception->getMessage(); + } + + return null; + } +} From 3683419771c98367a16b821467d9914d6d8dfe68 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:16:53 +1300 Subject: [PATCH 17/55] test(documents): cover the fractional-integer mode on every adapter Runs the engine family's 68 operator rows and 21 counter rows from the shared 7.4.1 fixture through each e2e adapter with the mode on, in one collection per test, plus a bulk update across differing stored values, an upsert of a stored and a new row, a bigint operator past 2^53 and the refusals without the mode. The family comes from fractionalEngine(), MariaDB in Base and overridden by the PostgreSQL and MongoDB classes. Co-Authored-By: Claude Opus 5.5 --- tests/e2e/Adapter/Base.php | 11 + tests/e2e/Adapter/MongoDBTest.php | 7 + tests/e2e/Adapter/PostgresTest.php | 7 + tests/e2e/Adapter/Schemaless/MongoDBTest.php | 7 + .../Adapter/Scopes/FractionalIntegerTests.php | 260 ++++++++++++++++++ .../e2e/Adapter/SharedTables/MongoDBTest.php | 7 + .../e2e/Adapter/SharedTables/PostgresTest.php | 7 + 7 files changed, 306 insertions(+) create mode 100644 tests/e2e/Adapter/Scopes/FractionalIntegerTests.php diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 429c3a9a4..747c5ac84 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -10,6 +10,7 @@ use Tests\E2E\Adapter\Scopes\CustomDocumentTypeTests; use Tests\E2E\Adapter\Scopes\DatabaseTests; use Tests\E2E\Adapter\Scopes\DocumentTests; +use Tests\E2E\Adapter\Scopes\FractionalIntegerTests; use Tests\E2E\Adapter\Scopes\GeneralTests; use Tests\E2E\Adapter\Scopes\IndexTests; use Tests\E2E\Adapter\Scopes\JoinComboTests; @@ -24,6 +25,7 @@ use Tests\E2E\Adapter\Scopes\SchemaReconciliationTests; use Tests\E2E\Adapter\Scopes\SpatialTests; use Tests\E2E\Adapter\Scopes\VectorTests; +use Tests\Unit\Support\FractionalEngine; use Utopia\Database\Adapter; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Adapter\Memory; @@ -49,6 +51,7 @@ abstract class Base extends TestCase use CustomDocumentTypeTests; use DatabaseTests; use DocumentTests; + use FractionalIntegerTests; use GeneralTests; use IndexTests; use JoinComboTests; @@ -116,6 +119,14 @@ protected static function getTestToken(): string return getenv('TEST_TOKEN') ?: getenv('UNIQUE_TEST_TOKEN') ?: (string) getmypid(); } + /** + * The engine family whose database 7.4.1 arithmetic the adapter under test reproduces. + */ + protected static function fractionalEngine(): FractionalEngine + { + return FractionalEngine::MariaDB; + } + /** * @param list> $rows * @return list diff --git a/tests/e2e/Adapter/MongoDBTest.php b/tests/e2e/Adapter/MongoDBTest.php index 15886f82d..09f5b0a97 100644 --- a/tests/e2e/Adapter/MongoDBTest.php +++ b/tests/e2e/Adapter/MongoDBTest.php @@ -5,6 +5,7 @@ use Exception; use Redis; use Tests\E2E\Adapter\Scopes\MongoReadFilterTests; +use Tests\Unit\Support\FractionalEngine; use Utopia\Cache\Adapter\Redis as RedisAdapter; use Utopia\Cache\Cache; use Utopia\Database\Adapter\Mongo; @@ -384,4 +385,10 @@ public function testReadsDropAStoredNonStringPermission(): void $this->assertSame('single', $updated->getAttribute('title')); $this->assertSame($permissions, $updated->getPermissions()); } + + #[\Override] + protected static function fractionalEngine(): FractionalEngine + { + return FractionalEngine::MongoDB; + } } diff --git a/tests/e2e/Adapter/PostgresTest.php b/tests/e2e/Adapter/PostgresTest.php index 7cb728001..169c1b198 100644 --- a/tests/e2e/Adapter/PostgresTest.php +++ b/tests/e2e/Adapter/PostgresTest.php @@ -3,6 +3,7 @@ namespace Tests\E2E\Adapter; use Redis; +use Tests\Unit\Support\FractionalEngine; use Utopia\Cache\Adapter\Redis as RedisAdapter; use Utopia\Cache\Cache; use Utopia\Database\Adapter\Postgres; @@ -242,4 +243,10 @@ public function testVectorSearchUsesTheIndex(): void $database->deleteCollection('vectorPlan'); } + + #[\Override] + protected static function fractionalEngine(): FractionalEngine + { + return FractionalEngine::PostgreSQL; + } } diff --git a/tests/e2e/Adapter/Schemaless/MongoDBTest.php b/tests/e2e/Adapter/Schemaless/MongoDBTest.php index a48d59fbd..5dc78ebfa 100644 --- a/tests/e2e/Adapter/Schemaless/MongoDBTest.php +++ b/tests/e2e/Adapter/Schemaless/MongoDBTest.php @@ -6,6 +6,7 @@ use Redis; use Tests\E2E\Adapter\Base; use Tests\E2E\Adapter\Scopes\MongoReadFilterTests; +use Tests\Unit\Support\FractionalEngine; use Utopia\Cache\Adapter\Redis as RedisAdapter; use Utopia\Cache\Cache; use Utopia\Database\Adapter\Mongo; @@ -100,4 +101,10 @@ protected function deleteIndex(string $collection, string $index): bool { return true; } + + #[\Override] + protected static function fractionalEngine(): FractionalEngine + { + return FractionalEngine::MongoDB; + } } diff --git a/tests/e2e/Adapter/Scopes/FractionalIntegerTests.php b/tests/e2e/Adapter/Scopes/FractionalIntegerTests.php new file mode 100644 index 000000000..689d9637c --- /dev/null +++ b/tests/e2e/Adapter/Scopes/FractionalIntegerTests.php @@ -0,0 +1,260 @@ +getDatabase(); + + if (! $database->getAdapter()->supports(Capability::Operators)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $collection = $this->fractionalCollection($database, 'fractional_operators'); + $engine = static::fractionalEngine(); + $row = 0; + + try { + $database->withRoundFractions(true, function () use ($database, $collection, $engine, &$row): void { + foreach (FractionalResults::operators() as $name => [$family, $stored, $method, $values, $expected]) { + if ($family !== $engine) { + continue; + } + + $id = 'operator'.$row++; + $database->createDocument($collection, new Document([Document::ID => $id, 'count' => $stored])); + $update = fn (): Document => $database->updateDocument($collection, $id, new Document(['count' => new Operator($method, 'count', $values)])); + + if ($expected === null) { + try { + $update(); + $this->fail("{$name}: the engine refused it, so it must be refused"); + } catch (StructureException) { + } + + $this->assertSame($stored, $database->getDocument($collection, $id)->getAttribute('count'), $name); + + continue; + } + + $this->assertSame($expected, $update()->getAttribute('count'), $name); + $this->assertSame($expected, $database->getDocument($collection, $id)->getAttribute('count'), $name); + } + }); + + $this->assertSame(68, $row, 'Every operator row of the engine family ran'); + } finally { + $database->deleteCollection($collection); + } + } + + public function testFractionalCountersStoreWhatTheEngineStored(): void + { + $database = $this->getDatabase(); + $collection = $this->fractionalCollection($database, 'fractional_counters'); + $engine = static::fractionalEngine(); + $row = 0; + + try { + $database->withRoundFractions(true, function () use ($database, $collection, $engine, &$row): void { + foreach (FractionalResults::counters() as $name => [$family, $increase, $start, $value, $bound, $stored]) { + if ($family !== $engine) { + continue; + } + + $id = 'counter'.$row++; + $database->createDocument($collection, new Document([Document::ID => $id, 'big' => $start])); + $change = fn (): Document => $increase + ? $database->increaseDocumentAttribute($collection, $id, 'big', $value, $bound) + : $database->decreaseDocumentAttribute($collection, $id, 'big', $value, $bound); + + if ($stored === null) { + try { + $change(); + $this->fail("{$name}: the engine refused it, so it must be refused"); + } catch (TypeException $exception) { + $this->assertSame('Change value must be an integer.', $exception->getMessage(), $name); + } + + $this->assertSame($start, $database->getDocument($collection, $id)->getAttribute('big'), $name); + + continue; + } + + $this->assertSame($increase ? $start + $value : $start - $value, $change()->getAttribute('big'), $name); + $this->assertSame($stored, $database->getDocument($collection, $id)->getAttribute('big'), $name); + } + }); + + $this->assertSame(21, $row, 'Every counter row of the engine family ran'); + } finally { + $database->deleteCollection($collection); + } + } + + public function testFractionalOperatorOnABigIntegerBeyondFloatPrecisionStaysExact(): void + { + $database = $this->getDatabase(); + + if (! $database->getAdapter()->supports(Capability::Operators)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $collection = $this->fractionalCollection($database, 'fractional_big'); + $expected = match (static::fractionalEngine()) { + FractionalEngine::MariaDB, FractionalEngine::MongoDB => 9007199254740994, + FractionalEngine::PostgreSQL => 9007199254740995, + }; + + try { + $database->createDocument($collection, new Document([Document::ID => 'huge', 'big' => 9007199254740993])); + + $database->withRoundFractions(true, fn (): Document => $database->updateDocument($collection, 'huge', new Document([ + 'big' => Operator::increment(1.5, 9007199254741000), + ]))); + + $this->assertSame($expected, $database->getDocument($collection, 'huge')->getAttribute('big')); + } finally { + $database->deleteCollection($collection); + } + } + + public function testFractionalBulkUpdateRoundsEachDocumentFromItsOwnStoredValue(): void + { + $database = $this->getDatabase(); + + if (! $database->getAdapter()->supports(Capability::Operators)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $collection = $this->fractionalCollection($database, 'fractional_bulk'); + $expected = match (static::fractionalEngine()) { + FractionalEngine::MariaDB => ['first' => 12, 'second' => 12, 'third' => 14], + FractionalEngine::PostgreSQL => ['first' => 12, 'second' => 13, 'third' => 14], + FractionalEngine::MongoDB => ['first' => 11, 'second' => 12, 'third' => 13], + }; + + try { + foreach (['first' => 10, 'second' => 11, 'third' => 12] as $id => $count) { + $database->createDocument($collection, new Document([Document::ID => $id, 'count' => $count])); + } + + $updated = $database->withRoundFractions(true, fn (): int => $database->updateDocuments( + $collection, + new Document(['count' => Operator::increment(1.5, 100)]), + )); + + $stored = []; + foreach (\array_keys($expected) as $id) { + $stored[$id] = $database->getDocument($collection, $id)->getAttribute('count'); + } + + $this->assertSame(3, $updated); + $this->assertSame($expected, $stored); + } finally { + $database->deleteCollection($collection); + } + } + + public function testFractionalUpsertRoundsAStoredDocumentFromItsValueAndANewOneFromZero(): void + { + $database = $this->getDatabase(); + + if (! $database->getAdapter()->supports(Capability::Operators) || ! $database->getAdapter()->hasFeature(Feature\Upserts::class)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $collection = $this->fractionalCollection($database, 'fractional_upsert'); + $expected = match (static::fractionalEngine()) { + FractionalEngine::MariaDB, FractionalEngine::MongoDB => ['first' => 12, 'second' => 2], + FractionalEngine::PostgreSQL => ['first' => 13, 'second' => 3], + }; + + try { + $database->createDocument($collection, new Document([Document::ID => 'first', 'count' => 10])); + + $database->withRoundFractions(true, fn (): int => $database->upsertDocuments($collection, [ + new Document([Document::ID => 'first', 'count' => Operator::increment(2.5, 100)]), + new Document([Document::ID => 'second', 'count' => Operator::increment(2.5, 100)]), + ])); + + $this->assertSame($expected['first'], $database->getDocument($collection, 'first')->getAttribute('count')); + $this->assertSame($expected['second'], $database->getDocument($collection, 'second')->getAttribute('count')); + } finally { + $database->deleteCollection($collection); + } + } + + public function testFractionsOnIntegersAreRefusedWithoutTheMode(): void + { + $database = $this->getDatabase(); + $collection = $this->fractionalCollection($database, 'fractional_refused'); + + try { + $database->createDocument($collection, new Document([Document::ID => 'meter', 'count' => 10])); + + try { + $database->increaseDocumentAttribute($collection, 'meter', 'count', 1.5); + $this->fail('A fractional change on an integer must be refused without the mode'); + } catch (TypeException $exception) { + $this->assertSame('Change value must be an integer.', $exception->getMessage()); + } + + if ($database->getAdapter()->supports(Capability::Operators)) { + try { + $database->updateDocument($collection, 'meter', new Document(['count' => Operator::increment(1.5, 100)])); + $this->fail('A fractional operator on an integer must be refused without the mode'); + } catch (StructureException) { + } + } + + $this->assertFalse($database->isRoundingFractions()); + $this->assertSame(10, $database->getDocument($collection, 'meter')->getAttribute('count')); + } finally { + $database->deleteCollection($collection); + } + } + + private function fractionalCollection(Database $database, string $id): string + { + $database->createCollection(Collection::create( + id: $id, + attributes: [ + Attribute::integer(key: 'count'), + Attribute::bigInteger(key: 'big'), + ], + permissions: [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + ], + documentSecurity: false, + )); + + return $id; + } +} diff --git a/tests/e2e/Adapter/SharedTables/MongoDBTest.php b/tests/e2e/Adapter/SharedTables/MongoDBTest.php index 5cc98d193..1f4e310ea 100644 --- a/tests/e2e/Adapter/SharedTables/MongoDBTest.php +++ b/tests/e2e/Adapter/SharedTables/MongoDBTest.php @@ -6,6 +6,7 @@ use Redis; use Tests\E2E\Adapter\Base; use Tests\E2E\Adapter\Scopes\MongoReadFilterTests; +use Tests\Unit\Support\FractionalEngine; use Utopia\Cache\Adapter\Redis as RedisAdapter; use Utopia\Cache\Cache; use Utopia\Database\Adapter\Mongo; @@ -205,4 +206,10 @@ protected function deleteIndex(string $collection, string $index): bool { return true; } + + #[\Override] + protected static function fractionalEngine(): FractionalEngine + { + return FractionalEngine::MongoDB; + } } diff --git a/tests/e2e/Adapter/SharedTables/PostgresTest.php b/tests/e2e/Adapter/SharedTables/PostgresTest.php index f58924ebc..254b62742 100644 --- a/tests/e2e/Adapter/SharedTables/PostgresTest.php +++ b/tests/e2e/Adapter/SharedTables/PostgresTest.php @@ -4,6 +4,7 @@ use Redis; use Tests\E2E\Adapter\Base; +use Tests\Unit\Support\FractionalEngine; use Utopia\Cache\Adapter\Redis as RedisAdapter; use Utopia\Cache\Cache; use Utopia\Database\Adapter\Postgres; @@ -92,4 +93,10 @@ protected function deleteIndex(string $collection, string $index): bool return true; } + + #[\Override] + protected static function fractionalEngine(): FractionalEngine + { + return FractionalEngine::PostgreSQL; + } } From 2b2528b3439ba223d25b939017304ec6fc4c4db3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:19:13 +1300 Subject: [PATCH 18/55] test(joins): expect a one-to-many parent-side join to read its children 8.1.0 rewrites a join on the parent side of a one-to-many relationship to compare the parent's $id with the two-way key its children store, so the join is no longer refused as virtual. The child side of a one-to-one stays refused. Co-Authored-By: Claude Opus 5.5 --- tests/unit/Joins/JoinInternalColumnsTest.php | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/unit/Joins/JoinInternalColumnsTest.php b/tests/unit/Joins/JoinInternalColumnsTest.php index 7cf423df5..f69bee478 100644 --- a/tests/unit/Joins/JoinInternalColumnsTest.php +++ b/tests/unit/Joins/JoinInternalColumnsTest.php @@ -237,10 +237,17 @@ public function testJoinConditionOverARelationshipNamesItsColumn(): void $this->assertSame([['rows' => 2]], $this->database->aggregate('persons', [Query::join('books', 'book', [Query::on('$id', 'owner')]), Query::count('*', 'rows')]), 'the child side of a one-to-many relationship holds a column'); $this->assertSame(2, $this->database->count('books', [Query::join('persons', 'person', [Query::on('owner', '$id')])])); - $this->assertInvalidQuery( - 'Invalid query: Cannot join on virtual relationship attribute: books', - fn (): mixed => $this->database->find('persons', [Query::join('books', 'book', [Query::on('books', '$id')])]), + $owned = $this->database->find('persons', [ + Query::join('books', 'book', [Query::on('books', '$id')]), + Query::select(['name', 'book.title']), + Query::orderAsc('book.title'), + ]); + $this->assertSame( + [['Ada', 'One'], ['Ada', 'Two']], + \array_map(static fn (Document $person): array => [$person->getAttribute('name'), $person->getAttribute('book.title')], $owned), + 'the parent side of a one-to-many relationship joins on the two-way key its children store', ); + $this->assertInvalidQuery( 'Invalid query: Cannot join on virtual relationship attribute: person', fn (): mixed => $this->database->find('persons', [Query::join('libraries', 'lib', [Query::on('$id', 'person')])]), From ac616c698079607799a483767c24d7a2c3ae00a3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:21:01 +1300 Subject: [PATCH 19/55] test(joins): narrow joined and aggregated values before comparing them Aggregate rows and joined attributes are mixed, and some adapters return numbers as strings, so they are read as integers only once checked numeric. The scoped-toggle test reads both toggles through one closure so the default and the scoped values are not inferred from earlier assertions. Co-Authored-By: Claude Opus 5.5 --- .../Adapter/Scopes/JoinRelationshipTests.php | 19 ++++++++++++------- tests/unit/Joins/RelationshipJoinTest.php | 7 ++++++- .../Relationships/NestedPreparationTest.php | 13 +++++-------- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/JoinRelationshipTests.php b/tests/e2e/Adapter/Scopes/JoinRelationshipTests.php index d13f54cfe..f3c6e9b0f 100644 --- a/tests/e2e/Adapter/Scopes/JoinRelationshipTests.php +++ b/tests/e2e/Adapter/Scopes/JoinRelationshipTests.php @@ -78,7 +78,7 @@ public function testJoinRelationshipOnAOneToManyAttributeMatchesTheRelatedKey(): Query::select(['name', 'purchase.amount']), Query::orderAsc('purchase.amount'), ]); - $this->assertSame([10, 20, 30, 40], \array_map(static fn (Document $row): int => (int) $row->getAttribute('purchase.amount'), $rows)); + $this->assertSame([10, 20, 30, 40], \array_map(static fn (Document $row): ?int => self::joinRelationshipInteger($row->getAttribute('purchase.amount')), $rows)); $this->assertSame(['Ada', 'Ada', 'Bob', 'Bob'], \array_map(static fn (Document $row): mixed => $row->getAttribute('name'), $rows)); } finally { $this->deleteJoinRelationshipCollections($database, ...self::JOIN_RELATIONSHIP_COLLECTIONS); @@ -130,16 +130,16 @@ public function testJoinRelationshipIsMatchedByEveryRead(): void $this->assertSame( [['Ada', 30], ['Bob', 70]], \array_map( - static fn (array $row): array => [$row['name'], (int) $row['total']], + static fn (array $row): array => [$row['name'], self::joinRelationshipInteger($row['total'])], $database->aggregate(self::JOIN_RELATIONSHIP_CUSTOMERS, [$join, Query::sum('purchase.amount', 'total'), Query::groupBy(['name']), Query::orderAsc('name')]), ), ); $this->assertSame( 20, - (int) $database->getDocument(self::JOIN_RELATIONSHIP_CUSTOMERS, 'c1', [ + self::joinRelationshipInteger($database->getDocument(self::JOIN_RELATIONSHIP_CUSTOMERS, 'c1', [ Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id'), Query::equal('purchase.status', ['open'])]), Query::select(['name', 'purchase.amount']), - ])->getAttribute('purchase.amount'), + ])->getAttribute('purchase.amount')), ); } finally { $this->deleteJoinRelationshipCollections($database, ...self::JOIN_RELATIONSHIP_COLLECTIONS); @@ -208,7 +208,7 @@ public function testJoinRelationshipAggregateRefusesAJoinedCollectionWithoutGran }); $rows = $database->getAuthorization()->skip(fn (): array => $database->aggregate($main, $queries)); - $this->assertSame(2, (int) $rows[0]['rows'], 'with authorization disabled every joined row is read'); + $this->assertSame(2, self::joinRelationshipInteger($rows[0]['rows']), 'with authorization disabled every joined row is read'); } finally { $this->deleteJoinRelationshipCollections($database, $main, $joined); } @@ -234,8 +234,8 @@ public function testJoinRelationshipAggregateFiltersAJoinedCollectionPerDocument Query::sum('entry.amount', 'total'), ]); - $this->assertSame([1, 5], [(int) $joinedRows[0]['rows'], (int) $joinedRows[0]['total']], 'a direct aggregate reads only the rows the caller holds read on'); - $this->assertSame([1, 5], [(int) $rows[0]['rows'], (int) $rows[0]['total']], 'an aggregate over the join reads the same rows'); + $this->assertSame([1, 5], [self::joinRelationshipInteger($joinedRows[0]['rows']), self::joinRelationshipInteger($joinedRows[0]['total'])], 'a direct aggregate reads only the rows the caller holds read on'); + $this->assertSame([1, 5], [self::joinRelationshipInteger($rows[0]['rows']), self::joinRelationshipInteger($rows[0]['total'])], 'an aggregate over the join reads the same rows'); }); } finally { $this->deleteJoinRelationshipCollections($database, $main, $joined); @@ -388,4 +388,9 @@ private function withJoinRelationshipRoles(Database $database, array $roles, cal } } } + + private static function joinRelationshipInteger(mixed $value): ?int + { + return \is_numeric($value) ? (int) $value : null; + } } diff --git a/tests/unit/Joins/RelationshipJoinTest.php b/tests/unit/Joins/RelationshipJoinTest.php index 9b8d20641..62877338a 100644 --- a/tests/unit/Joins/RelationshipJoinTest.php +++ b/tests/unit/Joins/RelationshipJoinTest.php @@ -246,7 +246,7 @@ public static function reads(): iterable yield 'sum' => [static fn (Database $database): int => (int) $database->sum('customers', 'ord.amount', [$join]), 100]; yield 'aggregate' => [ static fn (Database $database): array => \array_map( - static fn (array $row): array => [$row['name'], (int) $row['total']], + static fn (array $row): array => [$row['name'], self::integer($row['total'])], $database->aggregate('customers', [$join, Query::sum('ord.amount', 'total'), Query::groupBy(['name']), Query::orderAsc('name')]), ), [['Ada', 30], ['Bob', 70]], @@ -331,4 +331,9 @@ private function createDocument(string $collection, string $id, array $attribute ...$attributes, ])); } + + private static function integer(mixed $value): ?int + { + return \is_numeric($value) ? (int) $value : null; + } } diff --git a/tests/unit/Relationships/NestedPreparationTest.php b/tests/unit/Relationships/NestedPreparationTest.php index ae0874677..13de51348 100644 --- a/tests/unit/Relationships/NestedPreparationTest.php +++ b/tests/unit/Relationships/NestedPreparationTest.php @@ -425,14 +425,11 @@ public function testTheTogglesAreScopedAndOffByDefault(Closure $adapter): void { $database = $this->database($adapter(), RelationshipType::ManyToOne); - $this->assertFalse($database->isReplacingPlaceholders()); - $this->assertFalse($database->isCheckingGrants()); - $this->assertSame([true, true], $database->withReplacePlaceholders(true, fn (): array => $database->withCheckGrants(true, fn (): array => [ - $database->isReplacingPlaceholders(), - $database->isCheckingGrants(), - ]))); - $this->assertFalse($database->isReplacingPlaceholders()); - $this->assertFalse($database->isCheckingGrants()); + $toggles = static fn (): array => [$database->isReplacingPlaceholders(), $database->isCheckingGrants()]; + + $this->assertSame([false, false], $toggles()); + $this->assertSame([true, true], $database->withReplacePlaceholders(true, fn (): array => $database->withCheckGrants(true, $toggles))); + $this->assertSame([false, false], $toggles()); } public function testASnapshotCarriesBothTogglesIntoAChildCoroutine(): void From d1b836e4ff3ce1d231d760d05303a69d0718b04e Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:25:22 +1300 Subject: [PATCH 20/55] fix(documents): type the fraction resolution for the static analyser The upsert closure resolved fractions through a method on the database before reading the adapter, which dropped the Upserts narrowing that adapterHasFeature() asserts; the adapter is now read first. A stored value that is not a scalar is no longer handed to Fractional::operator(), so its operator stays unresolved and is refused. isFractional() asserts the float or numeric text it accepts, which replaces an inline @var. Co-Authored-By: Claude Opus 5.5 --- src/Database/Arithmetic/Fractional.php | 2 ++ src/Database/Trait/Documents.php | 7 +++++-- src/Database/Validator/Operator.php | 4 +++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Database/Arithmetic/Fractional.php b/src/Database/Arithmetic/Fractional.php index d34062920..f2cdcbf4c 100644 --- a/src/Database/Arithmetic/Fractional.php +++ b/src/Database/Arithmetic/Fractional.php @@ -27,6 +27,8 @@ public function __construct(private Arithmetic $arithmetic) /** * Whether a value is a finite number with a fractional part. + * + * @phpstan-assert-if-true float|numeric-string $value */ public static function isFractional(mixed $value): bool { diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index a0089e3bb..18006ada0 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -405,8 +405,10 @@ private static function resolveFractions(Document $document, array $keys, Docume $resolved = []; foreach ($keys as $key) { $operator = $document->getAttribute($key); + $current = $stored->getAttribute($key); if ($operator instanceof Operator) { - $resolved[$key] = $fractional->operator($operator, $stored->getAttribute($key))->setAttribute($key); + $answerable = $current === null || \is_int($current) || \is_float($current) || \is_string($current); + $resolved[$key] = ($answerable ? $fractional->operator($operator, $current) : $operator)->setAttribute($key); } } @@ -2657,11 +2659,12 @@ function () use ($collection, $increase, $chunk, $fractionalIds): array { throw new DatabaseException('Adapter does not support upserts'); } + $adapter = $this->adapter; + if ($fractionalIds !== []) { $this->resolveUpsertFractions($collection, $chunk, $fractionalIds); } - $adapter = $this->adapter; $batch = $this->authorization->skip(fn () => $adapter->upsertDocuments( $collection, $chunk, diff --git a/src/Database/Validator/Operator.php b/src/Database/Validator/Operator.php index abbdd29e5..47a8b8aa6 100644 --- a/src/Database/Validator/Operator.php +++ b/src/Database/Validator/Operator.php @@ -95,6 +95,9 @@ private function isNumericValueInBounds(mixed $value, Attribute $attribute): boo return $numeric >= $bounds->min && $numeric <= $bounds->max; } + /** + * @phpstan-assert-if-true float|numeric-string $value + */ private function isFraction(mixed $value, Attribute $attribute): bool { return $this->fractions && $attribute->isInteger() && Fractional::isFractional($value); @@ -116,7 +119,6 @@ private function isValidLimit(mixed $limit, Attribute $attribute, DatabaseOperat $bounds = $attribute->bounds(); if ($bounds !== null && $this->isFraction($limit, $attribute)) { - /** @var float|string $limit */ if ($this->isFractionInBounds($limit, $attribute)) { return true; } From e7fec2d8431fa790f4dfbbbc7ce2823a366925e0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:25:22 +1300 Subject: [PATCH 21/55] test(documents): narrow cursor and read values before comparing them Joined and aggregated values are mixed, so they are read as integers once checked numeric. The find() refusal is asserted with expectException() instead of an always-true instanceof check. Co-Authored-By: Claude Opus 5.5 --- tests/e2e/Adapter/Scopes/CursorTests.php | 11 ++++++++--- tests/e2e/Adapter/Scopes/ReadTests.php | 21 ++++++++++++--------- tests/unit/Documents/ResolveCursorTest.php | 4 ++-- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/CursorTests.php b/tests/e2e/Adapter/Scopes/CursorTests.php index 57e4bcdda..056016507 100644 --- a/tests/e2e/Adapter/Scopes/CursorTests.php +++ b/tests/e2e/Adapter/Scopes/CursorTests.php @@ -86,9 +86,9 @@ public function testResolveCursorTakesAJoinedOrderFromTheRowsTheCallerCanRead(): [['alice', 'a-first'], ['carol', 'c-only'], ['alice', 'a-second']], $this->cursorRows($database->find(self::CURSOR_CUSTOMERS, [...$queries, Query::limit(100)])), ); - $this->assertSame(25, (int) $database->resolveCursor(self::CURSOR_CUSTOMERS, 'alice', CursorDirection::After, $queries)->getAttribute('ord.amount'), 'after alice the list resumes behind its last paid order the caller can read'); - $this->assertSame(10, (int) $database->resolveCursor(self::CURSOR_CUSTOMERS, 'alice', CursorDirection::Before, $queries)->getAttribute('ord.amount')); - $this->assertSame(20, (int) $database->resolveCursor(self::CURSOR_CUSTOMERS, 'bob', CursorDirection::After, $queries)->getAttribute('ord.amount'), 'the joined values of a document the caller cannot read come from the rows it can read'); + $this->assertSame(25, self::cursorInteger($database->resolveCursor(self::CURSOR_CUSTOMERS, 'alice', CursorDirection::After, $queries)->getAttribute('ord.amount')), 'after alice the list resumes behind its last paid order the caller can read'); + $this->assertSame(10, self::cursorInteger($database->resolveCursor(self::CURSOR_CUSTOMERS, 'alice', CursorDirection::Before, $queries)->getAttribute('ord.amount'))); + $this->assertSame(20, self::cursorInteger($database->resolveCursor(self::CURSOR_CUSTOMERS, 'bob', CursorDirection::After, $queries)->getAttribute('ord.amount')), 'the joined values of a document the caller cannot read come from the rows it can read'); $this->assertSame([], $this->cursorPage($database, $queries, 'alice', CursorDirection::After)); $this->assertSame([], $this->cursorPage($database, $queries, 'alice', CursorDirection::Before)); @@ -244,4 +244,9 @@ private function cursorRows(array $documents): array { return \array_values(\array_map(static fn (Document $document): array => [$document->getId(), $document->getAttribute('ord.$id')], $documents)); } + + private static function cursorInteger(mixed $value): ?int + { + return \is_numeric($value) ? (int) $value : null; + } } diff --git a/tests/e2e/Adapter/Scopes/ReadTests.php b/tests/e2e/Adapter/Scopes/ReadTests.php index bbaffb952..840321261 100644 --- a/tests/e2e/Adapter/Scopes/ReadTests.php +++ b/tests/e2e/Adapter/Scopes/ReadTests.php @@ -69,13 +69,12 @@ public function testReadReturnsAggregateRowsAsDocuments(): void try { $totals = $database->read(self::READ_POSTS, [Query::count('*', 'posts'), Query::sum('views', 'views')]); $this->assertCount(1, $totals); - $this->assertInstanceOf(Document::class, $totals[0]); - $this->assertSame([3, 6], [(int) $totals[0]->getAttribute('posts'), (int) $totals[0]->getAttribute('views')]); + $this->assertSame([3, 6], [self::readInteger($totals[0]->getAttribute('posts')), self::readInteger($totals[0]->getAttribute('views'))]); $groups = $database->read(self::READ_POSTS, [Query::groupBy(['category']), Query::count('*', 'posts'), Query::orderAsc('category')]); $this->assertSame( [['news', 2], ['sport', 1]], - \array_map(static fn (Document $row): array => [$row->getAttribute('category'), (int) $row->getAttribute('posts')], $groups), + \array_map(static fn (Document $row): array => [$row->getAttribute('category'), self::readInteger($row->getAttribute('posts'))], $groups), ); } finally { $this->deleteReadPosts($database); @@ -102,14 +101,13 @@ public function testFindStillRefusesAnAggregation(): void $database = $this->getDatabase(); $this->seedReadPosts($database); + $this->expectException(QueryException::class); + if ($database->getAdapter()->supports(Capability::Aggregations)) { + $this->expectExceptionMessage('find() does not run aggregate or groupBy queries: use aggregate()'); + } + try { $database->find(self::READ_POSTS, [Query::count()]); - $this->fail('find() must refuse an aggregate query'); - } catch (QueryException $exception) { - $this->assertInstanceOf(QueryException::class, $exception); - if ($database->getAdapter()->supports(Capability::Aggregations)) { - $this->assertSame('find() does not run aggregate or groupBy queries: use aggregate()', $exception->getMessage()); - } } finally { $this->deleteReadPosts($database); } @@ -156,4 +154,9 @@ private function deleteReadPosts(Database $database): void $database->deleteCollection(self::READ_POSTS); } } + + private static function readInteger(mixed $value): ?int + { + return \is_numeric($value) ? (int) $value : null; + } } diff --git a/tests/unit/Documents/ResolveCursorTest.php b/tests/unit/Documents/ResolveCursorTest.php index 92d05378a..47aa32e07 100644 --- a/tests/unit/Documents/ResolveCursorTest.php +++ b/tests/unit/Documents/ResolveCursorTest.php @@ -444,10 +444,10 @@ private function page(array $queries, string $id, CursorDirection $direction): a { $cursor = $this->database->resolveCursor(self::CUSTOMERS, $id, $direction, $queries); - return $this->database->find(self::CUSTOMERS, [ + return \array_values($this->database->find(self::CUSTOMERS, [ ...$queries, $direction === CursorDirection::After ? Query::cursorAfter($cursor) : Query::cursorBefore($cursor), - ]); + ])); } /** From b95c5ab230b62ea10e5aaabdcedbfcfcc3e0ff68 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:39:05 +1300 Subject: [PATCH 22/55] refactor(joins): move join helpers out of the documents trait The documents trait had grown past 5000 lines, and the relationship join rewrite added three more helpers to it. The join helpers already formed one contiguous block (relationship join rewrite, joined collection resolution, join authorization, adapter hints, outer join id selections), so they move as is into a Joins trait used by Database. Call sites, constants and behaviour are unchanged. Co-Authored-By: Claude Opus 5.5 --- src/Database/Database.php | 1 + src/Database/Trait/Documents.php | 450 ----------------------------- src/Database/Trait/Joins.php | 470 +++++++++++++++++++++++++++++++ 3 files changed, 471 insertions(+), 450 deletions(-) create mode 100644 src/Database/Trait/Joins.php diff --git a/src/Database/Database.php b/src/Database/Database.php index 91f10ea3b..5dcf3f017 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -57,6 +57,7 @@ class Database use Trait\Databases; use Trait\Documents; use Trait\Indexes; + use Trait\Joins; use Trait\Relationships; use Trait\Transactions; diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 18006ada0..8a6fde771 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -54,14 +54,12 @@ use Utopia\Database\Validator\Queries\Documents as DocumentsValidator; use Utopia\Database\Validator\Queries\Narrow; use Utopia\Database\Validator\Query\Aggregate; -use Utopia\Database\Validator\Query\Join as JoinValidator; use Utopia\Database\Validator\Query\Joined\Collection as JoinedCollection; use Utopia\Database\Validator\Structure; use Utopia\Database\Validator\UID; use Utopia\Query\CursorDirection; use Utopia\Query\Method; use Utopia\Query\OrderDirection; -use Utopia\Query\Query as BaseQuery; use Utopia\Query\Schema\ColumnType; use Utopia\Query\Schema\IndexType; use WeakMap; @@ -5249,454 +5247,6 @@ private function validateDocumentsQueries(Document $collection, array $queries, } } - /** - * A one-to-many relationship attribute, and the child side of a two-way many-to-one, holds no column: the - * related collection stores this document's ID. A join of that related collection comparing such an attribute - * with the related `$id` is rewritten to compare this document's `$id` with the column storing it, keeping the - * operator and alias. The rewritten joins are copies; the caller's queries are left as given. - * - * @param array $queries - * @return array - */ - private function rewriteRelationshipJoins(Collection $collection, array $queries): array - { - $relationships = null; - - foreach ($queries as $index => $query) { - if (! $query->getMethod()->isJoin()) { - continue; - } - - $relationships ??= self::joinableRelationships($collection); - if ($relationships === []) { - return $queries; - } - - $joined = $query->getAttribute(); - $alias = $query->getAlias(); - $rewritten = null; - - foreach ($query->getJoinOnQueries() as $position => $condition) { - $column = self::relationshipJoinColumn($condition, $joined, $alias, $relationships); - if ($column === null) { - continue; - } - - $rewritten ??= clone $query; - $rewritten->getJoinOnQueries()[$position]->setValues([Document::ID, $condition->getValues()[1] ?? '=', $alias.'.'.$column]); - } - - if ($rewritten !== null) { - $queries[$index] = $rewritten; - } - } - - return $queries; - } - - /** - * The relationship attributes whose ID the related collection stores, by key. - * - * @return array - */ - private static function joinableRelationships(Collection $collection): array - { - $relationships = []; - foreach (self::relationshipAttributes($collection) as $attribute) { - $relationship = $attribute->relationship; - if ($relationship === null || $relationship->twoWayKey === null || $relationship->twoWayKey === '') { - continue; - } - - $storedByRelated = match ($relationship->type) { - RelationshipType::OneToMany => $attribute->side === RelationshipSide::Parent, - RelationshipType::ManyToOne => $attribute->side === RelationshipSide::Child && $relationship->twoWay, - RelationshipType::OneToOne, RelationshipType::ManyToMany => false, - }; - - if ($storedByRelated) { - $relationships[$attribute->key] = $attribute; - } - } - - return $relationships; - } - - /** - * The related column a join condition compares with this document's `$id`, or null when the condition does not - * compare a relationship attribute of this collection with the `$id` of the collection that relationship relates. - * - * @param array $relationships - */ - private static function relationshipJoinColumn(BaseQuery $condition, string $joined, string $alias, array $relationships): ?string - { - if ($condition->getMethod() !== Method::On) { - return null; - } - - $values = $condition->getValues(); - $left = $values[0] ?? null; - $right = $values[2] ?? null; - if (! \is_string($left) || ($right !== Document::ID && $right !== $alias.'.'.Document::ID)) { - return null; - } - - $relationship = ($relationships[$left] ?? null)?->relationship; - if ($relationship === null || $relationship->relatedCollection !== $joined) { - return null; - } - - return $relationship->twoWayKey; - } - - /** - * The collections the join queries name, each loaded once, by the id the join names it with. A - * read resolves them once and hands them to validation, authorization, the adapter and decoding. - * - * @param array $queries - * @return array - * - * @throws QueryException - */ - private function resolveJoinedCollections(array $queries): array - { - $collections = []; - - foreach ($queries as $query) { - if (! $query->getMethod()->isJoin()) { - continue; - } - - $id = $query->getAttribute(); - if ($id === '' || isset($collections[$id])) { - continue; - } - - $collection = $this->silent(fn () => $this->findCollection($id)); - if ($collection === null) { - throw new QueryException("Joined collection '{$id}' not found"); - } - - $collections[$id] = $collection; - } - - return $collections; - } - - /** - * @param array $queries - * @param list $relationships - * @param array|null $joinedCollections The collection each join names, by its id - * @return array{0: Document, 1: array, 2: bool}|null - */ - private function prepareFilterJoinQueries( - Document $collection, - array $queries, - array $relationships, - bool $collectionGranted, - ?array $joinedCollections = null, - ): ?array { - $parsed = Query::groupByType($queries); - $filters = $parsed->filters; - $joins = $parsed->joins; - - if (! empty($joins)) { - if (! $this->adapter->supports(Capability::Joins)) { - throw new QueryException('Join queries are not supported by this adapter'); - } - - $this->assertJoinCount($joins); - - $joinedCollections ??= $this->resolveJoinedCollections($joins); - $collection = $this->withJoinIndexed($this->withJoinAuthorization( - $collection, - $this->authorizeJoins($joins, PermissionType::Read, $joinedCollections), - $collectionGranted, - ), $joins, $joinedCollections); - } - - $queries = \array_merge($filters, $joins); - if ($queries !== []) { - $queries = $this->convertQueries($collection, $queries, $this->joinedCollectionsByAlias($joins, $joinedCollections)); - } - - $convertedQueries = $this->relationshipHook !== null - ? $this->relationshipHook->convertQueries($relationships, $queries, $collection) - : $queries; - - if ($convertedQueries === null) { - return null; - } - - return [$collection, $convertedQueries, $collectionGranted && empty($joins)]; - } - - /** - * The join cap the Join validator enforces holds without validation too: every join is one - * more table the engine plans and the permission filters check. - * - * @param array $joins - * - * @throws QueryException - */ - private function assertJoinCount(array $joins): void - { - $validator = new JoinValidator(); - if (! $validator->isValidCount(\count($joins))) { - throw new QueryException($validator->getDescription()); - } - } - - /** - * Set on the collection handed to the adapter for a join read when the caller - * holds the collection-level permission, so its rows are not filtered per document. - */ - public const string COLLECTION_GRANTED = 'collectionGranted'; - - /** - * Maps each joined table to whether the adapter filters its rows per document. - */ - public const string JOIN_DOCUMENT_SECURITY = 'joinDocumentSecurity'; - - /** - * Each joined collection is authorized once, however many joins read it. - * - * @param array $joins - * @param array|null $joinedCollections The collection each join names, by its id - * @return array - */ - private function authorizeJoins(array $joins, PermissionType $forPermission, ?array $joinedCollections = null): array - { - $joinedCollections ??= $this->resolveJoinedCollections($joins); - $joinDocumentSecurity = []; - $authorized = []; - - foreach ($joins as $joinQuery) { - $joinCollectionId = $joinQuery->getAttribute(); - if (isset($authorized[$joinCollectionId])) { - continue; - } - $authorized[$joinCollectionId] = true; - - $joinCollection = $joinedCollections[$joinCollectionId] ?? null; - - if ($joinCollection === null) { - throw new QueryException("Joined collection '{$joinCollectionId}' not found"); - } - - $granted = $this->authorization->isValid(new Input($forPermission, $joinCollection->getPermissionsByType($forPermission))); - $documentSecurity = (bool) $joinCollection->getAttribute('documentSecurity', false); - - if (! $granted && ! $documentSecurity) { - throw new AuthorizationException("Unauthorized access to joined collection '{$joinCollectionId}'"); - } - - foreach ($this->joinDocumentSecurityKeys($joinCollectionId, $joinCollection) as $key) { - $joinDocumentSecurity[$key] = ! $granted; - } - } - - return $joinDocumentSecurity; - } - - /** - * @return list - */ - private function joinDocumentSecurityKeys(string $joinCollectionId, Document $joinCollection): array - { - $keys = [ - $joinCollectionId, - $joinCollection->getId(), - $this->adapter->filter($joinCollectionId), - $this->adapter->filter($joinCollection->getId()), - ]; - - return \array_values(\array_unique(\array_filter( - $keys, - static fn (string $key): bool => $key !== '', - ))); - } - - /** - * @param array $joinDocumentSecurity - */ - private function withJoinAuthorization(Document $collection, array $joinDocumentSecurity, bool $collectionGranted): Document - { - if ($joinDocumentSecurity === []) { - return $collection; - } - - $adapterCollection = clone $collection; - $adapterCollection->setAttribute(self::COLLECTION_GRANTED, $collectionGranted); - $adapterCollection->setAttribute(self::JOIN_DOCUMENT_SECURITY, $joinDocumentSecurity); - - return $adapterCollection; - } - - /** - * Maps each joined collection, as its join query names it, to the attributes a join without a - * select returns under the join's alias. - */ - public const string JOIN_ATTRIBUTES = 'joinAttributes'; - - /** - * Relationship attributes are left out: only some sides of a relationship have a column, and a - * join does not populate related documents. - * - * @param array $joins - * @param array|null $joinedCollections The collection each join names, by its id - */ - private function withJoinAttributes(Document $collection, array $joins, ?array $joinedCollections = null): Document - { - if ($joins === []) { - return $collection; - } - - $joinedCollections ??= $this->resolveJoinedCollections($joins); - - $joinAttributes = []; - foreach ($joins as $join) { - $joinCollectionId = $join->getAttribute(); - if (isset($joinAttributes[$joinCollectionId])) { - continue; - } - - $keys = []; - if (isset($joinedCollections[$joinCollectionId])) { - foreach (Collection::fromDocument($joinedCollections[$joinCollectionId])->attributes() as $attribute) { - if ($attribute->relationship === null) { - $keys[] = $attribute->key; - } - } - } - $joinAttributes[$joinCollectionId] = $keys; - } - - $adapterCollection = clone $collection; - $adapterCollection->setAttribute(self::JOIN_ATTRIBUTES, $joinAttributes); - - return $adapterCollection; - } - - /** - * Under shared tables, maps each joined collection, as its join query names it, to the attributes one of its key - * or unique indexes leads with: the ones a join can look its rows up by. - */ - public const string JOIN_INDEXED = 'joinIndexed'; - - private const array LOOKUP_INDEX_TYPES = [IndexType::Key, IndexType::Index, IndexType::Unique]; - - /** - * @param array $joins - * @param array|null $joinedCollections The collection each join names, by its id - */ - private function withJoinIndexed(Document $collection, array $joins, ?array $joinedCollections = null): Document - { - if ($joins === [] || ! $this->adapter->hasSharedTables()) { - return $collection; - } - - $joinedCollections ??= $this->resolveJoinedCollections($joins); - - $joinIndexed = []; - foreach ($joins as $join) { - $joinCollectionId = $join->getAttribute(); - if (isset($joinIndexed[$joinCollectionId]) || ! isset($joinedCollections[$joinCollectionId])) { - continue; - } - - $leads = []; - foreach (Collection::fromDocument($joinedCollections[$joinCollectionId])->indexes() as $index) { - if (\in_array($index->type, self::LOOKUP_INDEX_TYPES, true) && isset($index->attributes[0])) { - $leads[] = $index->attributes[0]; - } - } - $joinIndexed[$joinCollectionId] = \array_values(\array_unique($leads)); - } - - $adapterCollection = clone $collection; - $adapterCollection->setAttribute(self::JOIN_INDEXED, $joinIndexed); - - return $adapterCollection; - } - - /** - * The collection each join reads, by the alias its values come back under. - * - * @param array $joins - * @param array|null $joinedCollections The collection each join names, by its id - * @return array - */ - private function joinedCollectionsByAlias(array $joins, ?array $joinedCollections = null): array - { - if ($joins === []) { - return []; - } - - $joinedCollections ??= $this->resolveJoinedCollections($joins); - - $collections = []; - foreach ($joins as $join) { - $collections[$join->getAlias()] = $joinedCollections[$join->getAttribute()] ?? new Document(); - } - - return $collections; - } - - /** - * The `alias.$id` of each join whose attributes a select names without it or `alias.*`, when an - * outer join can leave a joined row unmatched: the joined `$id` is what tells an unmatched row - * from a matched one when the row is decoded, so it is selected for that and left out of the - * result. - * - * @param array $selects - * @param array $joins - * @param array $joinedCollections The collection each join alias reads - * @return list - */ - private function outerJoinIdSelections(array $selects, array $joins, array $joinedCollections): array - { - if ($selects === [] || $joinedCollections === []) { - return []; - } - - $outer = false; - foreach ($joins as $join) { - if (\in_array($join->getMethod(), [Method::LeftJoin, Method::RightJoin, Method::FullOuterJoin], true)) { - $outer = true; - break; - } - } - if (! $outer) { - return []; - } - - $selectedAliases = []; - foreach ($selects as $select) { - foreach ($select->getValues() as $value) { - if (! \is_string($value)) { - continue; - } - if ($value === '*') { - return []; - } - $dot = \strpos($value, '.'); - if ($dot !== false) { - $selectedAliases[\substr($value, 0, $dot)][\substr($value, $dot + 1)] = true; - } - } - } - - $ids = []; - foreach ($selectedAliases as $alias => $attributes) { - if (isset($joinedCollections[$alias]) && ! isset($attributes[Document::ID]) && ! isset($attributes['*'])) { - $ids[] = $alias.'.'.Document::ID; - } - } - - return $ids; - } - /** * @param array $queries * @return array diff --git a/src/Database/Trait/Joins.php b/src/Database/Trait/Joins.php new file mode 100644 index 000000000..c45597cf6 --- /dev/null +++ b/src/Database/Trait/Joins.php @@ -0,0 +1,470 @@ + $queries + * @return array + */ + private function rewriteRelationshipJoins(Collection $collection, array $queries): array + { + $relationships = null; + + foreach ($queries as $index => $query) { + if (! $query->getMethod()->isJoin()) { + continue; + } + + $relationships ??= self::joinableRelationships($collection); + if ($relationships === []) { + return $queries; + } + + $joined = $query->getAttribute(); + $alias = $query->getAlias(); + $rewritten = null; + + foreach ($query->getJoinOnQueries() as $position => $condition) { + $column = self::relationshipJoinColumn($condition, $joined, $alias, $relationships); + if ($column === null) { + continue; + } + + $rewritten ??= clone $query; + $rewritten->getJoinOnQueries()[$position]->setValues([Document::ID, $condition->getValues()[1] ?? '=', $alias.'.'.$column]); + } + + if ($rewritten !== null) { + $queries[$index] = $rewritten; + } + } + + return $queries; + } + + /** + * The relationship attributes whose ID the related collection stores, by key. + * + * @return array + */ + private static function joinableRelationships(Collection $collection): array + { + $relationships = []; + foreach (self::relationshipAttributes($collection) as $attribute) { + $relationship = $attribute->relationship; + if ($relationship === null || $relationship->twoWayKey === null || $relationship->twoWayKey === '') { + continue; + } + + $storedByRelated = match ($relationship->type) { + RelationshipType::OneToMany => $attribute->side === RelationshipSide::Parent, + RelationshipType::ManyToOne => $attribute->side === RelationshipSide::Child && $relationship->twoWay, + RelationshipType::OneToOne, RelationshipType::ManyToMany => false, + }; + + if ($storedByRelated) { + $relationships[$attribute->key] = $attribute; + } + } + + return $relationships; + } + + /** + * The related column a join condition compares with this document's `$id`, or null when the condition does not + * compare a relationship attribute of this collection with the `$id` of the collection that relationship relates. + * + * @param array $relationships + */ + private static function relationshipJoinColumn(BaseQuery $condition, string $joined, string $alias, array $relationships): ?string + { + if ($condition->getMethod() !== Method::On) { + return null; + } + + $values = $condition->getValues(); + $left = $values[0] ?? null; + $right = $values[2] ?? null; + if (! \is_string($left) || ($right !== Document::ID && $right !== $alias.'.'.Document::ID)) { + return null; + } + + $relationship = ($relationships[$left] ?? null)?->relationship; + if ($relationship === null || $relationship->relatedCollection !== $joined) { + return null; + } + + return $relationship->twoWayKey; + } + + /** + * The collections the join queries name, each loaded once, by the id the join names it with. A + * read resolves them once and hands them to validation, authorization, the adapter and decoding. + * + * @param array $queries + * @return array + * + * @throws QueryException + */ + private function resolveJoinedCollections(array $queries): array + { + $collections = []; + + foreach ($queries as $query) { + if (! $query->getMethod()->isJoin()) { + continue; + } + + $id = $query->getAttribute(); + if ($id === '' || isset($collections[$id])) { + continue; + } + + $collection = $this->silent(fn () => $this->findCollection($id)); + if ($collection === null) { + throw new QueryException("Joined collection '{$id}' not found"); + } + + $collections[$id] = $collection; + } + + return $collections; + } + + /** + * @param array $queries + * @param list $relationships + * @param array|null $joinedCollections The collection each join names, by its id + * @return array{0: Document, 1: array, 2: bool}|null + */ + private function prepareFilterJoinQueries( + Document $collection, + array $queries, + array $relationships, + bool $collectionGranted, + ?array $joinedCollections = null, + ): ?array { + $parsed = Query::groupByType($queries); + $filters = $parsed->filters; + $joins = $parsed->joins; + + if (! empty($joins)) { + if (! $this->adapter->supports(Capability::Joins)) { + throw new QueryException('Join queries are not supported by this adapter'); + } + + $this->assertJoinCount($joins); + + $joinedCollections ??= $this->resolveJoinedCollections($joins); + $collection = $this->withJoinIndexed($this->withJoinAuthorization( + $collection, + $this->authorizeJoins($joins, PermissionType::Read, $joinedCollections), + $collectionGranted, + ), $joins, $joinedCollections); + } + + $queries = \array_merge($filters, $joins); + if ($queries !== []) { + $queries = $this->convertQueries($collection, $queries, $this->joinedCollectionsByAlias($joins, $joinedCollections)); + } + + $convertedQueries = $this->relationshipHook !== null + ? $this->relationshipHook->convertQueries($relationships, $queries, $collection) + : $queries; + + if ($convertedQueries === null) { + return null; + } + + return [$collection, $convertedQueries, $collectionGranted && empty($joins)]; + } + + /** + * The join cap the Join validator enforces holds without validation too: every join is one + * more table the engine plans and the permission filters check. + * + * @param array $joins + * + * @throws QueryException + */ + private function assertJoinCount(array $joins): void + { + $validator = new JoinValidator(); + if (! $validator->isValidCount(\count($joins))) { + throw new QueryException($validator->getDescription()); + } + } + + /** + * Set on the collection handed to the adapter for a join read when the caller + * holds the collection-level permission, so its rows are not filtered per document. + */ + public const string COLLECTION_GRANTED = 'collectionGranted'; + + /** + * Maps each joined table to whether the adapter filters its rows per document. + */ + public const string JOIN_DOCUMENT_SECURITY = 'joinDocumentSecurity'; + + /** + * Each joined collection is authorized once, however many joins read it. + * + * @param array $joins + * @param array|null $joinedCollections The collection each join names, by its id + * @return array + */ + private function authorizeJoins(array $joins, PermissionType $forPermission, ?array $joinedCollections = null): array + { + $joinedCollections ??= $this->resolveJoinedCollections($joins); + $joinDocumentSecurity = []; + $authorized = []; + + foreach ($joins as $joinQuery) { + $joinCollectionId = $joinQuery->getAttribute(); + if (isset($authorized[$joinCollectionId])) { + continue; + } + $authorized[$joinCollectionId] = true; + + $joinCollection = $joinedCollections[$joinCollectionId] ?? null; + + if ($joinCollection === null) { + throw new QueryException("Joined collection '{$joinCollectionId}' not found"); + } + + $granted = $this->authorization->isValid(new Input($forPermission, $joinCollection->getPermissionsByType($forPermission))); + $documentSecurity = (bool) $joinCollection->getAttribute('documentSecurity', false); + + if (! $granted && ! $documentSecurity) { + throw new AuthorizationException("Unauthorized access to joined collection '{$joinCollectionId}'"); + } + + foreach ($this->joinDocumentSecurityKeys($joinCollectionId, $joinCollection) as $key) { + $joinDocumentSecurity[$key] = ! $granted; + } + } + + return $joinDocumentSecurity; + } + + /** + * @return list + */ + private function joinDocumentSecurityKeys(string $joinCollectionId, Document $joinCollection): array + { + $keys = [ + $joinCollectionId, + $joinCollection->getId(), + $this->adapter->filter($joinCollectionId), + $this->adapter->filter($joinCollection->getId()), + ]; + + return \array_values(\array_unique(\array_filter( + $keys, + static fn (string $key): bool => $key !== '', + ))); + } + + /** + * @param array $joinDocumentSecurity + */ + private function withJoinAuthorization(Document $collection, array $joinDocumentSecurity, bool $collectionGranted): Document + { + if ($joinDocumentSecurity === []) { + return $collection; + } + + $adapterCollection = clone $collection; + $adapterCollection->setAttribute(self::COLLECTION_GRANTED, $collectionGranted); + $adapterCollection->setAttribute(self::JOIN_DOCUMENT_SECURITY, $joinDocumentSecurity); + + return $adapterCollection; + } + + /** + * Maps each joined collection, as its join query names it, to the attributes a join without a + * select returns under the join's alias. + */ + public const string JOIN_ATTRIBUTES = 'joinAttributes'; + + /** + * Relationship attributes are left out: only some sides of a relationship have a column, and a + * join does not populate related documents. + * + * @param array $joins + * @param array|null $joinedCollections The collection each join names, by its id + */ + private function withJoinAttributes(Document $collection, array $joins, ?array $joinedCollections = null): Document + { + if ($joins === []) { + return $collection; + } + + $joinedCollections ??= $this->resolveJoinedCollections($joins); + + $joinAttributes = []; + foreach ($joins as $join) { + $joinCollectionId = $join->getAttribute(); + if (isset($joinAttributes[$joinCollectionId])) { + continue; + } + + $keys = []; + if (isset($joinedCollections[$joinCollectionId])) { + foreach (Collection::fromDocument($joinedCollections[$joinCollectionId])->attributes() as $attribute) { + if ($attribute->relationship === null) { + $keys[] = $attribute->key; + } + } + } + $joinAttributes[$joinCollectionId] = $keys; + } + + $adapterCollection = clone $collection; + $adapterCollection->setAttribute(self::JOIN_ATTRIBUTES, $joinAttributes); + + return $adapterCollection; + } + + /** + * Under shared tables, maps each joined collection, as its join query names it, to the attributes one of its key + * or unique indexes leads with: the ones a join can look its rows up by. + */ + public const string JOIN_INDEXED = 'joinIndexed'; + + private const array LOOKUP_INDEX_TYPES = [IndexType::Key, IndexType::Index, IndexType::Unique]; + + /** + * @param array $joins + * @param array|null $joinedCollections The collection each join names, by its id + */ + private function withJoinIndexed(Document $collection, array $joins, ?array $joinedCollections = null): Document + { + if ($joins === [] || ! $this->adapter->hasSharedTables()) { + return $collection; + } + + $joinedCollections ??= $this->resolveJoinedCollections($joins); + + $joinIndexed = []; + foreach ($joins as $join) { + $joinCollectionId = $join->getAttribute(); + if (isset($joinIndexed[$joinCollectionId]) || ! isset($joinedCollections[$joinCollectionId])) { + continue; + } + + $leads = []; + foreach (Collection::fromDocument($joinedCollections[$joinCollectionId])->indexes() as $index) { + if (\in_array($index->type, self::LOOKUP_INDEX_TYPES, true) && isset($index->attributes[0])) { + $leads[] = $index->attributes[0]; + } + } + $joinIndexed[$joinCollectionId] = \array_values(\array_unique($leads)); + } + + $adapterCollection = clone $collection; + $adapterCollection->setAttribute(self::JOIN_INDEXED, $joinIndexed); + + return $adapterCollection; + } + + /** + * The collection each join reads, by the alias its values come back under. + * + * @param array $joins + * @param array|null $joinedCollections The collection each join names, by its id + * @return array + */ + private function joinedCollectionsByAlias(array $joins, ?array $joinedCollections = null): array + { + if ($joins === []) { + return []; + } + + $joinedCollections ??= $this->resolveJoinedCollections($joins); + + $collections = []; + foreach ($joins as $join) { + $collections[$join->getAlias()] = $joinedCollections[$join->getAttribute()] ?? new Document(); + } + + return $collections; + } + + /** + * The `alias.$id` of each join whose attributes a select names without it or `alias.*`, when an + * outer join can leave a joined row unmatched: the joined `$id` is what tells an unmatched row + * from a matched one when the row is decoded, so it is selected for that and left out of the + * result. + * + * @param array $selects + * @param array $joins + * @param array $joinedCollections The collection each join alias reads + * @return list + */ + private function outerJoinIdSelections(array $selects, array $joins, array $joinedCollections): array + { + if ($selects === [] || $joinedCollections === []) { + return []; + } + + $outer = false; + foreach ($joins as $join) { + if (\in_array($join->getMethod(), [Method::LeftJoin, Method::RightJoin, Method::FullOuterJoin], true)) { + $outer = true; + break; + } + } + if (! $outer) { + return []; + } + + $selectedAliases = []; + foreach ($selects as $select) { + foreach ($select->getValues() as $value) { + if (! \is_string($value)) { + continue; + } + if ($value === '*') { + return []; + } + $dot = \strpos($value, '.'); + if ($dot !== false) { + $selectedAliases[\substr($value, 0, $dot)][\substr($value, $dot + 1)] = true; + } + } + } + + $ids = []; + foreach ($selectedAliases as $alias => $attributes) { + if (isset($joinedCollections[$alias]) && ! isset($attributes[Document::ID]) && ! isset($attributes['*'])) { + $ids[] = $alias.'.'.Document::ID; + } + } + + return $ids; + } +} From 6a24a3484e1f18c116622afeae9ab996690b34b8 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:40:33 +1300 Subject: [PATCH 23/55] docs(cursors): warn that resolved cursors must not reach end users resolveCursor() reads the document with authorization skipped so a list can page past a document its caller cannot read, and the caller chooses the orders, so the cursor's order values (joined ones included) can expose attributes of an unreadable document. The cursor already carries only the fields paging needs; the docblock now says it must only go back into cursorAfter()/cursorBefore() and never be returned to an end user. Co-Authored-By: Claude Opus 5.5 --- src/Database/Trait/Cursors.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Database/Trait/Cursors.php b/src/Database/Trait/Cursors.php index e27ec523c..1d043dd22 100644 --- a/src/Database/Trait/Cursors.php +++ b/src/Database/Trait/Cursors.php @@ -33,6 +33,10 @@ trait Cursors * after the document starts behind the last of them in the list's order, a page before it ends ahead of the first, * and without such a row the value is null, as it is in the list. * + * The order values, joined ones included, come from or are chosen by a document the caller may not be allowed to + * read, and the caller picks which attributes the list orders by. Pass the cursor only to cursorAfter() or + * cursorBefore(); never return it, or any of its values, to an end user. + * * @param array $queries The list's queries; its cursor queries are left out * @return Document Empty when the collection holds no document with the ID * From afa4b498c899754a52fa6d0d3c341d4e378bd5f9 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:40:33 +1300 Subject: [PATCH 24/55] refactor(cursors): trim helper docblocks and build alias filters explicitly The private cursor and relationship join helpers narrated what their names and types already say; only the non-obvious parts stay (join order a paired row depends on, rows read with the caller's permissions, what a null result means, why a relationship join is rewritten). aliasFilter() built its query with `clone new Query(...)` to deep-copy nested query values; it now copies them explicitly before constructing the query, so the paired find() can still never mutate the caller's filters. Co-Authored-By: Claude Opus 5.5 --- src/Database/Trait/Cursors.php | 23 ++++++++++------------- src/Database/Trait/Joins.php | 12 +++--------- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/src/Database/Trait/Cursors.php b/src/Database/Trait/Cursors.php index 1d043dd22..5fd25c80d 100644 --- a/src/Database/Trait/Cursors.php +++ b/src/Database/Trait/Cursors.php @@ -136,8 +136,6 @@ public function resolveCursor(string $collection, string $id, CursorDirection $d } /** - * The join alias an order reads and the attribute it reads there, or the main alias and the order as given. - * * @param array $joinedCollections * @return array{0: string, 1: string} */ @@ -152,7 +150,7 @@ private function orderOwner(Collection $collection, string $attribute, array $jo } /** - * The ordered joins and every join their conditions reach through, in the order the list declares them. + * Keeps the list's join order: a paired row reads the rows paired for the joins before it. * * @param array $joins * @param array> $orders @@ -180,9 +178,7 @@ private function pairedJoins(array $joins, array $orders): array } /** - * The joined row the list pairs with the document first in the given orders, read with the caller's permissions: - * the join's conditions become filters on the joined collection with the values of the document or of the rows - * resolved for earlier joins, next to the list's filters that constrain only this join and the join's own filters. + * Read with the caller's permissions, so the row is the one the caller's list pairs with the document. * * @param array $rows The rows resolved for earlier joins, by alias * @param array $filters @@ -223,8 +219,6 @@ private function pairedRow(Query $join, string $alias, Document $document, array } /** - * Each column comparison of a join as its left column, operator and right column. - * * @return list */ private static function joinConditions(Query $join): array @@ -245,8 +239,6 @@ private static function joinConditions(Query $join): array } /** - * The alias a join column belongs to and its attribute, the given alias for a bare column. - * * @return array{0: string, 1: string} */ private static function columnOwner(string $column, string $alias): array @@ -272,7 +264,7 @@ private static function pairedValue(string $alias, string $attribute, Document $ } /** - * A join condition as a filter on the joined column, or null when no row can satisfy it. + * Null when no row can satisfy the condition. * * @throws QueryException */ @@ -294,7 +286,7 @@ private static function comparison(string $attribute, string $operator, mixed $v } /** - * A filter on the joined collection when the given filter constrains only this alias, else null. + * Null unless the filter constrains only this alias. */ private static function aliasFilter(BaseQuery $filter, string $alias): ?Query { @@ -322,6 +314,11 @@ private static function aliasFilter(BaseQuery $filter, string $alias): ?Query return null; } - return clone new Query($method, $attribute, $filter->getValues()); + $values = \array_map( + static fn (mixed $value): mixed => $value instanceof BaseQuery ? clone $value : $value, + $filter->getValues(), + ); + + return new Query($method, $attribute, $values); } } diff --git a/src/Database/Trait/Joins.php b/src/Database/Trait/Joins.php index c45597cf6..5d326c365 100644 --- a/src/Database/Trait/Joins.php +++ b/src/Database/Trait/Joins.php @@ -21,10 +21,9 @@ trait Joins { /** - * A one-to-many relationship attribute, and the child side of a two-way many-to-one, holds no column: the - * related collection stores this document's ID. A join of that related collection comparing such an attribute - * with the related `$id` is rewritten to compare this document's `$id` with the column storing it, keeping the - * operator and alias. The rewritten joins are copies; the caller's queries are left as given. + * A one-to-many parent attribute, or the child side of a two-way many-to-one, has no column: the related + * collection stores this document's ID under the two-way key. A join comparing such an attribute with the related + * `$id` compares this document's `$id` with that key instead. Rewritten joins are copies of the caller's. * * @param array $queries * @return array @@ -66,8 +65,6 @@ private function rewriteRelationshipJoins(Collection $collection, array $queries } /** - * The relationship attributes whose ID the related collection stores, by key. - * * @return array */ private static function joinableRelationships(Collection $collection): array @@ -94,9 +91,6 @@ private static function joinableRelationships(Collection $collection): array } /** - * The related column a join condition compares with this document's `$id`, or null when the condition does not - * compare a relationship attribute of this collection with the `$id` of the collection that relationship relates. - * * @param array $relationships */ private static function relationshipJoinColumn(BaseQuery $condition, string $joined, string $alias, array $relationships): ?string From 3d6cfe9ff1c46ec054d6c47bc8ff967cfa89c29b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:49:43 +1300 Subject: [PATCH 25/55] fix(arithmetic): refuse non-finite and overlong operands and read padded text An infinite or not-a-number operand or limit reached the exact path, where Decimal::fraction() handed 'inf' to BigInt and threw a raw InvalidArgumentException, or was compared as a float, where a NaN limit never held a result back. Fractional::operator() now leaves the operator unchanged for a non-finite float, so strict validation refuses it on every engine. The exact path multiplied and divided every digit it was given, quadratic in their count, so a long digit string cost seconds. Decimal::fraction() now drops trailing zeros past the fraction and refuses a number with more than twice MAXIMUM_DIGITS significant digits; an overlong limit on the exact path leaves the operator unchanged as well. is_numeric() accepts a form feed and vertical tab around a number, which trim() kept, so "\f1.5" reached BigInt and threw; split() and the sign of an oversized limit now strip the same whitespace is_numeric() accepts. Numeric text whose value is whole, such as '1e3', is a whole counter change, as appwrite's Counter answered it. Fractional::exceeds() compares a counter result with its bound exactly at any size, for the counter path to use. Co-Authored-By: Claude Opus 5.5 --- src/Database/Arithmetic/Decimal.php | 39 ++++++++++++-- src/Database/Arithmetic/Fractional.php | 44 +++++++++++++-- tests/unit/Arithmetic/DecimalTest.php | 31 +++++++++++ tests/unit/Arithmetic/FractionalTest.php | 69 ++++++++++++++++++++++++ 4 files changed, 175 insertions(+), 8 deletions(-) diff --git a/src/Database/Arithmetic/Decimal.php b/src/Database/Arithmetic/Decimal.php index 6a1b908fc..c0f340bad 100644 --- a/src/Database/Arithmetic/Decimal.php +++ b/src/Database/Arithmetic/Decimal.php @@ -16,6 +16,12 @@ */ private const int MAXIMUM_DIGITS = 64; + /** + * The most significant digits fraction() reads, far past any engine's precision: exact arithmetic on more costs + * time quadratic in their count. + */ + private const int MAXIMUM_SIGNIFICANT_DIGITS = 2 * self::MAXIMUM_DIGITS; + private const int MAXIMUM_EXPONENT_DIGITS = 18; private const string WHITESPACE = " \t\n\r\v\f"; @@ -82,15 +88,24 @@ public function ceil(): int|string : BigInt::toNative($this->truncated()); } + /** + * The integer a whole number writes; null for a number with a fraction. + */ + public function integer(): int|string|null + { + return $this->fractional ? null : BigInt::toNative($this->truncated()); + } + /** * A number as an integer numerator and the power of ten that divides it, from its text; null for text that is - * not numeric or a number whose leading digit is more than MAXIMUM_DIGITS powers of ten from the units. + * not numeric, a float that is not finite, a number whose leading digit is more than MAXIMUM_DIGITS powers of + * ten from the units, or one with more than MAXIMUM_SIGNIFICANT_DIGITS significant digits. * * @return array{int|string, int}|null */ public static function fraction(int|float|string $value): ?array { - if (\is_string($value) && ! \is_numeric($value)) { + if ((\is_string($value) && ! \is_numeric($value)) || (\is_float($value) && ! \is_finite($value))) { return null; } @@ -98,12 +113,13 @@ public static function fraction(int|float|string $value): ?array if ($order === null) { return [0, 0]; } - if (\abs($order) > self::MAXIMUM_DIGITS) { + if (\abs($order) > self::MAXIMUM_DIGITS || self::isOverlong($value)) { return null; } [$mantissa, $exponent] = self::split($value); [$whole, $fraction] = \explode('.', $mantissa, 2) + [1 => '']; + $fraction = \rtrim($fraction, '0'); $scale = \strlen($fraction) - (int) $exponent; $digits = \ltrim($whole, '+') . $fraction; @@ -115,6 +131,21 @@ public static function fraction(int|float|string $value): ?array return [BigInt::toNative(BigInt::normalizeInteger($digits)), $scale]; } + /** + * Whether a number carries more significant digits than fraction() reads. + */ + public static function isOverlong(int|float|string $value): bool + { + [$mantissa] = self::split($value); + + return \strlen(\trim(\str_replace('.', '', \ltrim($mantissa, '+-')), '0')) > self::MAXIMUM_SIGNIFICANT_DIGITS; + } + + public static function isNegative(int|float|string $value): bool + { + return \str_starts_with(\ltrim((string) $value, self::WHITESPACE), '-'); + } + /** * Whether a number fraction() leaves out is closer to zero than any result rather than beyond every result. */ @@ -153,7 +184,7 @@ public static function order(int|float|string $value): ?int */ private static function split(int|float|string $value): array { - $parts = \explode('e', \strtolower(\trim((string) $value)), 2); + $parts = \explode('e', \strtolower(\trim((string) $value, self::WHITESPACE)), 2); return [$parts[0], $parts[1] ?? '0']; } diff --git a/src/Database/Arithmetic/Fractional.php b/src/Database/Arithmetic/Fractional.php index f2cdcbf4c..aae610bf8 100644 --- a/src/Database/Arithmetic/Fractional.php +++ b/src/Database/Arithmetic/Fractional.php @@ -51,6 +51,16 @@ public static function isWhole(int|float|string $value): bool }; } + /** + * The integer a whole number writes: an integer, a finite float without a fractional part, or numeric text + * such as 1e3; null for anything else. + */ + public static function wholeValue(int|float|string $value): int|string|null + { + return BigInt::integralValue($value) + ?? (\is_string($value) && \is_numeric($value) ? Decimal::parse($value)?->integer() : null); + } + /** * Whether operator() needs the stored value: a numeric operator with a fractional amount or limit. */ @@ -94,7 +104,7 @@ public function operator(Operator $operator, int|float|string|null $stored): Ope $rounding = $this->arithmetic->rounding($method, $whole); if (\in_array($method, $this->arithmetic->exact, true)) { $exact = self::rational($method, $current, $amount); - if ($exact === null) { + if ($exact === null || ($limit !== null && Decimal::isOverlong($limit))) { return $operator; } if ($limit !== null && self::isBeyondRational($method, $exact, $amount, $limit)) { @@ -130,7 +140,7 @@ public function counter(int|string $stored, int|float|string $value, bool $incre throw new TypeException('Value must be numeric and greater than 0'); } - $whole = BigInt::integralValue($value); + $whole = self::wholeValue($value); if ($whole !== null) { return $whole; } @@ -149,6 +159,32 @@ public function counter(int|string $stored, int|float|string $value, bool $incre return BigInt::compare($result, $stored) >= 0 ? BigInt::subtract($result, $stored) : BigInt::subtract($stored, $result); } + /** + * Whether $stored plus or minus $value passes $bound, compared exactly at any size. A number Decimal::fraction() + * leaves out is compared in floating point. + */ + public static function exceeds(int|string $stored, int|float|string $value, int|float|string $bound, bool $increase): bool + { + $change = Decimal::fraction($value); + $limit = Decimal::fraction($bound); + if ($change === null || $limit === null) { + $result = $increase ? (float) $stored + (float) $value : (float) $stored - (float) $value; + + return $increase ? $result > (float) $bound : $result < (float) $bound; + } + + [$amount, $scale] = $change; + [$boundary, $boundaryScale] = $limit; + $scaled = BigInt::multiply($stored, self::powerOfTen($scale)); + $result = $increase ? BigInt::add($scaled, $amount) : BigInt::subtract($scaled, $amount); + $comparison = BigInt::compare( + BigInt::multiply($result, self::powerOfTen($boundaryScale)), + BigInt::multiply($boundary, self::powerOfTen($scale)), + ); + + return $increase ? $comparison > 0 : $comparison < 0; + } + /** * A maximum on an integer column rounded down to the whole number it allows. Integer text and anything that * is not a finite number come back unchanged. @@ -209,7 +245,7 @@ private static function change(Operator $operator, int|string $change): Operator private static function number(mixed $value): int|float|string|null { - return \is_int($value) || \is_float($value) || (\is_string($value) && \is_numeric($value)) ? $value : null; + return \is_int($value) || (\is_float($value) && \is_finite($value)) || (\is_string($value) && \is_numeric($value)) ? $value : null; } /** @@ -301,7 +337,7 @@ private static function isBeyondRational(OperatorType $method, array $exact, int */ private static function compareWithOversized(int|string $numerator, int|float|string $limit): int { - $negative = \str_starts_with(\ltrim((string) $limit), '-'); + $negative = Decimal::isNegative($limit); if (! Decimal::isTiny($limit)) { return $negative ? 1 : -1; } diff --git a/tests/unit/Arithmetic/DecimalTest.php b/tests/unit/Arithmetic/DecimalTest.php index 86947d28a..7f7e0124f 100644 --- a/tests/unit/Arithmetic/DecimalTest.php +++ b/tests/unit/Arithmetic/DecimalTest.php @@ -88,6 +88,14 @@ public static function fractions(): iterable yield 'beyond the native integer range' => ['18446744073709551615.5', ['184467440737095516155', 1]]; yield 'too large to write out' => ['1e1000000000', null]; yield 'too close to zero to write out' => ['1e-1000000000', null]; + yield 'padded with every numeric whitespace' => [" \t\n\r\v\f-2.75 \t\n\r\v\f", [-275, 2]]; + yield 'led by a form feed' => ["\f1.5", [15, 1]]; + yield 'trailing zeros past the fraction' => ['1.5' . \str_repeat('0', 10000), [15, 1]]; + yield 'as many significant digits as are read' => ['1.' . \str_repeat('3', 127), ['1' . \str_repeat('3', 127), 127]]; + yield 'more significant digits than are read' => ['1.' . \str_repeat('3', 128), null]; + yield 'infinite' => [\INF, null]; + yield 'negative infinite' => [-\INF, null]; + yield 'not a number' => [\NAN, null]; } /** @@ -115,6 +123,29 @@ public static function orders(): iterable yield 'just past the written-out range' => ['1e-65', -65, true]; yield 'an exponent too long for an integer' => ['1e99999999999999999999', \PHP_INT_MAX, false]; yield 'a negative exponent too long for an integer' => ['1e-99999999999999999999', \PHP_INT_MIN, true]; + yield 'led by a form feed' => ["\f0.001", -3, false]; + } + + /** + * @return iterable + */ + public static function integers(): iterable + { + yield 'an integer' => [7, 7]; + yield 'integer text' => ['-7', -7]; + yield 'exponent text' => ['1e3', 1000]; + yield 'whole decimal exponent text' => ['2.50e1', 25]; + yield 'whole decimal text' => ['10.000', 10]; + yield 'exponent text beyond the native range' => ['1e19', '10000000000000000000']; + yield 'fractional text' => ['1.5', null]; + yield 'fractional exponent text' => ['1e-3', null]; + yield 'a fractional float' => [2.5, null]; + } + + #[DataProvider('integers')] + public function testOnlyAWholeNumberIsAnInteger(int|float|string $value, int|string|null $expected): void + { + $this->assertSame($expected, Decimal::parse($value)?->integer()); } #[DataProvider('orders')] diff --git a/tests/unit/Arithmetic/FractionalTest.php b/tests/unit/Arithmetic/FractionalTest.php index 27ee93b52..121307369 100644 --- a/tests/unit/Arithmetic/FractionalTest.php +++ b/tests/unit/Arithmetic/FractionalTest.php @@ -110,6 +110,72 @@ public function testAResultThatIsNotFiniteIsLeftForTheCallerToRefuse(): void $this->assertSame($operator, self::fractional(FractionalEngine::MariaDB)->operator($operator, \PHP_INT_MAX)); } + /** + * @return iterable + */ + public static function nonFiniteOperands(): iterable + { + foreach (['an infinite' => \INF, 'a negative infinite' => -\INF, 'a not-a-number' => \NAN] as $name => $value) { + yield "{$name} increment under a fractional maximum" => [new Operator(OperatorType::Increment, 'count', [$value, 10.5])]; + yield "{$name} multiplication under a fractional maximum" => [new Operator(OperatorType::Multiply, 'count', [$value, 10.5])]; + yield "{$name} division above a fractional minimum" => [new Operator(OperatorType::Divide, 'count', [$value, 0.5])]; + yield "a fractional increment under {$name} maximum" => [new Operator(OperatorType::Increment, 'count', [1.5, $value])]; + yield "a fractional multiplication under {$name} maximum" => [new Operator(OperatorType::Multiply, 'count', [1.5, $value])]; + yield "a fractional division above {$name} minimum" => [new Operator(OperatorType::Divide, 'count', [0.5, $value])]; + yield "a fractional power under {$name} maximum" => [new Operator(OperatorType::Power, 'count', [0.5, $value])]; + } + } + + #[DataProvider('nonFiniteOperands')] + public function testANonFiniteOperandOrLimitIsLeftForTheCallerToRefuse(Operator $operator): void + { + foreach (FractionalEngine::cases() as $engine) { + $this->assertSame($operator, self::fractional($engine)->operator($operator, 10), $engine->name); + } + } + + /** + * @return iterable + */ + public static function overlongOperands(): iterable + { + yield 'a factor' => [Operator::multiply('1.' . \str_repeat('3', 10000), 100)]; + yield 'a divisor' => [Operator::divide('1.' . \str_repeat('3', 10000), 0)]; + yield 'a modulus' => [Operator::modulo('1.' . \str_repeat('3', 10000))]; + yield 'a maximum' => [Operator::multiply(1.5, '15.' . \str_repeat('9', 200))]; + yield 'a minimum' => [Operator::divide(0.5, '-1.' . \str_repeat('9', 200))]; + } + + #[DataProvider('overlongOperands')] + public function testANumberWithMoreDigitsThanAnExactEngineReadIsLeftForTheCallerToRefuse(Operator $operator): void + { + $this->assertSame($operator, self::fractional(FractionalEngine::PostgreSQL)->operator($operator, 10)); + } + + public function testTrailingZerosPastTheFractionAreReadByTheirValue(): void + { + $resolved = self::fractional(FractionalEngine::PostgreSQL)->operator(Operator::multiply('1.5' . \str_repeat('0', 10000), 100), 10); + + $this->assertSame(15, self::stores(10, $resolved)); + } + + /** + * @return iterable + */ + public static function whitespacePaddedOperands(): iterable + { + yield 'a factor led by a form feed' => [Operator::multiply("\f1.5", 100), 15]; + yield 'a factor padded with every numeric whitespace' => [Operator::multiply(" \t\n\r\v\f1.5 \t\n\r\v\f", 100), 15]; + yield 'a huge negative maximum led by a form feed' => [Operator::multiply(1.5, "\f-1e1000000000"), 10]; + yield 'a tiny negative minimum led by a form feed' => [Operator::divide(0.5, "\f-1e-1000000000"), 20]; + } + + #[DataProvider('whitespacePaddedOperands')] + public function testNumericTextPaddedWithWhitespaceIsReadByItsValue(Operator $operator, int $expected): void + { + $this->assertSame($expected, self::stores(10, self::fractional(FractionalEngine::PostgreSQL)->operator($operator, 10))); + } + /** * @return iterable */ @@ -186,6 +252,9 @@ public static function wholeCounterChanges(): iterable yield 'whole float' => [2.0, 2]; yield 'integer text' => ['2', 2]; yield 'integer text beyond the native range' => ['18446744073709551615', '18446744073709551615']; + yield 'exponent text' => ['1e3', 1000]; + yield 'whole decimal exponent text' => ['2.50e1', 25]; + yield 'exponent text beyond the native range' => ['1e19', '10000000000000000000']; } #[DataProvider('wholeCounterChanges')] From 9f16e5ddd08f88374cad9c12e3de7b62687ff476 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:49:43 +1300 Subject: [PATCH 26/55] refactor(documents): move the fractional-integer mode into a Fractions trait The mode added about ten helpers to the 5000-line Documents trait. They move to Trait\Fractions, used by Database, leaving one call per write path in Documents: prepareIntegerChange() for a counter, resolveFractions() for updateDocument(), prepareFractions() and lockedFractions() for the bulk update, and resolveUpsertFractions() for upserts. Cutting array items and listing fractional keys is now one pass, and an upsert lists each document's keys once instead of again inside its transaction. A bulk update still reads each document locked on its own: no adapter reads several documents locked. The counter's bound check compared floats, so stored 9007199254740993 plus 0.5 under a maximum of 9007199254740993 passed and stored ...994; it now uses Fractional::exceeds(), exact at any size. Whole numeric text such as '1e3' is a whole change and a whole bound instead of a refusal. Co-Authored-By: Claude Opus 5.5 --- src/Database/Database.php | 1 + src/Database/Trait/Documents.php | 255 +------------------- src/Database/Trait/Fractions.php | 249 +++++++++++++++++++ tests/unit/Documents/RoundFractionsTest.php | 77 ++++++ 4 files changed, 340 insertions(+), 242 deletions(-) create mode 100644 src/Database/Trait/Fractions.php diff --git a/src/Database/Database.php b/src/Database/Database.php index 91f10ea3b..bbcd6aa48 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -56,6 +56,7 @@ class Database use Trait\Cursors; use Trait\Databases; use Trait\Documents; + use Trait\Fractions; use Trait\Indexes; use Trait\Relationships; use Trait\Transactions; diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 18006ada0..3763c23f9 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -283,175 +283,6 @@ private function integerBound(int|float|string $bound, string $name): int|string return BigInt::integralValue($bound) ?? throw new TypeException($name.' must be an integer.'); } - /** - * A bound on an integer attribute as the whole number it allows: with fractions rounded, a fractional maximum - * rounds down and a fractional minimum up. - * - * @throws TypeException - */ - private function integerLimit(int|float|string $bound, bool $maximum, bool $round): int|string - { - if ($round) { - $bound = ($maximum ? Fractional::maximum($bound) : Fractional::minimum($bound)) ?? $bound; - } - - return $this->integerBound($bound, $maximum ? 'Max' : 'Min'); - } - - /** - * With fractions rounded, whether a change on an integer attribute is answered from the stored value it - * changes: a float or decimal text change, which returns its exact result, or a whole change whose bound is - * not a whole number on an engine that refused a fractional counter. - */ - private function isExactChange(int|float|string $value, int|float|string|null $bound): bool - { - if (BigInt::integralValue($value) === null && ! Fractional::isFractional($value)) { - return false; - } - - if (! BigInt::isIntegerValue($value)) { - return true; - } - - return $bound !== null - && \is_numeric($bound) - && BigInt::integralValue($bound) === null - && $this->adapter->arithmetic()->refusesCounterFractions; - } - - /** - * @throws TypeException - */ - private function assertNumericBound(int|float|string $bound, string $name): void - { - if (! \is_numeric($bound) || (\is_float($bound) && \is_nan($bound))) { - throw new TypeException($name.' must be an integer.'); - } - } - - /** - * The whole-number change that stores what the engine stored for $value added to or taken from the stored - * value, with the exact result. The exact result is held to the bound as given, before the engine's own - * refusals, in the order database 7.4.1 checked them. - * - * @return array{int|string, float} - * - * @throws LimitException - * @throws TypeException - */ - private function exactChange(mixed $stored, int|float|string $value, int|float|string|null $bound, bool $increase): array - { - $stored ??= 0; - if (! BigInt::isIntegerValue($stored)) { - throw new TypeException('Attribute value must be an integer.'); - } - - $exact = $increase ? (float) $stored + (float) $value : (float) $stored - (float) $value; - if ($bound !== null && ($increase ? $exact > (float) $bound : $exact < (float) $bound)) { - throw new LimitException('Attribute value exceeds '.($increase ? 'maximum' : 'minimum').' limit: '.$bound); - } - - $arithmetic = $this->adapter->arithmetic(); - $change = (new Fractional($arithmetic))->counter(BigInt::toNative($stored), $value, $increase); - - if ($bound !== null && $arithmetic->refusesCounterFractions && BigInt::integralValue($bound) === null) { - throw new TypeException(($increase ? 'Max' : 'Min').' must be an integer.'); - } - - return [$change, $exact]; - } - - /** - * The keys of the single-valued integer attributes a document changes with a numeric operator whose amount or - * limit is fractional. - * - * @return list - */ - private static function fractionalKeys(Collection $collection, Document $document): array - { - $keys = []; - foreach ($collection->attributes() as $attribute) { - $value = $document->getAttribute($attribute->key); - if ($value instanceof Operator && ! $attribute->array && $attribute->isInteger() && Fractional::needsStored($value)) { - $keys[] = $attribute->key; - } - } - - return $keys; - } - - /** - * Cut each fractional item a document adds to an integer array toward zero, as every engine read it back. - */ - private static function wholeItems(Collection $collection, Document $document): void - { - foreach ($collection->attributes() as $attribute) { - $value = $document->getAttribute($attribute->key); - if ($value instanceof Operator && $attribute->array && $attribute->isInteger()) { - $document->setAttribute($attribute->key, Fractional::whole($value)); - } - } - } - - /** - * The operators under $keys answered from the stored document as the engine answered them; one the engine - * refused comes back unchanged. - * - * @param list $keys - * @return array - */ - private static function resolveFractions(Document $document, array $keys, Document $stored, Fractional $fractional): array - { - $resolved = []; - foreach ($keys as $key) { - $operator = $document->getAttribute($key); - $current = $stored->getAttribute($key); - if ($operator instanceof Operator) { - $answerable = $current === null || \is_int($current) || \is_float($current) || \is_string($current); - $resolved[$key] = ($answerable ? $fractional->operator($operator, $current) : $operator)->setAttribute($key); - } - } - - return $resolved; - } - - /** - * @param array $operators - * - * @throws StructureException when the engine refused an operator, which still carries its fraction - */ - private function assertResolvedFractions(Collection $collection, array $operators, Document $stored): void - { - if (! $this->validation()->get()) { - return; - } - - $validator = new PartialStructure( - collection: $collection, - profile: $this->profile(), - currentDocument: $stored->isEmpty() ? null : $stored, - ); - if (! $validator->isValid(new Document($operators))) { - throw new StructureException($validator->getDescription()); - } - } - - /** - * The stored document, read locked for the write's transaction under its own tenant, whatever the caller may - * read. - */ - private function readLocked(Collection $collection, Document $document): Document - { - $read = fn (): Document => $this->authorization->skip(fn () => $this->silent( - fn () => $this->readDocument($collection, $document->getId(), forUpdate: true) - )); - $tenant = $document->getTenant(); - - return $this->hasSharedTables() && $this->isTenantPerDocument() && $tenant !== null - ? $this->withTenant($tenant, $read) - : $read(); - } - /** * Cached validator instances keyed by context and a * stable schema/authorization fingerprint. @@ -1774,11 +1605,7 @@ public function updateDocument(string $collection, string $id, Document $documen $document = $this->removeUnknownAttributes($collection, $document); if ($round) { - self::wholeItems($collection, $document); - $keys = self::fractionalKeys($collection, $document); - if ($keys !== []) { - $document->setAttributes(self::resolveFractions($document, $keys, $old, new Fractional($this->adapter->arithmetic()))); - } + $this->resolveFractions($collection, $document, $old); } $shouldUpdate = false; @@ -2102,11 +1929,7 @@ public function updateDocuments( $updatedAt = $updates->getUpdatedAt(); $updates[Document::UPDATED_AT] = ($updatedAt === null || ! $this->datePreservation()->get()) ? DateTime::now() : $updatedAt; - $fractionalKeys = []; - if ($this->fractionRounding()->get()) { - self::wholeItems($collection, $updates); - $fractionalKeys = self::fractionalKeys($collection, $updates); - } + $fractionalKeys = $this->fractionRounding()->get() ? $this->prepareFractions($collection, $updates) : []; $fractional = $fractionalKeys === [] ? null : new Fractional($this->adapter->arithmetic()); $decodedUpdates = clone $updates; @@ -2203,9 +2026,7 @@ public function updateDocuments( $resolved = []; if ($fractional !== null) { - $stored = $this->readLocked($collection, $document); - $resolved = self::resolveFractions($decodedUpdates, $fractionalKeys, $stored, $fractional); - $this->assertResolvedFractions($collection, $resolved, $stored); + $resolved = $this->lockedFractions($collection, $decodedUpdates, $fractionalKeys, $document, $fractional); $signature = \serialize(\array_map(static fn (Operator $operator): array => $operator->toArray(), $resolved)); $groups[$signature]['operators'] = $resolved; $groups[$signature]['indexes'][] = $index; @@ -2449,18 +2270,14 @@ private function upsert( )); $existing = $this->findDocumentsToUpsert($collection->getId(), $documents); $round = $this->fractionRounding()->get(); - $fractionalIds = []; + $fractionalKeys = []; foreach ($documents as $key => $document) { $old = $existing[$this->upsertKey($document)] ?? new Document(); $document = $this->removeUnknownAttributes($collection, $document); - $fractions = false; - if ($round) { - self::wholeItems($collection, $document); - $fractions = self::fractionalKeys($collection, $document) !== []; - } + $keys = $round ? $this->prepareFractions($collection, $document) : []; // Extract operators early to avoid comparison issues $documentArray = $document->getArrayCopy(); @@ -2590,7 +2407,7 @@ private function upsert( collection: $collection, profile: $this->profile(), currentDocument: $old->isEmpty() ? null : $old, - fractions: $fractions, + fractions: $keys !== [], ); if (! $validator->isValid($document)) { @@ -2622,8 +2439,8 @@ private function upsert( if (! empty($operators)) { $operatorIds[$identity] = true; } - if ($fractions) { - $fractionalIds[$identity] = true; + if ($keys !== []) { + $fractionalKeys[$identity] = $keys; } $old = $this->castBefore($collection, $old); $document = $this->castBefore($collection, $document); @@ -2654,15 +2471,15 @@ private function upsert( $batch = $this->withMutation( Event::DocumentsUpsert, \array_map(static fn (Change $change): Document => $change->new, $chunk), - function () use ($collection, $increase, $chunk, $fractionalIds): array { + function () use ($collection, $increase, $chunk, $fractionalKeys): array { if (! $this->adapterHasFeature(Feature\Upserts::class)) { throw new DatabaseException('Adapter does not support upserts'); } $adapter = $this->adapter; - if ($fractionalIds !== []) { - $this->resolveUpsertFractions($collection, $chunk, $fractionalIds); + if ($fractionalKeys !== []) { + $this->resolveUpsertFractions($collection, $chunk, $fractionalKeys); } $batch = $this->authorization->skip(fn () => $adapter->upsertDocuments( @@ -2738,30 +2555,6 @@ function () use ($collection, $increase, $chunk, $fractionalIds): array { return [$created, $updated]; } - /** - * Answer the fractional operators of the upserted documents under $identities from their stored versions, read - * locked in the write's transaction; a document not stored yet is answered from zero. - * - * @param array $chunk - * @param array $identities - * - * @throws StructureException when the engine refused an operator - */ - private function resolveUpsertFractions(Collection $collection, array $chunk, array $identities): void - { - $fractional = new Fractional($this->adapter->arithmetic()); - foreach ($chunk as $change) { - if (! isset($identities[$this->getDocumentIdentity($change->new)])) { - continue; - } - - $stored = $this->readLocked($collection, $change->new); - $resolved = self::resolveFractions($change->new, self::fractionalKeys($collection, $change->new), $stored, $fractional); - $this->assertResolvedFractions($collection, $resolved, $stored); - $change->new->setAttributes($this->castBefore($collection, new Document($resolved))->getArrayCopy()); - } - } - /** * Load the stored documents an upsert batch will be compared against, in one * read per tenant instead of one per document. A batch of N documents costs a @@ -2856,18 +2649,7 @@ public function increaseDocumentAttribute( $exact = false; if ($this->isDeclaredInteger($numericAttribute ?? $this->declaredAttribute($collection, $attribute))) { - $round = $this->fractionRounding()->get(); - $exact = $round && $this->isExactChange($value, $max); - if ($exact) { - if ($max !== null) { - $this->assertNumericBound($max, 'Max'); - } - } else { - $this->assertIntegerChange($value); - if ($max !== null) { - $max = $this->integerLimit($max, true, $round); - } - } + [$exact, $max] = $this->prepareIntegerChange($value, $max, true); } $cacheTarget = $collection->getId() === self::METADATA @@ -2994,18 +2776,7 @@ public function decreaseDocumentAttribute( $exact = false; if ($this->isDeclaredInteger($numericAttribute ?? $this->declaredAttribute($collection, $attribute))) { - $round = $this->fractionRounding()->get(); - $exact = $round && $this->isExactChange($value, $min); - if ($exact) { - if ($min !== null) { - $this->assertNumericBound($min, 'Min'); - } - } else { - $this->assertIntegerChange($value); - if ($min !== null) { - $min = $this->integerLimit($min, false, $round); - } - } + [$exact, $min] = $this->prepareIntegerChange($value, $min, false); } $cacheTarget = $collection->getId() === self::METADATA diff --git a/src/Database/Trait/Fractions.php b/src/Database/Trait/Fractions.php new file mode 100644 index 000000000..5a3427a87 --- /dev/null +++ b/src/Database/Trait/Fractions.php @@ -0,0 +1,249 @@ +fractionRounding()->get(); + if ($round && $this->isExactChange($value, $bound)) { + if ($bound !== null) { + $this->assertNumericBound($bound, $increase ? 'Max' : 'Min'); + } + + return [true, $bound]; + } + + $this->assertIntegerChange($value); + + return [false, $bound === null ? null : $this->integerLimit($bound, $increase, $round)]; + } + + /** + * A whole change whose bound is not whole is answered from the stored value only where the engine refused a + * fractional counter, which refused that bound after the limit check. + */ + private function isExactChange(int|float|string $value, int|float|string|null $bound): bool + { + if (Fractional::wholeValue($value) === null && ! Fractional::isFractional($value)) { + return false; + } + + if (! BigInt::isIntegerValue($value)) { + return true; + } + + return $bound !== null + && \is_numeric($bound) + && Fractional::wholeValue($bound) === null + && $this->adapter->arithmetic()->refusesCounterFractions; + } + + /** + * @throws TypeException + */ + private function integerLimit(int|float|string $bound, bool $maximum, bool $round): int|string + { + if ($round) { + $bound = ($maximum ? Fractional::maximum($bound) : Fractional::minimum($bound)) ?? $bound; + } + + return $this->integerBound($bound, $maximum ? 'Max' : 'Min'); + } + + /** + * @throws TypeException + */ + private function assertNumericBound(int|float|string $bound, string $name): void + { + if (! \is_numeric($bound) || (\is_float($bound) && \is_nan($bound))) { + throw new TypeException($name.' must be an integer.'); + } + } + + /** + * The whole-number change to apply and the exact result to return. The bound is checked before the engine's + * own refusals, in the order database 7.4.1 checked them. + * + * @return array{int|string, float} + * + * @throws LimitException + * @throws TypeException + */ + private function exactChange(mixed $stored, int|float|string $value, int|float|string|null $bound, bool $increase): array + { + $stored ??= 0; + if (! BigInt::isIntegerValue($stored)) { + throw new TypeException('Attribute value must be an integer.'); + } + $stored = BigInt::toNative($stored); + + if ($bound !== null && Fractional::exceeds($stored, $value, $bound, $increase)) { + throw new LimitException('Attribute value exceeds '.($increase ? 'maximum' : 'minimum').' limit: '.$bound); + } + + $arithmetic = $this->adapter->arithmetic(); + $change = (new Fractional($arithmetic))->counter($stored, $value, $increase); + + if ($bound !== null && $arithmetic->refusesCounterFractions && Fractional::wholeValue($bound) === null) { + throw new TypeException(($increase ? 'Max' : 'Min').' must be an integer.'); + } + + return [$change, $increase ? (float) $stored + (float) $value : (float) $stored - (float) $value]; + } + + /** + * Cuts the fractional items a document adds to an integer array toward zero, as every engine read them back, + * and lists the keys of its operators on single integers that need the stored value. + * + * @return list + */ + private function prepareFractions(Collection $collection, Document $document): array + { + $keys = []; + foreach ($collection->attributes() as $attribute) { + $value = $document->getAttribute($attribute->key); + if (! $value instanceof Operator || ! $attribute->isInteger()) { + continue; + } + + if ($attribute->array) { + $document->setAttribute($attribute->key, Fractional::whole($value)); + } elseif (Fractional::needsStored($value)) { + $keys[] = $attribute->key; + } + } + + return $keys; + } + + /** + * Answers a document's fractional operators from the stored document it changes; strict validation refuses + * any the engine refused. + */ + private function resolveFractions(Collection $collection, Document $document, Document $stored): void + { + $keys = $this->prepareFractions($collection, $document); + if ($keys !== []) { + $document->setAttributes(self::answerFractions($document, $keys, $stored, new Fractional($this->adapter->arithmetic()))); + } + } + + /** + * The operators of $changes under $keys answered from the document $target names, read locked for the write; + * a document not stored yet is answered from zero. Each document is its own locked read: no adapter reads + * several documents locked. + * + * @param list $keys + * @return array + * + * @throws StructureException when the engine refused an operator + */ + private function lockedFractions(Collection $collection, Document $changes, array $keys, Document $target, Fractional $fractional): array + { + $stored = $this->readLocked($collection, $target); + $resolved = self::answerFractions($changes, $keys, $stored, $fractional); + $this->assertResolvedFractions($collection, $resolved, $stored); + + return $resolved; + } + + /** + * @param array $chunk + * @param array> $keys The fractional keys of each upserted document, by its identity + * + * @throws StructureException when the engine refused an operator + */ + private function resolveUpsertFractions(Collection $collection, array $chunk, array $keys): void + { + $fractional = new Fractional($this->adapter->arithmetic()); + foreach ($chunk as $change) { + $documentKeys = $keys[$this->getDocumentIdentity($change->new)] ?? []; + if ($documentKeys === []) { + continue; + } + + $resolved = $this->lockedFractions($collection, $change->new, $documentKeys, $change->new, $fractional); + $change->new->setAttributes($this->castBefore($collection, new Document($resolved))->getArrayCopy()); + } + } + + /** + * @param list $keys + * @return array + */ + private static function answerFractions(Document $changes, array $keys, Document $stored, Fractional $fractional): array + { + $resolved = []; + foreach ($keys as $key) { + $operator = $changes->getAttribute($key); + $current = $stored->getAttribute($key); + if ($operator instanceof Operator) { + $answerable = $current === null || \is_int($current) || \is_float($current) || \is_string($current); + $resolved[$key] = ($answerable ? $fractional->operator($operator, $current) : $operator)->setAttribute($key); + } + } + + return $resolved; + } + + /** + * @param array $operators + * + * @throws StructureException + */ + private function assertResolvedFractions(Collection $collection, array $operators, Document $stored): void + { + if (! $this->validation()->get()) { + return; + } + + $validator = new PartialStructure( + collection: $collection, + profile: $this->profile(), + currentDocument: $stored->isEmpty() ? null : $stored, + ); + if (! $validator->isValid(new Document($operators))) { + throw new StructureException($validator->getDescription()); + } + } + + /** + * Read under the document's own tenant and whatever the caller may read. + */ + private function readLocked(Collection $collection, Document $document): Document + { + $read = fn (): Document => $this->authorization->skip(fn () => $this->silent( + fn () => $this->readDocument($collection, $document->getId(), forUpdate: true) + )); + $tenant = $document->getTenant(); + + return $this->hasSharedTables() && $this->isTenantPerDocument() && $tenant !== null + ? $this->withTenant($tenant, $read) + : $read(); + } +} diff --git a/tests/unit/Documents/RoundFractionsTest.php b/tests/unit/Documents/RoundFractionsTest.php index 266fe0265..fbd5d678b 100644 --- a/tests/unit/Documents/RoundFractionsTest.php +++ b/tests/unit/Documents/RoundFractionsTest.php @@ -342,6 +342,83 @@ public function testAFractionalChangePastItsBoundIsRefusedAndRolledBack(bool $in $this->assertSame(5, $this->stored($database, 'count')); } + /** + * @return iterable + */ + public static function boundsBeyondFloatPrecision(): iterable + { + yield 'an increase past a maximum' => [true, 9007199254740993, null]; + yield 'an increase past a maximum given as text' => [true, '9007199254740993', null]; + yield 'a decrease past a minimum' => [false, 9007199254740993, null]; + yield 'an increase within a maximum' => [true, 9007199254740994, 9007199254740994]; + yield 'a decrease within a minimum' => [false, 9007199254740992, 9007199254740992]; + } + + #[DataProvider('boundsBeyondFloatPrecision')] + public function testAFractionalChangeIsHeldToABoundBeyondFloatPrecisionExactly(bool $increase, int|string $bound, int|string|null $stored): void + { + $database = $this->database(); + $this->create($database, ['big' => 9007199254740993]); + $change = fn (): Document => $increase + ? $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'big', 0.5, $bound) + : $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'big', 0.5, $bound); + + if ($stored === null) { + $this->assertLimited($change); + $this->assertSame(9007199254740993, $this->stored($database, 'big')); + + return; + } + + $change(); + $this->assertSame($stored, $this->stored($database, 'big')); + } + + /** + * @return iterable + */ + public static function engines(): iterable + { + foreach (FractionalEngine::cases() as $engine) { + yield $engine->name => [$engine]; + } + } + + #[DataProvider('engines')] + public function testAWholeChangeWrittenWithAnExponentChangesByItsValue(FractionalEngine $engine): void + { + $database = $this->database(self::engine($engine)); + $this->create($database, ['count' => 5]); + + $this->assertSame(1005.0, $database->increaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', '1e3', '2e3')->getAttribute('count')); + $this->assertSame(980.0, $database->decreaseDocumentAttribute(self::COLLECTION, self::DOCUMENT, 'count', '2.50e1')->getAttribute('count')); + $this->assertSame(980, $this->stored($database, 'count')); + } + + /** + * @return iterable + */ + public static function nonFiniteOperators(): iterable + { + foreach (FractionalEngine::cases() as $engine) { + yield "{$engine->name}: an infinite factor under a fractional maximum" => [$engine, Operator::multiply(\INF, 10.5)]; + yield "{$engine->name}: a fractional factor under a not-a-number maximum" => [$engine, Operator::multiply(1.5, \NAN)]; + yield "{$engine->name}: a fractional increment under an infinite maximum" => [$engine, Operator::increment(1.5, \INF)]; + yield "{$engine->name}: a fractional decrement above a negative infinite minimum" => [$engine, Operator::decrement(1.5, -\INF)]; + } + } + + #[DataProvider('nonFiniteOperators')] + public function testANonFiniteOperandOrLimitIsRefused(FractionalEngine $engine, Operator $operator): void + { + $database = $this->database(self::engine($engine)); + $this->create($database, ['count' => 10]); + + $this->assertStructureRefused(fn () => $database->updateDocument(self::COLLECTION, self::DOCUMENT, new Document(['count' => $operator]))); + $this->assertStructureRefused(fn () => $database->updateDocuments(self::COLLECTION, new Document(['count' => clone $operator]))); + $this->assertSame(10, $this->stored($database, 'count')); + } + public function testAFractionalChangeOnAMissingDocumentIsNotFound(): void { $this->expectException(NotFoundException::class); From ba176739855446f69f936f41500de13f07f44d71 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:49:56 +1300 Subject: [PATCH 27/55] refactor(arithmetic): name the exactly computed operators exactOperators $exact read as a flag; the list names the operators the engine computed exactly, beside refusedWithoutLimit. Co-Authored-By: Claude Opus 5.5 --- src/Database/Adapter/Postgres.php | 2 +- src/Database/Arithmetic.php | 4 ++-- src/Database/Arithmetic/Fractional.php | 2 +- tests/unit/Adapter/ArithmeticTest.php | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 54968eec7..f1d8cfb4c 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -89,7 +89,7 @@ public function arithmetic(): Arithmetic division: Rounding::HalfAwayFromZero, power: Rounding::HalfEven, refusesCounterFractions: true, - exact: [OperatorType::Multiply, OperatorType::Divide, OperatorType::Modulo], + exactOperators: [OperatorType::Multiply, OperatorType::Divide, OperatorType::Modulo], refusedWithoutLimit: [OperatorType::Increment, OperatorType::Decrement, OperatorType::Multiply, OperatorType::Divide], ); } diff --git a/src/Database/Arithmetic.php b/src/Database/Arithmetic.php index 9d8a716bf..bc39fd642 100644 --- a/src/Database/Arithmetic.php +++ b/src/Database/Arithmetic.php @@ -13,7 +13,7 @@ final readonly class Arithmetic { /** - * @param list $exact + * @param list $exactOperators * @param list $refusedWithoutLimit */ public function __construct( @@ -22,7 +22,7 @@ public function __construct( public Rounding $division, public Rounding $power, public bool $refusesCounterFractions = false, - public array $exact = [], + public array $exactOperators = [], public array $refusedWithoutLimit = [], ) { } diff --git a/src/Database/Arithmetic/Fractional.php b/src/Database/Arithmetic/Fractional.php index aae610bf8..e30a1f16f 100644 --- a/src/Database/Arithmetic/Fractional.php +++ b/src/Database/Arithmetic/Fractional.php @@ -102,7 +102,7 @@ public function operator(Operator $operator, int|float|string|null $stored): Ope } $rounding = $this->arithmetic->rounding($method, $whole); - if (\in_array($method, $this->arithmetic->exact, true)) { + if (\in_array($method, $this->arithmetic->exactOperators, true)) { $exact = self::rational($method, $current, $amount); if ($exact === null || ($limit !== null && Decimal::isOverlong($limit))) { return $operator; diff --git a/tests/unit/Adapter/ArithmeticTest.php b/tests/unit/Adapter/ArithmeticTest.php index b2fb0423c..a0b35c10d 100644 --- a/tests/unit/Adapter/ArithmeticTest.php +++ b/tests/unit/Adapter/ArithmeticTest.php @@ -126,7 +126,7 @@ private static function describe(Arithmetic $arithmetic): array $arithmetic->division, $arithmetic->power, $arithmetic->refusesCounterFractions, - $arithmetic->exact, + $arithmetic->exactOperators, $arithmetic->refusedWithoutLimit, ]; } From 24c4ebb555f5e1f9abb1c210f103c45a2dcd278a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:50:17 +1300 Subject: [PATCH 28/55] fix(mirror): check grants on the source only The destination replays writes the source already allowed. Forwarding the grant check to it judged related documents against the destination's own copies, so whenever the destination was behind (a related document not replicated yet) it refused a write the source made; the refusal went to onError() and the destination silently drifted. setCheckGrants() and withCheckGrants() now reach the source only, and replicate() runs the destination write with the check off, as the snapshot it applies carries the caller's toggle. Co-Authored-By: Claude Opus 5.5 --- src/Database/Mirror.php | 23 ++++++++++--------- tests/unit/MirrorReplicationTest.php | 33 ++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php index ef8a71a7c..c8bb9ea87 100644 --- a/src/Database/Mirror.php +++ b/src/Database/Mirror.php @@ -490,10 +490,14 @@ public function setReplacePlaceholders(bool $replace): static return parent::setReplacePlaceholders($replace); } + /** + * Only the source checks grants: the destination replays what the source already allowed, and judging it against + * the destination's own copies would refuse writes the source made whenever the destination is behind. + */ #[\Override] public function setCheckGrants(bool $check): static { - $this->delegate(__FUNCTION__, \func_get_args()); + $this->source->setCheckGrants($check); return parent::setCheckGrants($check); } @@ -566,10 +570,7 @@ public function withReplacePlaceholders(bool $replace, callable $callback): mixe #[\Override] public function withCheckGrants(bool $check, callable $callback): mixed { - $destination = $this->destination; - $scoped = $destination === null ? $callback : fn (): mixed => $destination->withCheckGrants($check, $callback); - - return parent::withCheckGrants($check, fn (): mixed => $this->source->withCheckGrants($check, $scoped)); + return parent::withCheckGrants($check, fn (): mixed => $this->source->withCheckGrants($check, $callback)); } #[\Override] @@ -1701,10 +1702,10 @@ private function decorating(Event $event, string $collection, ?callable $onNext) /** * Applies a write to the destination under the authorization, relationship, silence, tenant and toggle state the - * caller has at the time of the call, without its request timestamp (the source checked it), and reports a - * failure through onError(). Inside a coroutine the write runs in a coroutine of its own, in order with every other - * destination change made through the mirror (see inOrder()); outside one it runs before this returns, since a - * task that yields outside a scheduler never resumes. + * caller has at the time of the call, without its request timestamp or grant check (the source checked both), + * and reports a failure through onError(). Inside a coroutine the write runs in a coroutine of its own, in order + * with every other destination change made through the mirror (see inOrder()); outside one it runs before this + * returns, since a task that yields outside a scheduler never resumes. * * @param Closure(): void $write */ @@ -1719,7 +1720,9 @@ private function replicate(string $action, Closure $write): void $apply = function () use ($action, $destination, $snapshot, $write): void { try { $destination->withSnapshot($snapshot, function () use ($destination, $write): void { - $destination->withRequestTimestamp(null, $write); + $destination->withCheckGrants(false, function () use ($destination, $write): void { + $destination->withRequestTimestamp(null, $write); + }); }); } catch (Throwable $error) { $this->logError($action, $error); diff --git a/tests/unit/MirrorReplicationTest.php b/tests/unit/MirrorReplicationTest.php index d71e5268e..fdf2b1720 100644 --- a/tests/unit/MirrorReplicationTest.php +++ b/tests/unit/MirrorReplicationTest.php @@ -400,6 +400,39 @@ public function testADocumentWrittenThroughARelationshipIsNotOvertakenByALaterWr $this->assertFalse($this->destination->getDocument(self::PARENTS, 'parent')->isEmpty()); } + public function testTheGrantCheckRunsOnTheSourceOnly(): void + { + $this->relate(); + $kept = [Permission::read(Role::any()), Permission::update(Role::any()), Permission::read(Role::user('alice'))]; + $source = $this->mirror->getSource(); + $child = static fn (string $id): Document => new Document([Document::ID => $id, 'title' => $id, Document::PERMISSIONS => $kept]); + $this->authorization->skip(function () use ($source, $child): void { + $source->createDocument(self::CHILDREN, $child('single child')); + $source->createDocument(self::CHILDREN, $child('batched child')); + }); + + $this->mirror->withCheckGrants(true, fn (): Document => $this->mirror->createDocument(self::PARENTS, new Document([ + Document::ID => 'single', + 'title' => 'single', + 'children' => [$child('single child')], + ]))); + $this->mirror->setCheckGrants(true); + $this->mirror->createDocuments(self::PARENTS, [new Document([ + Document::ID => 'batched', + 'title' => 'batched', + 'children' => [$child('batched child')], + ])]); + + $this->assertSame([], $this->errors); + foreach (['single child', 'batched child'] as $id) { + $replicated = $this->authorization->skip(fn (): Document => $this->destination->getDocument(self::CHILDREN, $id)); + $this->assertSame($kept, $replicated->getPermissions(), $id); + } + $this->assertFalse($this->authorization->skip(fn (): Document => $this->destination->getDocument(self::PARENTS, 'single'))->isEmpty()); + $this->assertFalse($this->authorization->skip(fn (): Document => $this->destination->getDocument(self::PARENTS, 'batched'))->isEmpty()); + $this->assertSame([true, true, false], [$this->mirror->isCheckingGrants(), $source->isCheckingGrants(), $this->destination->isCheckingGrants()]); + } + /** * @return iterable */ From 18e0cfe51119edc68e99a1fe08105f1da8584347 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:50:17 +1300 Subject: [PATCH 29/55] refactor(state): pass every snapshot field explicitly Snapshot is only built by Database::snapshot(), which 8.0 documents as internal, so its constructor is not public API. Defaulting the three 8.1 toggles to false let another construction site (CoroutineStateTest) drop them silently; without defaults a site that forgets one fails to construct. The class is marked @internal to match the documented status. Co-Authored-By: Claude Opus 5.5 --- src/Database/State/Snapshot.php | 8 +++++--- tests/unit/State/CoroutineStateTest.php | 3 +++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Database/State/Snapshot.php b/src/Database/State/Snapshot.php index be301923b..45fe2f50f 100644 --- a/src/Database/State/Snapshot.php +++ b/src/Database/State/Snapshot.php @@ -8,6 +8,8 @@ * The scoped state a database handle's reads depend on, as one coroutine sees it. Taken with * Database::snapshot() and applied with Database::withSnapshot(), it lets work started in another coroutine run * under its caller's state, however long the caller keeps that state. + * + * @internal */ final readonly class Snapshot { @@ -47,9 +49,9 @@ public function __construct( public bool $preserveSequence, public bool $ignoreDuplicates, public ?DateTime $requestTimestamp, - public bool $replacePlaceholders = false, - public bool $checkGrants = false, - public bool $roundFractions = false, + public bool $replacePlaceholders, + public bool $checkGrants, + public bool $roundFractions, ) { } } diff --git a/tests/unit/State/CoroutineStateTest.php b/tests/unit/State/CoroutineStateTest.php index 5e89d68c8..a809fdfe7 100644 --- a/tests/unit/State/CoroutineStateTest.php +++ b/tests/unit/State/CoroutineStateTest.php @@ -258,6 +258,9 @@ private function snapshot(bool $population): Snapshot preserveSequence: false, ignoreDuplicates: false, requestTimestamp: null, + replacePlaceholders: false, + checkGrants: false, + roundFractions: false, ); } From 5d3b5c4a8ae11d33ca26785b0be3dfe81905f65d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:50:36 +1300 Subject: [PATCH 30/55] fix(relationships): read granted permissions in the write's transaction The grant check compared a related document's permissions with the stored ones read before createDocument() and updateDocument() opened their transaction, and without a lock, so a change committed in between was not seen by the write that relied on it. Both writes now prepare their related documents inside the mutation, after the parent row is locked, and read the stored permissions with forUpdate, holding them until the related write commits. Each attempt prepares a copy of the document given, so a retried transaction starts from the caller's document rather than from the copies an earlier attempt changed. The hook takes the toggles, the lock and the staged view as one Preparation instead of reading the database's toggles itself, so the write decides. createDocuments(), upsertDocument() and upsertDocuments() write related documents before the parents' transaction, so they still read without a lock; setCheckGrants() documents that limit. Narrating docblocks on the private preparation helpers are trimmed. Co-Authored-By: Claude Opus 5.5 --- src/Database/Database.php | 5 +++ src/Database/Hook/Preparation.php | 28 ++++++++++++ src/Database/Hook/Relationships.php | 63 +++++++++----------------- src/Database/Trait/Documents.php | 68 +++++++++++++++-------------- 4 files changed, 90 insertions(+), 74 deletions(-) create mode 100644 src/Database/Hook/Preparation.php diff --git a/src/Database/Database.php b/src/Database/Database.php index 91f10ea3b..0b9a5c569 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -1200,6 +1200,11 @@ public function withReplacePlaceholders(bool $replace, callable $callback): mixe * has may be sent back, and permissions it inherits from its parent are never checked. Malformed permissions * throw a structure exception and an unheld role an authorization exception, before anything is written. * + * createDocument() and updateDocument() read the permissions a related document already has locked, in the + * transaction that writes it. createDocuments(), upsertDocument() and upsertDocuments() write related documents + * before the transaction of the documents relating them, so they read them without a lock: a change committed + * between the check and the related write is not seen. A Mirror checks on its source only. + * * @return $this */ public function setCheckGrants(bool $check): static diff --git a/src/Database/Hook/Preparation.php b/src/Database/Hook/Preparation.php new file mode 100644 index 000000000..5b115ed8c --- /dev/null +++ b/src/Database/Hook/Preparation.php @@ -0,0 +1,28 @@ +database->isReplacingPlaceholders(); - $check = $this->database->isCheckingGrants() && $this->database->getAuthorization()->getStatus(); - if ((! $replace && ! $check) || isset($this->writeStacks[$this->coroutine()])) { + if (isset($this->writeStacks[$this->coroutine()])) { return $document; } - foreach ($this->prepareNestedValues($collection, $document, [], $replace, $check, $resolve) as $key => $value) { + foreach ($this->prepareNestedValues($collection, $document, [], $preparation) as $key => $value) { $document->setAttribute($key, $value); } @@ -3201,18 +3194,14 @@ public function prepareNested(Document $collection, Document $document, ?Closure } /** - * The relationship values of $document that hold related documents as documents, with copies of those - * documents prepared. - * * @param list $writeStack The collections of the writes relating $document, innermost last - * @param (Closure(string, string): ?Document)|null $resolve * @return array * * @throws AuthorizationException * @throws RelationshipException * @throws StructureException */ - private function prepareNestedValues(Document $collection, Document $document, array $writeStack, bool $replace, bool $check, ?Closure $resolve): array + private function prepareNestedValues(Document $collection, Document $document, array $writeStack, Preparation $preparation): array { $depth = \count($writeStack); $prepared = []; @@ -3242,9 +3231,7 @@ private function prepareNestedValues(Document $collection, Document $document, a $relatedCollection, $nested instanceof Document ? clone $nested : self::toDocument($nested), [...$writeStack, $collection->getId()], - $replace, - $check, - $resolve, + $preparation, ); } @@ -3256,26 +3243,25 @@ private function prepareNestedValues(Document $collection, Document $document, a /** * @param list $writeStack - * @param (Closure(string, string): ?Document)|null $resolve * * @throws AuthorizationException * @throws RelationshipException * @throws StructureException */ - private function prepareNestedDocument(Document $collection, Document $document, array $writeStack, bool $replace, bool $check, ?Closure $resolve): Document + private function prepareNestedDocument(Document $collection, Document $document, array $writeStack, Preparation $preparation): Document { $id = $document->getId(); $new = $id === ''; - if ($replace && ($new || Id::isPlaceholder($id))) { + if ($preparation->replace && ($new || Id::isPlaceholder($id))) { $document->setAttribute(Document::ID, Id::unique()); $new = true; } - if ($check) { - $this->authorizeGrants($collection, $document, $new, $resolve); + if ($preparation->check) { + $this->authorizeGrants($collection, $document, $new, $preparation); } - foreach ($this->prepareNestedValues($collection, $document, $writeStack, $replace, $check, $resolve) as $key => $value) { + foreach ($this->prepareNestedValues($collection, $document, $writeStack, $preparation) as $key => $value) { $document->setAttribute($key, $value); } @@ -3283,14 +3269,10 @@ private function prepareNestedDocument(Document $collection, Document $document, } /** - * Refuse permissions $document adds over its current ones for a role the caller does not hold. - * - * @param (Closure(string, string): ?Document)|null $resolve - * * @throws AuthorizationException * @throws StructureException */ - private function authorizeGrants(Document $collection, Document $document, bool $new, ?Closure $resolve): void + private function authorizeGrants(Document $collection, Document $document, bool $new, Preparation $preparation): void { $permissions = $document->getAttribute(Document::PERMISSIONS); if ($permissions === null) { @@ -3302,7 +3284,7 @@ private function authorizeGrants(Document $collection, Document $document, bool throw new StructureException($validator->getDescription()); } - $current = $new ? [] : (Permission::aggregate($this->grantedView($collection, $document->getId(), $resolve)->getPermissions()) ?? []); + $current = $new ? [] : (Permission::aggregate($this->grantedView($collection, $document->getId(), $preparation)->getPermissions()) ?? []); $authorization = $this->database->getAuthorization(); foreach (\array_diff(Permission::aggregate($document->getPermissions()) ?? [], $current) as $granted) { @@ -3315,17 +3297,14 @@ private function authorizeGrants(Document $collection, Document $document, bool } /** - * The related document $id as a write would find it, whether or not the caller can read it: the document - * $resolve gives for it, else the stored one. - * - * @param (Closure(string, string): ?Document)|null $resolve + * Read whether or not the caller can read the document, as the write that changes it will find it. */ - private function grantedView(Document $collection, string $id, ?Closure $resolve): Document + private function grantedView(Document $collection, string $id, Preparation $preparation): Document { - $staged = $resolve === null ? null : $resolve($collection->getId(), $id); + $staged = $preparation->resolve === null ? null : ($preparation->resolve)($collection->getId(), $id); return $staged ?? $this->database->getAuthorization()->skip(fn (): Document => $this->database->skipRelationships( - fn (): Document => $this->database->getDocument($collection->getId(), $id, [Query::select([Document::PERMISSIONS])]) + fn (): Document => $this->database->getDocument($collection->getId(), $id, [Query::select([Document::PERMISSIONS])], $preparation->lock) )); } diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 18006ada0..e8e79c388 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -39,6 +39,7 @@ use Utopia\Database\Exception\Timeout as TimeoutException; use Utopia\Database\Exception\Type as TypeException; use Utopia\Database\Hook\Lifecycle; +use Utopia\Database\Hook\Preparation; use Utopia\Database\Id; use Utopia\Database\Operator; use Utopia\Database\PermissionType; @@ -1244,7 +1245,7 @@ public function authorizeCreate(string $collection, Document $document, ?callabl } if ($this->relationshipHook?->isEnabled()) { - $this->relationshipHook->authorize($definition, $this->prepareNestedCopy($definition, $document, $resolve), resolve: $resolve); + $this->relationshipHook->authorize($definition, $this->prepareNestedCopy($definition, $document, $this->nestedPreparation($resolve)), resolve: $resolve); } } @@ -1277,7 +1278,7 @@ public function authorizeUpdate(string $collection, Document $old, Document $doc } if ($this->relationshipHook?->isEnabled()) { - $this->relationshipHook->authorize($definition, $this->prepareNestedCopy($definition, $document, $resolve), $old, $resolve); + $this->relationshipHook->authorize($definition, $this->prepareNestedCopy($definition, $document, $this->nestedPreparation($resolve)), $old, $resolve); } } @@ -1296,18 +1297,19 @@ public function createDocument(string $collection, Document $document): Document $collection = $this->silent(fn () => $this->getCollection($collection)); - $document = $this->prepareNested($collection, $document); $document = $this->prepareDocument($collection, $document); + $preparation = $this->nestedPreparation(lock: true); /** @var array}> $copies */ $copies = []; try { - $document = $this->withMutation(Event::DocumentCreate, $document, function () use ($collection, $document, &$copies) { + $document = $this->withMutation(Event::DocumentCreate, $document, function () use ($collection, $document, $preparation, &$copies) { $hook = $this->relationshipHook; if ($hook?->isEnabled()) { if ($copies !== [] && $copies !== null) { $hook->restore($copies); } + $document = $this->prepareNestedCopy($collection, $document, $preparation); $document = $this->silent(function () use ($hook, $collection, $document, &$copies): Document { return $hook->afterDocumentCreate($collection, $document, $copies); }); @@ -1405,56 +1407,56 @@ public function createPrepared(array $documents): void }); } - /** - * Whether a document given $id is created under a generated ID. - */ private function generatesId(string $id): bool { return empty($id) || (Id::isPlaceholder($id) && $this->placeholderReplacement()->get()); } - private function preparesNested(): bool + /** + * Null when the write leaves its related documents as given. + * + * @param (callable(string, string): ?Document)|null $resolve + */ + private function nestedPreparation(?callable $resolve = null, bool $lock = false): ?Preparation { - return $this->placeholderReplacement()->get() || $this->grantChecking()->get(); + $replace = $this->placeholderReplacement()->get(); + $check = $this->grantChecking()->get() && $this->authorization->getStatus(); + if ((! $replace && ! $check) || ! $this->relationshipHook?->isEnabled()) { + return null; + } + + return new Preparation($replace, $check, $lock, $resolve === null ? null : $resolve(...)); } /** - * The document with its related documents prepared by the relationship hook, when placeholders are replaced or - * grants checked. - * - * @param (Closure(string, string): ?Document)|null $resolve - * * @throws AuthorizationException * @throws RelationshipException * @throws StructureException */ - private function prepareNested(Document $collection, Document $document, ?Closure $resolve = null): Document + private function prepareNested(Document $collection, Document $document, Preparation $preparation): Document { $hook = $this->relationshipHook; - if ($hook === null || ! $this->preparesNested() || ! $hook->isEnabled()) { + if ($hook === null) { return $document; } - return $this->silent(fn (): Document => $hook->prepareNested($collection, $document, $resolve)); + return $this->silent(fn (): Document => $hook->prepareNested($collection, $document, $preparation)); } /** - * A copy of $document as prepareNested() prepares it, leaving $document unchanged; $document itself when - * neither placeholders are replaced nor grants checked. - * - * @param (callable(string, string): ?Document)|null $resolve + * Prepares a copy, so a write that runs again, as a retried transaction does, starts from the document given. * * @throws AuthorizationException * @throws RelationshipException * @throws StructureException */ - private function prepareNestedCopy(Document $collection, Document $document, ?callable $resolve): Document + private function prepareNestedCopy(Document $collection, Document $document, ?Preparation $preparation): Document { - if (! $this->preparesNested()) { + if ($preparation === null) { return $document; } - return $this->prepareNested($collection, clone $document, $resolve === null ? null : $resolve(...)); + return $this->prepareNested($collection, clone $document, $preparation); } /** @@ -1596,9 +1598,10 @@ public function createDocuments( $modified = 0; $hasRelationships = self::relationshipAttributes($collection) !== []; - if ($hasRelationships && $this->preparesNested()) { + $preparation = $hasRelationships ? $this->nestedPreparation() : null; + if ($preparation !== null) { foreach ($documents as $index => $document) { - $documents[$index] = $this->prepareNested($collection, $document); + $documents[$index] = $this->prepareNested($collection, $document, $preparation); } } @@ -1726,20 +1729,21 @@ public function updateDocument(string $collection, string $id, Document $documen } $collection = $this->silent(fn () => $this->getCollection($collection)); - $document = $this->prepareNested($collection, $document); + $preparation = $this->nestedPreparation(lock: true); $newUpdatedAt = $document->getUpdatedAt(); $hasOperators = false; $round = $this->fractionRounding()->get(); $cacheTarget = $collection->getId() === self::METADATA ? new Document([Document::ID => $id, Document::COLLECTION => self::METADATA]) : $collection->getId(); - $document = $this->withMutation(Event::DocumentUpdate, $cacheTarget, function () use ($collection, $id, $document, $newUpdatedAt, $round, &$hasOperators) { + $document = $this->withMutation(Event::DocumentUpdate, $cacheTarget, function () use ($collection, $id, $document, $preparation, $newUpdatedAt, $round, &$hasOperators) { $old = $this->authorization->skip(fn () => $this->silent( fn () => $this->readDocument($collection, $id, forUpdate: true) )); if ($old->isEmpty()) { return new Document(); } + $document = $this->prepareNestedCopy($collection, $document, $preparation); $time = DateTime::nowAfter($old->getUpdatedAt() ?: null); $skipPermissionsUpdate = true; @@ -2432,14 +2436,14 @@ private function upsert( $seenIds = []; $hasRelationships = self::relationshipAttributes($collection) !== []; $replacing = $this->placeholderReplacement()->get(); - $preparesNested = $hasRelationships && $this->preparesNested(); - if ($replacing || $preparesNested) { + $preparation = $hasRelationships ? $this->nestedPreparation() : null; + if ($replacing || $preparation !== null) { foreach ($documents as $key => $document) { if ($replacing && Id::isPlaceholder($document->getId())) { $document->setAttribute(Document::ID, Id::unique()); } - if ($preparesNested) { - $documents[$key] = $this->prepareNested($collection, $document); + if ($preparation !== null) { + $documents[$key] = $this->prepareNested($collection, $document, $preparation); } } } From c795eed71967136139cd5693495823829cff58dd Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:50:46 +1300 Subject: [PATCH 31/55] fix(documents): refuse related documents in bulk updates while preparing updateDocuments() handed related documents given as documents to the relationship hook with no grant check and no placeholder replacement, so with setCheckGrants(true) a bulk update could give a related document a role the caller does not hold (the Memory adapter wrote it). While either toggle is on, a related document given as a document (or an associative array) in a bulk update is now refused with a relationship exception before anything is read or written; related documents given by ID still link. Checking grants once and replacing placeholders per updated document was considered and rejected: - the SQL adapters cannot write such a value in a bulk update at all (the update set reaches the adapter as given: an unknown PDO type or a missing column), so a working bulk write of related documents never existed there; - one placeholder would need one generated ID per updated document, while Mirror replays the update document on the destination, which would generate different IDs and drift; - whether a placeholder means one shared document or one per updated document is ambiguous, and appwrite refuses bulk updates on collections with relationships, so no caller relies on it. Defaults are unchanged: with both toggles off the 8.0 behaviour stays. Co-Authored-By: Claude Opus 5.5 --- src/Database/Database.php | 4 ++ src/Database/Trait/Documents.php | 19 +++++++ .../Relationships/NestedPreparationTests.php | 50 +++++++++++++++++ .../Relationships/NestedPreparationTest.php | 56 +++++++++++++++++++ 4 files changed, 129 insertions(+) diff --git a/src/Database/Database.php b/src/Database/Database.php index 0b9a5c569..a0be32680 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -1165,6 +1165,8 @@ public function withPreserveSequence(bool $preserve, callable $callback): mixed * covers the documents given to createDocument(), createDocuments(), upsertDocument() and upsertDocuments(), and * every related document they or updateDocument() are given as a document, as deep as the write relates them. A * related document given as the string Id::PLACEHOLDER stays a reference to the document stored under that ID. + * While it is on, updateDocuments() refuses a related document given as a document with a relationship + * exception: one generated ID could not serve every document it updates. * * @return $this */ @@ -1199,6 +1201,8 @@ public function withReplacePlaceholders(bool $replace, callable $callback): mixe * may only add permissions for roles the caller holds, as deep as the write relates it. Permissions it already * has may be sent back, and permissions it inherits from its parent are never checked. Malformed permissions * throw a structure exception and an unheld role an authorization exception, before anything is written. + * While it is on, updateDocuments() refuses a related document given as a document with a relationship + * exception, as it is not checked there. * * createDocument() and updateDocument() read the permissions a related document already has locked, in the * transaction that writes it. createDocuments(), upsertDocument() and upsertDocuments() write related documents diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index e8e79c388..853f232a7 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -1459,6 +1459,21 @@ private function prepareNestedCopy(Document $collection, Document $document, ?Pr return $this->prepareNested($collection, clone $document, $preparation); } + private static function holdsRelatedDocument(Collection $collection, Document $document): bool + { + foreach (self::relationshipAttributes($collection) as $attribute) { + $value = $document->getAttribute($attribute->key); + $values = \is_array($value) && \array_is_list($value) ? $value : [$value]; + foreach ($values as $related) { + if ($related instanceof Document || (\is_array($related) && ! \array_is_list($related))) { + return true; + } + } + } + + return false; + } + /** * @throws DatabaseException */ @@ -2065,6 +2080,10 @@ public function updateDocuments( $collection = $this->silent(fn () => $this->getCollection($collection)); + if ($this->nestedPreparation() !== null && self::holdsRelatedDocument($collection, $updates)) { + throw new RelationshipException('Invalid relationship value. A bulk update takes related documents by ID while placeholders are replaced or grants are checked.'); + } + $documentSecurity = $collection->getAttribute('documentSecurity', false); $skipAuth = $this->authorization->isValid(new Input(PermissionType::Update, $collection->getPermissionsByType(PermissionType::Update))); diff --git a/tests/e2e/Adapter/Scopes/Relationships/NestedPreparationTests.php b/tests/e2e/Adapter/Scopes/Relationships/NestedPreparationTests.php index 29e606b28..df4cfe58b 100644 --- a/tests/e2e/Adapter/Scopes/Relationships/NestedPreparationTests.php +++ b/tests/e2e/Adapter/Scopes/Relationships/NestedPreparationTests.php @@ -8,8 +8,10 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization as AuthorizationException; +use Utopia\Database\Exception\Relationship as RelationshipException; use Utopia\Database\Id; use Utopia\Database\Permission; +use Utopia\Database\Query; use Utopia\Database\Relationship; use Utopia\Database\RelationshipDeleteAction; use Utopia\Database\Role; @@ -179,6 +181,54 @@ public function testNestedPreparationRefusesAnUnheldPlaceholderInWritesAndPreChe $this->assertTrue($database->getDocument($children, Id::PLACEHOLDER)->isEmpty()); } + public function testNestedPreparationRefusesRelatedDocumentsInABulkUpdate(): void + { + $database = $this->getDatabase(); + if (! $database->getAdapter()->hasFeature(Feature\Relationships::class)) { + $this->expectNotToPerformAssertions(); + + return; + } + + [$parents, $children, $owners] = $this->createNestedPreparationCollections($database); + $database->createDocument($parents, new Document([Document::ID => 'bulk', 'name' => 'bulk'])); + $scopes = [ + 'placeholders replaced' => static fn (callable $callback): mixed => $database->withReplacePlaceholders(true, $callback), + 'grants checked' => static fn (callable $callback): mixed => $database->withCheckGrants(true, $callback), + ]; + $permissions = [Permission::update(Role::user('nestedPreparationUnheld'))]; + $values = [ + 'children' => [new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'bulk child', Document::PERMISSIONS => $permissions])], + 'owner' => ['name' => 'bulk owner', Document::PERMISSIONS => $permissions], + ]; + + foreach ($scopes as $toggle => $scope) { + foreach ($values as $key => $value) { + try { + $scope(fn (): mixed => $database->updateDocuments($parents, new Document([$key => $value]))); + $this->fail("A bulk update with {$key} given as a document is refused while {$toggle}"); + } catch (RelationshipException $error) { + $this->assertSame('Invalid relationship value. A bulk update takes related documents by ID while placeholders are replaced or grants are checked.', $error->getMessage()); + } + } + } + + $this->assertSame([], $database->find($children)); + $this->assertSame([], $database->find($owners)); + $stored = $database->getDocument($parents, 'bulk'); + $this->assertSame([], $this->nestedPreparationIds($stored->getAttribute('children'))); + $this->assertNull($stored->getAttribute('owner')); + + $database->createDocument($owners, new Document([Document::ID => 'linked owner', 'name' => 'linked owner'])); + $scopes['placeholders replaced'](fn (): mixed => $scopes['grants checked'](fn (): mixed => $database->updateDocuments( + $parents, + new Document(['owner' => 'linked owner']), + [Query::equal(Document::ID, ['bulk'])], + ))); + + $this->assertSame(['linked owner'], $this->nestedPreparationIds($database->getDocument($parents, 'bulk')->getAttribute('owner'))); + } + /** * @return array{string, string, string} The parent, child and owner collection IDs */ diff --git a/tests/unit/Relationships/NestedPreparationTest.php b/tests/unit/Relationships/NestedPreparationTest.php index 13de51348..87baf9509 100644 --- a/tests/unit/Relationships/NestedPreparationTest.php +++ b/tests/unit/Relationships/NestedPreparationTest.php @@ -22,6 +22,7 @@ use Utopia\Database\Document; use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Authorization as AuthorizationException; +use Utopia\Database\Exception\Relationship as RelationshipException; use Utopia\Database\Exception\Structure as StructureException; use Utopia\Database\Hook\Permissions; use Utopia\Database\Hook\Relationships; @@ -59,6 +60,8 @@ final class NestedPreparationTest extends TestCase private const string MALFORMED = 'Wildcard permission "*" has been replaced. Use "any" instead.'; + private const string BULK_REFUSAL = 'Invalid relationship value. A bulk update takes related documents by ID while placeholders are replaced or grants are checked.'; + private Authorization $authorization; /** @@ -417,6 +420,59 @@ public function testAPlaceholderWithAnUnheldRoleWritesNothingAndWithAHeldRoleIsC $this->assertSame([Permission::read(Role::user(self::USER))], $database->getDocument(self::SECOND, $childId)->getPermissions()); } + /** + * @param Closure(): Adapter $adapter + */ + #[DataProvider('types')] + public function testABulkUpdateRefusesARelatedDocumentGivenAsADocument(Closure $adapter, RelationshipType $type): void + { + $toggles = [ + 'placeholders replaced' => static fn (Database $database): Database => $database->setReplacePlaceholders(true), + 'grants checked' => static fn (Database $database): Database => $database->setCheckGrants(true), + ]; + + foreach ($toggles as $toggle => $enable) { + $database = $this->database($adapter(), $type); + $database->createDocuments(self::FIRST, [new Document([Document::ID => 'one', 'name' => 'one']), new Document([Document::ID => 'two', 'name' => 'two'])]); + $enable($database); + $related = ['name' => 'child', Document::PERMISSIONS => [Permission::update(Role::team('admin'))]]; + $toMany = \in_array($type, [RelationshipType::OneToMany, RelationshipType::ManyToMany], true); + $values = [ + 'document' => self::value($type, new Document([Document::ID => Id::PLACEHOLDER, ...$related])), + 'array' => $toMany ? [$related] : $related, + ]; + + foreach ($values as $form => $value) { + $failure = $this->failure(fn () => $database->updateDocuments(self::FIRST, new Document([self::NEXT => $value]))); + + $this->assertInstanceOf(RelationshipException::class, $failure, "{$toggle}, {$form}"); + $this->assertSame(self::BULK_REFUSAL, $failure->getMessage(), "{$toggle}, {$form}"); + } + + $this->assertSame(['one', 'two'], self::names($database, self::FIRST), $toggle); + $this->assertSame([], self::names($database, self::SECOND), $toggle); + $this->assertSame([], self::ids($database->getDocument(self::FIRST, 'one')->getAttribute(self::NEXT)), $toggle); + $this->assertSame([], self::ids($database->getDocument(self::FIRST, 'two')->getAttribute(self::NEXT)), $toggle); + } + } + + /** + * @param Closure(): Adapter $adapter + */ + #[DataProvider('adapters')] + public function testABulkUpdateLinksARelatedDocumentByIdWhileBothTogglesAreOn(Closure $adapter): void + { + $database = $this->database($adapter(), RelationshipType::ManyToOne); + $database->createDocument(self::SECOND, new Document([Document::ID => 'stored', 'name' => 'stored'])); + $database->createDocuments(self::FIRST, [new Document([Document::ID => 'one', 'name' => 'one']), new Document([Document::ID => 'two', 'name' => 'two'])]); + $database->setReplacePlaceholders(true)->setCheckGrants(true); + + $this->assertSame(2, $database->updateDocuments(self::FIRST, new Document([self::NEXT => 'stored']))); + + $this->assertSame(['stored'], self::ids($database->getDocument(self::FIRST, 'one')->getAttribute(self::NEXT))); + $this->assertSame(['stored'], self::ids($database->getDocument(self::FIRST, 'two')->getAttribute(self::NEXT))); + } + /** * @param Closure(): Adapter $adapter */ From 0e1210c5a2e2bc14718ca401f0ef4695f786df2d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:53:25 +1300 Subject: [PATCH 32/55] fix(adapter): round fractions on SQLite, Memory and Redis as 7.4.1 stored them Adapter::arithmetic() said every adapter without an engine of its own followed MariaDB, but SQLite, Memory and Redis all shipped in 7.4.1 with arithmetic of their own (git show 7.4.1:src/Database/Adapter/*.php): - SQLite bound a float as text, computed COALESCE(col, 0) + :v in double precision and kept the REAL in the INTEGER-affinity column; the quotient of two integers was integer division, and % cast both operands to INTEGER. Database::casting() then read the column with (int), cutting toward zero. Replaying the 7.4.1 SQL on SQLite: 10 + 1.5 kept 11.5 and read 11, -12 + 0.5 read -11, 7 / 2 stored 3, 10 % 2.75 stored 0, 11 % 4.5 stored 3, 10 % 0.5 stored NULL, and a counter 5 + 1.5 kept 6.5. - Memory and Redis applyOperator() computed in PHP floats, stored the float and were cast with (int) on read; modulo took (int) of the operand, so 11 % 4.5 stored 3 and % 0.5 threw DivisionByZeroError. All three now declare toward-zero rounding everywhere and the new Arithmetic::$truncatesModulus, which cuts a fractional modulus toward zero before taking the remainder exactly, and refuses one cut to zero. The fixture gains a Memory family (MongoDB's rows but for the modulo ones), replayed against the 7.4.1 Memory logic row by row; the SQLite, Memory and Redis e2e classes run it. SQLite compared a fractional operator limit, bound as text, with a number, which SQLite orders below all text: a fractional maximum never held and a fractional minimum always held. That is not reproduced; limits hold as on Memory and Redis. MariaDB and MySQL keep the default arithmetic, which stays the answer for an adapter that had no 7.4.1 engine. Co-Authored-By: Claude Opus 5.5 --- src/Database/Adapter.php | 2 +- src/Database/Adapter/Memory.php | 18 +++++ src/Database/Adapter/Redis.php | 18 +++++ src/Database/Adapter/SQLite.php | 20 +++++ src/Database/Arithmetic.php | 6 +- src/Database/Arithmetic/Fractional.php | 7 ++ tests/e2e/Adapter/MemoryTest.php | 7 ++ tests/e2e/Adapter/RedisTest.php | 7 ++ tests/e2e/Adapter/SQLiteTest.php | 7 ++ .../Adapter/Scopes/FractionalIntegerTests.php | 6 +- tests/e2e/Adapter/SharedTables/SQLiteTest.php | 7 ++ tests/unit/Adapter/ArithmeticTest.php | 13 ++-- tests/unit/Arithmetic/FractionalTest.php | 32 ++++++++ tests/unit/Documents/RoundFractionsTest.php | 6 +- tests/unit/Support/FractionalEngine.php | 4 +- tests/unit/Support/FractionalResults.php | 77 ++++++++++++++++++- 16 files changed, 220 insertions(+), 17 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 5acb19da6..a62620729 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -815,7 +815,7 @@ abstract public function limits(): Limits; /** * How this adapter's engine stored a fractional result in an integer column on database 7.4.1. MariaDB and * MySQL computed a fractional operand in DOUBLE, rounded half to even, and a division by a whole number in - * exact DECIMAL, rounded half away from zero; adapters without a 7.4.1 engine of their own answer the same. + * exact DECIMAL, rounded half away from zero; an adapter that had no 7.4.1 engine answers the same. */ public function arithmetic(): Arithmetic { diff --git a/src/Database/Adapter/Memory.php b/src/Database/Adapter/Memory.php index 071f1fd2c..654c91779 100644 --- a/src/Database/Adapter/Memory.php +++ b/src/Database/Adapter/Memory.php @@ -3,6 +3,8 @@ namespace Utopia\Database\Adapter; use Utopia\Database\Adapter; +use Utopia\Database\Arithmetic; +use Utopia\Database\Arithmetic\Rounding; use Utopia\Database\Attribute; use Utopia\Database\Capability; use Utopia\Database\Database; @@ -1956,6 +1958,22 @@ public function getSizeOfCollectionOnDisk(string $collection): int return $this->getSizeOfCollection($collection); } + /** + * Memory stored a fractional result as a float and every read cast it to an integer, cutting it toward zero; + * a fractional modulus was cut toward zero before the remainder was taken. + */ + #[\Override] + public function arithmetic(): Arithmetic + { + return new Arithmetic( + counter: Rounding::TowardZero, + operators: Rounding::TowardZero, + division: Rounding::TowardZero, + power: Rounding::TowardZero, + truncatesModulus: true, + ); + } + /** * No index is bounded by bytes here, but index validation subtracts from the index length, so it is * Mongo's positive cap. diff --git a/src/Database/Adapter/Redis.php b/src/Database/Adapter/Redis.php index 3935445bd..786de9eb8 100644 --- a/src/Database/Adapter/Redis.php +++ b/src/Database/Adapter/Redis.php @@ -7,6 +7,8 @@ use Redis as RedisClient; use Utopia\Database\Adapter; use Utopia\Database\Adapter\Redis\Write; +use Utopia\Database\Arithmetic; +use Utopia\Database\Arithmetic\Rounding; use Utopia\Database\Attribute; use Utopia\Database\Capability; use Utopia\Database\Change; @@ -1630,6 +1632,22 @@ public function increaseDocumentAttribute(Document $collection, string $id, stri }); } + /** + * Redis stored a fractional result as a float and every read cast it to an integer, cutting it toward zero; a + * fractional modulus was cut toward zero before the remainder was taken. + */ + #[\Override] + public function arithmetic(): Arithmetic + { + return new Arithmetic( + counter: Rounding::TowardZero, + operators: Rounding::TowardZero, + division: Rounding::TowardZero, + power: Rounding::TowardZero, + truncatesModulus: true, + ); + } + #[\Override] public function limits(): Limits { diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index f198125b2..e1dc81f70 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -13,6 +13,8 @@ use Utopia\Database\Adapter\SQL\Expression; use Utopia\Database\Adapter\SQL\Hook\Permission; use Utopia\Database\Adapter\SQL\JoinAlias; +use Utopia\Database\Arithmetic; +use Utopia\Database\Arithmetic\Rounding; use Utopia\Database\Attribute; use Utopia\Database\Builder\Scoping; use Utopia\Database\Builder\SQLite as SQLiteBuilder; @@ -144,6 +146,24 @@ public function reconnect(): void $this->registerUserFunctions(); } + /** + * SQLite kept a fractional result as a REAL in the INTEGER column and every read cast it to an integer, cutting + * it toward zero, as it cut the quotient of two integers; % cast a fractional modulus to an integer first. Its + * comparison of a fractional operator limit as text, which never held a maximum and always held a minimum, is + * not reproduced: limits hold as on Memory and Redis. + */ + #[\Override] + public function arithmetic(): Arithmetic + { + return new Arithmetic( + counter: Rounding::TowardZero, + operators: Rounding::TowardZero, + division: Rounding::TowardZero, + power: Rounding::TowardZero, + truncatesModulus: true, + ); + } + /** * SQLite has no server-side connection id, so the handle's object id names the connection: unique only within * the process and only while the handle lives. diff --git a/src/Database/Arithmetic.php b/src/Database/Arithmetic.php index bc39fd642..7fa34ce88 100644 --- a/src/Database/Arithmetic.php +++ b/src/Database/Arithmetic.php @@ -7,8 +7,9 @@ /** * How an engine stored a fractional result in an integer column on database 7.4.1, which handed fractional * amounts and limits to the engine: the rounding of a counter change, of an operator, of a division by a whole - * number and of a power; whether it refused a fractional counter change; the operators it computed exactly; and - * the operators whose fractional amount it read as an integer, and so refused, unless a limit came with it. + * number and of a power; whether it refused a fractional counter change; the operators it computed exactly; the + * operators whose fractional amount it read as an integer, and so refused, unless a limit came with it; and + * whether it cut a fractional modulus toward zero before taking the remainder. */ final readonly class Arithmetic { @@ -24,6 +25,7 @@ public function __construct( public bool $refusesCounterFractions = false, public array $exactOperators = [], public array $refusedWithoutLimit = [], + public bool $truncatesModulus = false, ) { } diff --git a/src/Database/Arithmetic/Fractional.php b/src/Database/Arithmetic/Fractional.php index e30a1f16f..2880a2f81 100644 --- a/src/Database/Arithmetic/Fractional.php +++ b/src/Database/Arithmetic/Fractional.php @@ -97,6 +97,13 @@ public function operator(Operator $operator, int|float|string|null $stored): Ope } $whole = ! self::isFractional($amount); + if ($method === OperatorType::Modulo && ! $whole && $this->arithmetic->truncatesModulus) { + $modulus = self::integer($amount); + + return $modulus === null || BigInt::compare($modulus, 0) === 0 + ? $operator + : self::change($operator, BigInt::subtract(BigInt::modulo($current, $modulus), $current)); + } if ($limit === null && ! $whole && \in_array($method, $this->arithmetic->refusedWithoutLimit, true)) { return $operator; } diff --git a/tests/e2e/Adapter/MemoryTest.php b/tests/e2e/Adapter/MemoryTest.php index 1d63dfbf7..810a14b0d 100644 --- a/tests/e2e/Adapter/MemoryTest.php +++ b/tests/e2e/Adapter/MemoryTest.php @@ -3,6 +3,7 @@ namespace Tests\E2E\Adapter; use Redis; +use Tests\Unit\Support\FractionalEngine; use Utopia\Cache\Adapter\Redis as RedisAdapter; use Utopia\Cache\Cache; use Utopia\Database\Adapter\Memory; @@ -919,4 +920,10 @@ public function testNegationOperatorsExcludeNullRows(): void Query::notSearch('bio', 'unrelated'), ])); } + + #[\Override] + protected static function fractionalEngine(): FractionalEngine + { + return FractionalEngine::Memory; + } } diff --git a/tests/e2e/Adapter/RedisTest.php b/tests/e2e/Adapter/RedisTest.php index 99f84f614..91a176632 100644 --- a/tests/e2e/Adapter/RedisTest.php +++ b/tests/e2e/Adapter/RedisTest.php @@ -3,6 +3,7 @@ namespace Tests\E2E\Adapter; use Redis; +use Tests\Unit\Support\FractionalEngine; use Utopia\Cache\Adapter\None as NoneCacheAdapter; use Utopia\Cache\Cache; use Utopia\Database\Adapter\Redis as RedisAdapter; @@ -203,4 +204,10 @@ private function findStoredPayloads(Redis $client, string $id): array return $payloads; } + + #[\Override] + protected static function fractionalEngine(): FractionalEngine + { + return FractionalEngine::Memory; + } } diff --git a/tests/e2e/Adapter/SQLiteTest.php b/tests/e2e/Adapter/SQLiteTest.php index 4060d3efb..74462e2d6 100644 --- a/tests/e2e/Adapter/SQLiteTest.php +++ b/tests/e2e/Adapter/SQLiteTest.php @@ -3,6 +3,7 @@ namespace Tests\E2E\Adapter; use Redis; +use Tests\Unit\Support\FractionalEngine; use Utopia\Cache\Adapter\Redis as RedisAdapter; use Utopia\Cache\Cache; use Utopia\Database\Adapter\SQLite; @@ -131,4 +132,10 @@ protected function deleteIndex(string $collection, string $index): bool return true; } + + #[\Override] + protected static function fractionalEngine(): FractionalEngine + { + return FractionalEngine::Memory; + } } diff --git a/tests/e2e/Adapter/Scopes/FractionalIntegerTests.php b/tests/e2e/Adapter/Scopes/FractionalIntegerTests.php index 689d9637c..c2efef914 100644 --- a/tests/e2e/Adapter/Scopes/FractionalIntegerTests.php +++ b/tests/e2e/Adapter/Scopes/FractionalIntegerTests.php @@ -122,7 +122,7 @@ public function testFractionalOperatorOnABigIntegerBeyondFloatPrecisionStaysExac $collection = $this->fractionalCollection($database, 'fractional_big'); $expected = match (static::fractionalEngine()) { - FractionalEngine::MariaDB, FractionalEngine::MongoDB => 9007199254740994, + FractionalEngine::MariaDB, FractionalEngine::MongoDB, FractionalEngine::Memory => 9007199254740994, FractionalEngine::PostgreSQL => 9007199254740995, }; @@ -153,7 +153,7 @@ public function testFractionalBulkUpdateRoundsEachDocumentFromItsOwnStoredValue( $expected = match (static::fractionalEngine()) { FractionalEngine::MariaDB => ['first' => 12, 'second' => 12, 'third' => 14], FractionalEngine::PostgreSQL => ['first' => 12, 'second' => 13, 'third' => 14], - FractionalEngine::MongoDB => ['first' => 11, 'second' => 12, 'third' => 13], + FractionalEngine::MongoDB, FractionalEngine::Memory => ['first' => 11, 'second' => 12, 'third' => 13], }; try { @@ -190,7 +190,7 @@ public function testFractionalUpsertRoundsAStoredDocumentFromItsValueAndANewOneF $collection = $this->fractionalCollection($database, 'fractional_upsert'); $expected = match (static::fractionalEngine()) { - FractionalEngine::MariaDB, FractionalEngine::MongoDB => ['first' => 12, 'second' => 2], + FractionalEngine::MariaDB, FractionalEngine::MongoDB, FractionalEngine::Memory => ['first' => 12, 'second' => 2], FractionalEngine::PostgreSQL => ['first' => 13, 'second' => 3], }; diff --git a/tests/e2e/Adapter/SharedTables/SQLiteTest.php b/tests/e2e/Adapter/SharedTables/SQLiteTest.php index de5d81ddd..736f65cb5 100644 --- a/tests/e2e/Adapter/SharedTables/SQLiteTest.php +++ b/tests/e2e/Adapter/SharedTables/SQLiteTest.php @@ -4,6 +4,7 @@ use Redis; use Tests\E2E\Adapter\Base; +use Tests\Unit\Support\FractionalEngine; use Utopia\Cache\Adapter\Redis as RedisAdapter; use Utopia\Cache\Cache; use Utopia\Database\Adapter\SQLite; @@ -154,4 +155,10 @@ protected function deleteIndex(string $collection, string $index): bool return true; } + + #[\Override] + protected static function fractionalEngine(): FractionalEngine + { + return FractionalEngine::Memory; + } } diff --git a/tests/unit/Adapter/ArithmeticTest.php b/tests/unit/Adapter/ArithmeticTest.php index a0b35c10d..2df06d189 100644 --- a/tests/unit/Adapter/ArithmeticTest.php +++ b/tests/unit/Adapter/ArithmeticTest.php @@ -32,13 +32,14 @@ final class ArithmeticTest extends TestCase */ public static function adapters(): iterable { - $mariaDB = [Rounding::HalfEven, Rounding::HalfEven, Rounding::HalfAwayFromZero, Rounding::HalfEven, false, [], []]; + $mariaDB = [Rounding::HalfEven, Rounding::HalfEven, Rounding::HalfAwayFromZero, Rounding::HalfEven, false, [], [], false]; + $memory = [Rounding::TowardZero, Rounding::TowardZero, Rounding::TowardZero, Rounding::TowardZero, false, [], [], true]; yield 'MariaDB' => [static fn (): Adapter => new MariaDB(new stdClass()), $mariaDB]; yield 'MySQL' => [static fn (): Adapter => new MySQL(new stdClass()), $mariaDB]; - yield 'SQLite' => [static fn (): Adapter => new SQLite(new PDO('sqlite::memory:')), $mariaDB]; - yield 'Memory' => [static fn (): Adapter => new Memory(), $mariaDB]; - yield 'Redis' => [static fn (): Adapter => new Redis(new RedisClient()), $mariaDB]; + yield 'SQLite' => [static fn (): Adapter => new SQLite(new PDO('sqlite::memory:')), $memory]; + yield 'Memory' => [static fn (): Adapter => new Memory(), $memory]; + yield 'Redis' => [static fn (): Adapter => new Redis(new RedisClient()), $memory]; yield 'PostgreSQL' => [static fn (): Adapter => new Postgres(new stdClass()), [ Rounding::HalfEven, Rounding::HalfAwayFromZero, @@ -47,8 +48,9 @@ public static function adapters(): iterable true, [OperatorType::Multiply, OperatorType::Divide, OperatorType::Modulo], [OperatorType::Increment, OperatorType::Decrement, OperatorType::Multiply, OperatorType::Divide], + false, ]]; - yield 'MongoDB' => [self::mongo(...), [Rounding::TowardZero, Rounding::TowardZero, Rounding::TowardZero, Rounding::TowardZero, false, [], []]]; + yield 'MongoDB' => [self::mongo(...), [Rounding::TowardZero, Rounding::TowardZero, Rounding::TowardZero, Rounding::TowardZero, false, [], [], false]]; } /** @@ -128,6 +130,7 @@ private static function describe(Arithmetic $arithmetic): array $arithmetic->refusesCounterFractions, $arithmetic->exactOperators, $arithmetic->refusedWithoutLimit, + $arithmetic->truncatesModulus, ]; } diff --git a/tests/unit/Arithmetic/FractionalTest.php b/tests/unit/Arithmetic/FractionalTest.php index 121307369..252c1d6b5 100644 --- a/tests/unit/Arithmetic/FractionalTest.php +++ b/tests/unit/Arithmetic/FractionalTest.php @@ -8,6 +8,7 @@ use Tests\Unit\Support\FractionalEngine; use Tests\Unit\Support\FractionalResults; use Utopia\Database\Adapter\MariaDB; +use Utopia\Database\Adapter\Memory; use Utopia\Database\Adapter\Mongo; use Utopia\Database\Adapter\Postgres; use Utopia\Database\Arithmetic\Fractional; @@ -176,6 +177,36 @@ public function testNumericTextPaddedWithWhitespaceIsReadByItsValue(Operator $op $this->assertSame($expected, self::stores(10, self::fractional(FractionalEngine::PostgreSQL)->operator($operator, 10))); } + /** + * @return iterable + */ + public static function truncatedModuli(): iterable + { + yield 'a modulus cut toward zero' => [11, Operator::modulo(4.5), 3]; + yield 'a negative modulus cut toward zero' => [7, Operator::modulo(-2.5), 1]; + yield 'a negative value keeps its sign' => [-11, Operator::modulo(3.5), -2]; + yield 'modulus text cut toward zero' => [13, Operator::modulo('3.5'), 1]; + yield 'a value beyond float precision' => ['9007199254740993', Operator::modulo(2.5), 1]; + yield 'a modulus cut to zero is left for the caller to refuse' => [10, Operator::modulo(0.5), null]; + yield 'a negative modulus cut to zero is left for the caller to refuse' => [10, Operator::modulo('-0.75'), null]; + } + + #[DataProvider('truncatedModuli')] + public function testAnEngineThatCutAFractionalModulusTakesTheRemainderOfItsWholePart(int|string $stored, Operator $operator, int|string|null $expected): void + { + $resolved = self::fractional(FractionalEngine::Memory)->operator($operator, $stored); + + if ($expected === null) { + $this->assertSame($operator, $resolved); + + return; + } + + $change = $resolved->getValue(); + $this->assertTrue(BigInt::isIntegerValue($change)); + $this->assertSame($expected, $resolved->getMethod() === OperatorType::Increment ? BigInt::add($stored, $change) : BigInt::subtract($stored, $change)); + } + /** * @return iterable */ @@ -436,6 +467,7 @@ private static function fractional(FractionalEngine $engine): Fractional $adapter = match ($engine) { FractionalEngine::MariaDB => new MariaDB(new stdClass()), FractionalEngine::PostgreSQL => new Postgres(new stdClass()), + FractionalEngine::Memory => new Memory(), FractionalEngine::MongoDB => new Mongo(new class () extends Client { public function __construct() { diff --git a/tests/unit/Documents/RoundFractionsTest.php b/tests/unit/Documents/RoundFractionsTest.php index fbd5d678b..26da29c69 100644 --- a/tests/unit/Documents/RoundFractionsTest.php +++ b/tests/unit/Documents/RoundFractionsTest.php @@ -15,6 +15,7 @@ use Utopia\Cache\Adapter\None; use Utopia\Cache\Cache; use Utopia\Database\Adapter; +use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Adapter\Memory; use Utopia\Database\Adapter\Mongo; use Utopia\Database\Adapter\Postgres; @@ -896,7 +897,7 @@ private function database(?Adapter $adapter = null, bool $round = true): Databas $authorization = new Authorization(); $authorization->addRole(Role::any()->toString()); - $database = (new Database($adapter ?? new Memory(), new Cache(new None()))) + $database = (new Database($adapter ?? self::engine(FractionalEngine::MariaDB), new Cache(new None()))) ->setAuthorization($authorization) ->setDatabase('round_fractions') ->setNamespace('round_fractions_'.\uniqid()); @@ -955,7 +956,8 @@ public function arithmetic(): Arithmetic private static function arithmetic(FractionalEngine $engine): Arithmetic { return match ($engine) { - FractionalEngine::MariaDB => (new Memory())->arithmetic(), + FractionalEngine::MariaDB => (new MariaDB(new stdClass()))->arithmetic(), + FractionalEngine::Memory => (new Memory())->arithmetic(), FractionalEngine::PostgreSQL => (new Postgres(new stdClass()))->arithmetic(), FractionalEngine::MongoDB => (new Mongo(new class () extends Client { public function __construct() diff --git a/tests/unit/Support/FractionalEngine.php b/tests/unit/Support/FractionalEngine.php index 291b66b98..9e7e3953c 100644 --- a/tests/unit/Support/FractionalEngine.php +++ b/tests/unit/Support/FractionalEngine.php @@ -3,11 +3,13 @@ namespace Tests\Unit\Support; /** - * The engine family whose 7.4.1 arithmetic a fixture row records: MariaDB also stands for MySQL. + * The engine family whose 7.4.1 arithmetic a fixture row records: MariaDB also stands for MySQL, and Memory for + * Redis and SQLite. */ enum FractionalEngine { case MariaDB; case PostgreSQL; case MongoDB; + case Memory; } diff --git a/tests/unit/Support/FractionalResults.php b/tests/unit/Support/FractionalResults.php index 8aef9af82..bdc5e2687 100644 --- a/tests/unit/Support/FractionalResults.php +++ b/tests/unit/Support/FractionalResults.php @@ -6,7 +6,8 @@ /** * What appwrite on database 7.4.1 stored for a fractional operand or limit on an INT column, read back from - * MariaDB 10.11, PostgreSQL (appwrite/postgres 0.1.0) and MongoDB 8.2.5; null where that engine refused it. + * MariaDB 10.11, PostgreSQL (appwrite/postgres 0.1.0) and MongoDB 8.2.5, and what 7.4.1's Memory adapter stored; + * null where that engine refused it. */ final class FractionalResults { @@ -219,13 +220,82 @@ public static function operators(): iterable yield 'MongoDB: mod -11%3.5' => [FractionalEngine::MongoDB, -11, OperatorType::Modulo, [3.5], 0]; yield 'MongoDB: inc 10+2 max10.5' => [FractionalEngine::MongoDB, 10, OperatorType::Increment, [2, 10.5], 10]; yield 'MongoDB: mul 5*3 max15.5' => [FractionalEngine::MongoDB, 5, OperatorType::Multiply, [3, 15.5], 15]; + yield 'Memory: inc 11+1.5' => [FractionalEngine::Memory, 11, OperatorType::Increment, [1.5], 12]; + yield 'Memory: inc 10+1.5' => [FractionalEngine::Memory, 10, OperatorType::Increment, [1.5], 11]; + yield 'Memory: inc 10+1.25' => [FractionalEngine::Memory, 10, OperatorType::Increment, [1.25], 11]; + yield 'Memory: inc 10+1.75' => [FractionalEngine::Memory, 10, OperatorType::Increment, [1.75], 11]; + yield 'Memory: inc -11+0.5' => [FractionalEngine::Memory, -11, OperatorType::Increment, [0.5], -10]; + yield 'Memory: inc -12+0.5' => [FractionalEngine::Memory, -12, OperatorType::Increment, [0.5], -11]; + yield 'Memory: dec 11-0.5' => [FractionalEngine::Memory, 11, OperatorType::Decrement, [0.5], 10]; + yield 'Memory: dec 12-0.5' => [FractionalEngine::Memory, 12, OperatorType::Decrement, [0.5], 11]; + yield 'Memory: dec -10-0.5' => [FractionalEngine::Memory, -10, OperatorType::Decrement, [0.5], -10]; + yield 'Memory: inc 10+2 max13.9' => [FractionalEngine::Memory, 10, OperatorType::Increment, [2, 13.9], 12]; + yield 'Memory: inc 12+2 max13.9' => [FractionalEngine::Memory, 12, OperatorType::Increment, [2, 13.9], 12]; + yield 'Memory: inc 12+1.5 max13.9' => [FractionalEngine::Memory, 12, OperatorType::Increment, [1.5, 13.9], 13]; + yield 'Memory: inc 12+1.5 max13.5' => [FractionalEngine::Memory, 12, OperatorType::Increment, [1.5, 13.5], 13]; + yield 'Memory: dec 12-1 min11.1' => [FractionalEngine::Memory, 12, OperatorType::Decrement, [1, 11.1], 12]; + yield 'Memory: dec 13-1 min11.1' => [FractionalEngine::Memory, 13, OperatorType::Decrement, [1, 11.1], 12]; + yield 'Memory: dec 12-0.5 min11.5' => [FractionalEngine::Memory, 12, OperatorType::Decrement, [0.5, 11.5], 11]; + yield 'Memory: mul 11*1.5' => [FractionalEngine::Memory, 11, OperatorType::Multiply, [1.5], 16]; + yield 'Memory: mul 5*2.5' => [FractionalEngine::Memory, 5, OperatorType::Multiply, [2.5], 12]; + yield 'Memory: mul 7*1.5' => [FractionalEngine::Memory, 7, OperatorType::Multiply, [1.5], 10]; + yield 'Memory: mul 7*-1.5' => [FractionalEngine::Memory, 7, OperatorType::Multiply, [-1.5], -10]; + yield 'Memory: mul 3*1.25' => [FractionalEngine::Memory, 3, OperatorType::Multiply, [1.25], 3]; + yield 'Memory: mul 12*2 max25.5' => [FractionalEngine::Memory, 12, OperatorType::Multiply, [2, 25.5], 24]; + yield 'Memory: mul 13*2 max25.5' => [FractionalEngine::Memory, 13, OperatorType::Multiply, [2, 25.5], 13]; + yield 'Memory: mul 10*1.5 max15' => [FractionalEngine::Memory, 10, OperatorType::Multiply, [1.5, 15], 15]; + yield 'Memory: div 10/1.5' => [FractionalEngine::Memory, 10, OperatorType::Divide, [1.5], 6]; + yield 'Memory: div 9/2.5' => [FractionalEngine::Memory, 9, OperatorType::Divide, [2.5], 3]; + yield 'Memory: div 10/2 min5.5' => [FractionalEngine::Memory, 10, OperatorType::Divide, [2, 5.5], 10]; + yield 'Memory: div 10/0.5' => [FractionalEngine::Memory, 10, OperatorType::Divide, [0.5], 20]; + yield 'Memory: mod 10%2.75' => [FractionalEngine::Memory, 10, OperatorType::Modulo, [2.75], 0]; + yield 'Memory: mod 11%4.5' => [FractionalEngine::Memory, 11, OperatorType::Modulo, [4.5], 3]; + yield 'Memory: pow 16^0.5' => [FractionalEngine::Memory, 16, OperatorType::Power, [0.5], 4]; + yield 'Memory: pow 10^0.5' => [FractionalEngine::Memory, 10, OperatorType::Power, [0.5], 3]; + yield 'Memory: pow 2^2.5' => [FractionalEngine::Memory, 2, OperatorType::Power, [2.5], 5]; + yield 'Memory: pow 4^1.5' => [FractionalEngine::Memory, 4, OperatorType::Power, [1.5], 8]; + yield 'Memory: pow 3^2 max8.5' => [FractionalEngine::Memory, 3, OperatorType::Power, [2, 8.5], 3]; + yield 'Memory: pow 2^3 max8.5' => [FractionalEngine::Memory, 2, OperatorType::Power, [3, 8.5], 8]; + yield 'Memory: inc 11+1.5 max20' => [FractionalEngine::Memory, 11, OperatorType::Increment, [1.5, 20], 12]; + yield 'Memory: inc 10+1.5 max20' => [FractionalEngine::Memory, 10, OperatorType::Increment, [1.5, 20], 11]; + yield 'Memory: inc -12+0.5 max20' => [FractionalEngine::Memory, -12, OperatorType::Increment, [0.5, 20], -11]; + yield 'Memory: inc 10+1.25 max20' => [FractionalEngine::Memory, 10, OperatorType::Increment, [1.25, 20], 11]; + yield 'Memory: dec 11-0.5 min0' => [FractionalEngine::Memory, 11, OperatorType::Decrement, [0.5, 0], 10]; + yield 'Memory: dec 12-0.5 min0' => [FractionalEngine::Memory, 12, OperatorType::Decrement, [0.5, 0], 11]; + yield 'Memory: dec -10-0.5 min-100' => [FractionalEngine::Memory, -10, OperatorType::Decrement, [0.5, -100], -10]; + yield 'Memory: mul 5*2.5 max100' => [FractionalEngine::Memory, 5, OperatorType::Multiply, [2.5, 100], 12]; + yield 'Memory: mul 11*1.5 max100' => [FractionalEngine::Memory, 11, OperatorType::Multiply, [1.5, 100], 16]; + yield 'Memory: mul 7*-1.5 max100' => [FractionalEngine::Memory, 7, OperatorType::Multiply, [-1.5, 100], -10]; + yield 'Memory: mul 3*1.25 max100' => [FractionalEngine::Memory, 3, OperatorType::Multiply, [1.25, 100], 3]; + yield 'Memory: div 9/2.5 min0' => [FractionalEngine::Memory, 9, OperatorType::Divide, [2.5, 0], 3]; + yield 'Memory: div 10/1.5 min0' => [FractionalEngine::Memory, 10, OperatorType::Divide, [1.5, 0], 6]; + yield 'Memory: div 5/0.4 min0' => [FractionalEngine::Memory, 5, OperatorType::Divide, [0.4, 0], 12]; + yield 'Memory: pow 2^2.5 max100' => [FractionalEngine::Memory, 2, OperatorType::Power, [2.5, 100], 5]; + yield 'Memory: pow 10^0.5 max100' => [FractionalEngine::Memory, 10, OperatorType::Power, [0.5, 100], 3]; + yield 'Memory: mod -10%2.75' => [FractionalEngine::Memory, -10, OperatorType::Modulo, [2.75], 0]; + yield 'Memory: mod 10%2.5' => [FractionalEngine::Memory, 10, OperatorType::Modulo, [2.5], 0]; + yield 'Memory: mod 11%2.5' => [FractionalEngine::Memory, 11, OperatorType::Modulo, [2.5], 1]; + yield 'Memory: mod 7%-2.5' => [FractionalEngine::Memory, 7, OperatorType::Modulo, [-2.5], 1]; + yield 'Memory: pow 4^-0.5' => [FractionalEngine::Memory, 4, OperatorType::Power, [-0.5], 0]; + yield 'Memory: pow 4^-0.5 max100' => [FractionalEngine::Memory, 4, OperatorType::Power, [-0.5, 100], 0]; + yield 'Memory: pow 16^-0.25' => [FractionalEngine::Memory, 16, OperatorType::Power, [-0.25], 0]; + yield 'Memory: div 5/2 min0.5' => [FractionalEngine::Memory, 5, OperatorType::Divide, [2, 0.5], 2]; + yield 'Memory: div 7/2 min0.5' => [FractionalEngine::Memory, 7, OperatorType::Divide, [2, 0.5], 3]; + yield 'Memory: div -5/2 min-10.5' => [FractionalEngine::Memory, -5, OperatorType::Divide, [2, -10.5], -2]; + yield 'Memory: div 5/2 min2.6' => [FractionalEngine::Memory, 5, OperatorType::Divide, [2, 2.6], 5]; + yield 'Memory: mod 11%3.5' => [FractionalEngine::Memory, 11, OperatorType::Modulo, [3.5], 2]; + yield 'Memory: mod 13%3.5' => [FractionalEngine::Memory, 13, OperatorType::Modulo, [3.5], 1]; + yield 'Memory: mod -11%3.5' => [FractionalEngine::Memory, -11, OperatorType::Modulo, [3.5], -2]; + yield 'Memory: inc 10+2 max10.5' => [FractionalEngine::Memory, 10, OperatorType::Increment, [2, 10.5], 10]; + yield 'Memory: mul 5*3 max15.5' => [FractionalEngine::Memory, 5, OperatorType::Multiply, [3, 15.5], 15]; } /** * Fractional counter changes: the engine, whether it increases, the start, the change, the bound and the value * stored, or null where the engine refused the change. MariaDB rounded half to even. PostgreSQL bound the - * change next to an integer column and refused any fraction. MongoDB stored the double and every read cut it - * toward zero; these rows cut each change on its own, which matches it while one change is applied at a time. + * change next to an integer column and refused any fraction. MongoDB and Memory stored the double and every + * read cut it toward zero; these rows cut each change on its own, which matches them while one change is + * applied at a time. * * @return iterable */ @@ -258,6 +328,7 @@ public static function counters(): iterable yield "MariaDB: {$name}" => [FractionalEngine::MariaDB, $increase, $start, $value, $bound, $mariaDB]; yield "PostgreSQL: {$name}" => [FractionalEngine::PostgreSQL, $increase, $start, $value, $bound, null]; yield "MongoDB: {$name}" => [FractionalEngine::MongoDB, $increase, $start, $value, $bound, $mongoDB]; + yield "Memory: {$name}" => [FractionalEngine::Memory, $increase, $start, $value, $bound, $mongoDB]; } foreach (FractionalEngine::cases() as $engine) { From d8e4cd95dc0e81ddad7a11b861b672e32b45d7e0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:54:46 +1300 Subject: [PATCH 33/55] test(e2e): cover fractional counters on INT and the bound refusals per engine The counter rows ran only on the BIGINT attribute, so an INT column never saw a fractional change on a real engine; they now run on both, the INT one skipping the rows beyond its range. PostgreSQL and MongoDB had no case of a fractional change past its bound being refused and rolled back, and PostgreSQL's refusal of a whole change under a fractional bound, after the limit check, was only tested on Memory; both now run on every adapter. Co-Authored-By: Claude Opus 5.5 --- .../Adapter/Scopes/FractionalIntegerTests.php | 117 +++++++++++++++--- 1 file changed, 100 insertions(+), 17 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/FractionalIntegerTests.php b/tests/e2e/Adapter/Scopes/FractionalIntegerTests.php index c2efef914..2b69468fb 100644 --- a/tests/e2e/Adapter/Scopes/FractionalIntegerTests.php +++ b/tests/e2e/Adapter/Scopes/FractionalIntegerTests.php @@ -10,6 +10,7 @@ use Utopia\Database\Collection; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Exception\Limit as LimitException; use Utopia\Database\Exception\Structure as StructureException; use Utopia\Database\Exception\Type as TypeException; use Utopia\Database\Operator; @@ -71,40 +72,122 @@ public function testFractionalCountersStoreWhatTheEngineStored(): void $database = $this->getDatabase(); $collection = $this->fractionalCollection($database, 'fractional_counters'); $engine = static::fractionalEngine(); - $row = 0; + $rows = ['count' => 0, 'big' => 0]; try { - $database->withRoundFractions(true, function () use ($database, $collection, $engine, &$row): void { + $database->withRoundFractions(true, function () use ($database, $collection, $engine, &$rows): void { foreach (FractionalResults::counters() as $name => [$family, $increase, $start, $value, $bound, $stored]) { if ($family !== $engine) { continue; } - $id = 'counter'.$row++; - $database->createDocument($collection, new Document([Document::ID => $id, 'big' => $start])); - $change = fn (): Document => $increase - ? $database->increaseDocumentAttribute($collection, $id, 'big', $value, $bound) - : $database->decreaseDocumentAttribute($collection, $id, 'big', $value, $bound); + foreach (\array_keys($rows) as $attribute) { + if ($attribute === 'count' && \abs($start) > Database::MAX_INT) { + continue; + } + + $id = $attribute.$rows[$attribute]++; + $database->createDocument($collection, new Document([Document::ID => $id, $attribute => $start])); + $change = fn (): Document => $increase + ? $database->increaseDocumentAttribute($collection, $id, $attribute, $value, $bound) + : $database->decreaseDocumentAttribute($collection, $id, $attribute, $value, $bound); + + if ($stored === null) { + try { + $change(); + $this->fail("{$name} on {$attribute}: the engine refused it, so it must be refused"); + } catch (TypeException $exception) { + $this->assertSame('Change value must be an integer.', $exception->getMessage(), "{$name} on {$attribute}"); + } + + $this->assertSame($start, $database->getDocument($collection, $id)->getAttribute($attribute), "{$name} on {$attribute}"); + + continue; + } + + $this->assertSame($increase ? $start + $value : $start - $value, $change()->getAttribute($attribute), "{$name} on {$attribute}"); + $this->assertSame($stored, $database->getDocument($collection, $id)->getAttribute($attribute), "{$name} on {$attribute}"); + } + } + }); + + $this->assertSame(['count' => 15, 'big' => 21], $rows, 'Every counter row of the engine family ran on an INT and a BIGINT column'); + } finally { + $database->deleteCollection($collection); + } + } + + public function testFractionalCounterPastItsBoundIsRefusedAndRolledBack(): void + { + $database = $this->getDatabase(); + $collection = $this->fractionalCollection($database, 'fractional_past_bound'); + + try { + $database->withRoundFractions(true, function () use ($database, $collection): void { + foreach (['count', 'big'] as $attribute) { + foreach ([[true, 6], [false, 4]] as [$increase, $bound]) { + $id = $attribute.($increase ? 'Increase' : 'Decrease'); + $database->createDocument($collection, new Document([Document::ID => $id, $attribute => 5])); - if ($stored === null) { try { - $change(); - $this->fail("{$name}: the engine refused it, so it must be refused"); - } catch (TypeException $exception) { - $this->assertSame('Change value must be an integer.', $exception->getMessage(), $name); + $increase + ? $database->increaseDocumentAttribute($collection, $id, $attribute, 1.5, $bound) + : $database->decreaseDocumentAttribute($collection, $id, $attribute, 1.5, $bound); + $this->fail("A fractional change past its bound on {$attribute} must be refused"); + } catch (LimitException $exception) { + $this->assertSame('Attribute value exceeds '.($increase ? 'maximum' : 'minimum').' limit: '.$bound, $exception->getMessage()); } - $this->assertSame($start, $database->getDocument($collection, $id)->getAttribute('big'), $name); + $this->assertSame(5, $database->getDocument($collection, $id)->getAttribute($attribute), $id); + } + } + }); + } finally { + $database->deleteCollection($collection); + } + } + + public function testWholeCounterChangeUnderAFractionalBoundIsAnsweredAsTheEngineAnswered(): void + { + $database = $this->getDatabase(); + $collection = $this->fractionalCollection($database, 'fractional_whole_change'); + $refuses = static::fractionalEngine() === FractionalEngine::PostgreSQL; + $cases = [ + 'within a fractional maximum' => [true, 2, 10.5, $refuses ? 'Max must be an integer.' : 10], + 'past a fractional maximum' => [true, 3, 10.5, null], + 'within a fractional minimum' => [false, 2, 5.5, $refuses ? 'Min must be an integer.' : 6], + 'past a fractional minimum' => [false, 3, 5.5, null], + ]; + + try { + $database->withRoundFractions(true, function () use ($database, $collection, $cases): void { + $row = 0; + foreach ($cases as $name => [$increase, $value, $bound, $outcome]) { + $id = 'whole'.$row++; + $database->createDocument($collection, new Document([Document::ID => $id, 'count' => 8])); + $change = fn (): Document => $increase + ? $database->increaseDocumentAttribute($collection, $id, 'count', $value, $bound) + : $database->decreaseDocumentAttribute($collection, $id, 'count', $value, $bound); + + if (\is_int($outcome)) { + $this->assertSame($outcome, $change()->getAttribute('count'), $name); + $this->assertSame($outcome, $database->getDocument($collection, $id)->getAttribute('count'), $name); continue; } - $this->assertSame($increase ? $start + $value : $start - $value, $change()->getAttribute('big'), $name); - $this->assertSame($stored, $database->getDocument($collection, $id)->getAttribute('big'), $name); + try { + $change(); + $this->fail("{$name}: the change must be refused"); + } catch (LimitException|TypeException $exception) { + $outcome === null + ? $this->assertInstanceOf(LimitException::class, $exception, $name) + : $this->assertSame($outcome, $exception->getMessage(), $name); + } + + $this->assertSame(8, $database->getDocument($collection, $id)->getAttribute('count'), $name); } }); - - $this->assertSame(21, $row, 'Every counter row of the engine family ran'); } finally { $database->deleteCollection($collection); } From e4724eb6e3537ca46b68c86b511ab23a7b689eed Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 17:55:01 +1300 Subject: [PATCH 34/55] docs(arithmetic): say a non-finite operand or limit leaves the operator unchanged Co-Authored-By: Claude Opus 5.5 --- src/Database/Arithmetic/Fractional.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Database/Arithmetic/Fractional.php b/src/Database/Arithmetic/Fractional.php index 2880a2f81..c11c4a84c 100644 --- a/src/Database/Arithmetic/Fractional.php +++ b/src/Database/Arithmetic/Fractional.php @@ -82,7 +82,8 @@ public static function needsStored(Operator $operator): bool /** * The whole-number increment or decrement that stores what the engine stored for $operator applied to * $stored, or an increment by 0 where its limit held the value back. The operator comes back unchanged where - * the engine refused it, an operand is not a number, or the result is not finite, so the caller refuses it. + * the engine refused it, an operand or limit is not a finite number, or the result is not finite, so the caller + * refuses it. */ public function operator(Operator $operator, int|float|string|null $stored): Operator { From ffaf4cc9591ce6020374c5bfc1936275f62c3938 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 18:10:24 +1300 Subject: [PATCH 35/55] docs: describe the 8.1.0 library additions 8.1.0 moves work appwrite did app-side into the library as opt-in and additive APIs, so callers need the exact signatures and, above all, the rules that differ from 8.0.3 only when they opt in. The CHANGELOG lists each new public API with its signature and the two changes a caller can see without opting in: joins on a to-many relationship attribute that 8.0 refused, and the internal Snapshot constructor. UPGRADE.md gains an 8.0-to-8.1 guide: when placeholders are replaced, the grant check's scope, lock and its permission-oracle caveat, the per-engine fractional rules with PostgreSQL's 7.4.1 refusals and MongoDB's approximation, Mirror replay, the authorization bypass of resolveCursor(), read() beside find(), the to-many join rewrite and why hasFeature() answers spatial support behind a pool. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 77 +++++++++++++++ UPGRADE.md | 273 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 350 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 133926119..0ee311fbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,82 @@ # Changelog +## 8.1.0 + +8.1.0 adds opt-in and additive APIs for work that callers had to do themselves. With none of the new toggles set, a +`Database` behaves as in 8.0.3, except for the joins listed under Changed. See +[Upgrading from 8.0 to 8.1](UPGRADE.md#upgrading-from-80-to-81). + +### Added + +- `Id::PLACEHOLDER` (`'unique()'`) and `Id::isPlaceholder(string $id): bool`. +- `Database::setReplacePlaceholders(bool $replace): static`, `isReplacingPlaceholders(): bool` and + `withReplacePlaceholders(bool $replace, callable $callback): mixed`, off by default. While on, a document whose `$id` + is `Id::PLACEHOLDER` gets a generated ID, as one with an empty `$id` does: the documents given to + `createDocument()`, `createDocuments()`, `upsertDocument()` and `upsertDocuments()`, and every related document + given as a document to those and to `updateDocument()`, at every depth the write relates. A related document given + as the string `'unique()'` is still a reference to the document stored under that ID. +- `Database::setCheckGrants(bool $check): static`, `isCheckingGrants(): bool` and + `withCheckGrants(bool $check, callable $callback): mixed`, off by default. While on and while authorization is + enabled, a related document given as a document to `createDocument()`, `createDocuments()`, `updateDocument()`, + `upsertDocument()`, `upsertDocuments()`, `authorizeCreate()` or `authorizeUpdate()` may add permissions only for + roles the caller holds. An unheld role throws `Exception\Authorization` (`Permissions must be one of: ()`) + and malformed permissions throw `Exception\Structure`, before anything is written. +- While either toggle is on, `updateDocuments()` refuses a related document given as a document with + `Exception\Relationship`. Related documents given by ID still link. +- `Database::setRoundFractions(bool $round): static`, `isRoundingFractions(): bool` and + `withRoundFractions(bool $round, callable $callback): mixed`, off by default. While on, a fractional change, amount + or limit on a single-valued integer or big integer attribute is stored as the adapter's engine stored it on 7.4.1, + instead of being refused. This covers `increaseDocumentAttribute()`, `decreaseDocumentAttribute()` and the numeric + operators of `updateDocument()`, `updateDocuments()`, `upsertDocument()` and `upsertDocuments()`. Fractional items + added to an integer array are cut toward zero. What the engine refused is still refused. +- `Adapter::arithmetic(): Arithmetic` declares how the adapter's engine stored a fractional result in an integer + column on 7.4.1. The base answer is MariaDB's and MySQL's. `Postgres`, `Mongo`, `SQLite`, `Memory` and `Redis` + override it. `Pool` asks the pooled adapter once per pool, and `ReadWritePool` answers it wherever reads go, without + opening the sticky window. +- `Utopia\Database\Arithmetic`, a `final readonly` class: + `new Arithmetic(Rounding $counter, Rounding $operators, Rounding $division, Rounding $power, bool + $refusesCounterFractions = false, array $exactOperators = [], array $refusedWithoutLimit = [], bool + $truncatesModulus = false)`, and `rounding(OperatorType $method, bool $whole): Rounding`. +- `Utopia\Database\Arithmetic\Rounding`, an enum with the cases `HalfEven`, `HalfAwayFromZero` and `TowardZero`, and + the methods `apply(int|string $base, float $fraction): int|string`, `round(float $value): int|string` and + `ratio(int|string $numerator, int|string $denominator): int|string`. +- `Utopia\Database\Arithmetic\Decimal`, a number read exactly from its text: + `Decimal::parse(int|float|string $value): ?Decimal`, `floor(): int|string`, `ceil(): int|string`, + `integer(): int|string|null`, `Decimal::fraction(int|float|string $value): ?array`, + `Decimal::isOverlong(int|float|string $value): bool`, `Decimal::isNegative(int|float|string $value): bool`, + `Decimal::isTiny(int|float|string $value): bool` and `Decimal::order(int|float|string $value): ?int`. +- `Utopia\Database\Arithmetic\Fractional`, which resolves fractional amounts and limits from a stored value: + `new Fractional(Arithmetic $arithmetic)`, + `operator(Operator $operator, int|float|string|null $stored): Operator`, + `counter(int|string $stored, int|float|string $value, bool $increase): int|string`, and the static methods + `isFractional(mixed $value): bool`, `isWhole(int|float|string $value): bool`, + `wholeValue(int|float|string $value): int|string|null`, `needsStored(Operator $operator): bool`, + `exceeds(int|string $stored, int|float|string $value, int|float|string $bound, bool $increase): bool`, + `maximum(int|float|string|null $bound): int|float|string|null`, + `minimum(int|float|string|null $bound): int|float|string|null` and `whole(Operator $operator): Operator`. +- `Validator\Operator` and `Validator\Structure` (and so `Validator\PartialStructure`) take a trailing + `bool $fractions = false`. When it is set, a numeric operator on an integer attribute may carry a finite fractional + amount or limit within the attribute's range, for a caller that resolves it before writing. +- `Database::resolveCursor(string $collection, string $id, CursorDirection $direction, array $queries = []): + Document` returns the cursor that a list with `$queries` pages from when it names a document by ID. The document is + read with authorization skipped, and only the cursor fields are returned: `$id`, `$sequence`, `$collection` and + the values the list orders by, including values from joined rows. Pass the result only to `cursorAfter()` or + `cursorBefore()`. It returns an empty `Document` when no document has the ID. +- `Database::read(string $collection, array $queries = [], PermissionType $forPermission = PermissionType::Read): + array` returns a list of `Document`. For aggregate or `groupBy()` queries these are the rows of `aggregate()`; + otherwise they are what `find()` returns. Aggregate queries under a permission other than read throw + `Exception\Query`. `find()` still refuses aggregate queries. +- `ParsedQuery::isAggregation(): bool` tells whether the queries aggregate or group. + +### Changed + +- A join condition that compares the parent side of a one-to-many relationship, or the child side of a two-way + many-to-one relationship, with the `$id` of the related collection is now read through the two-way key the related + documents store. In 8.0 it threw `Exception\Query` (`Cannot join on virtual relationship attribute: `). Every + other join on a relationship side that holds no column is still refused. +- `State\Snapshot` is `@internal`. Its constructor takes three more required arguments, `replacePlaceholders`, + `checkGrants` and `roundFractions`. `Database::snapshot()` fills them in. + ## 8.0.3 ### Added diff --git a/UPGRADE.md b/UPGRADE.md index 5ac2f735b..589807ccc 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -1,3 +1,276 @@ +# Upgrading from 8.0 to 8.1 + +8.1 is a minor release. Every change in it is additive or opt-in. With none of the new toggles set, a `Database` +behaves exactly as in 8.0.3, with two exceptions: joins on a to-many relationship attribute (see +[Joining on a to-many relationship](#joining-on-a-to-many-relationship)) and the `@internal` `State\Snapshot` +constructor. [CHANGELOG.md](CHANGELOG.md) lists every new signature. + +- [Placeholder IDs](#placeholder-ids) +- [Grant check on related documents](#grant-check-on-related-documents) +- [Fractional operands on integer attributes](#fractional-operands-on-integer-attributes) +- [Cursor from an ID](#cursor-from-an-id) +- [One read for documents and aggregates](#one-read-for-documents-and-aggregates) +- [Joining on a to-many relationship](#joining-on-a-to-many-relationship) +- [Spatial support behind a pool](#spatial-support-behind-a-pool) +- [`State\Snapshot`](#statesnapshot) + +The three new toggles follow the 8.0 toggle idiom (see [Configuration toggles](#configuration-toggles)). Each one is +scoped to the calling coroutine and the coroutines it starts. `withSnapshot()` carries it into relationship writes and +other work that runs in another coroutine. + +| Toggle | Set | Read | Scope | +|---|---|---|---| +| Placeholder IDs | `setReplacePlaceholders(bool $replace): static` | `isReplacingPlaceholders(): bool` | `withReplacePlaceholders(bool $replace, callable $callback): mixed` | +| Grant check | `setCheckGrants(bool $check): static` | `isCheckingGrants(): bool` | `withCheckGrants(bool $check, callable $callback): mixed` | +| Fractions | `setRoundFractions(bool $round): static` | `isRoundingFractions(): bool` | `withRoundFractions(bool $round, callable $callback): mixed` | + +## Placeholder IDs + +Opt-in. By default 8.1 stores a `$id` of `'unique()'` as given, as 8.0 deliberately does: only an empty `$id` asks for +a generated one. `Id::PLACEHOLDER` names the placeholder (`'unique()'`), and `Id::isPlaceholder(string $id): bool` +checks for it. + +With `setReplacePlaceholders(true)`, a document whose `$id` is `Id::PLACEHOLDER` gets a generated ID, just as a +document with an empty `$id` does: + +- the documents given to `createDocument()`, `createDocuments()`, `upsertDocument()` and `upsertDocuments()`. An + upsert with a placeholder ID always creates a new document. +- every related document given as a document (a `Document` or an associative array) to those methods and to + `updateDocument()`, as deep as the write relates them. This needs the relationship hook to be attached and enabled. + A related document with an empty `$id` also gets its ID before anything is written, so links and junction rows see + the final ID. + +A related document given as the string `'unique()'` is still a reference to the document stored under that ID: it +links that document, or it is refused as missing, as in 8.0. While the toggle is on, a document stored with the +literal ID `'unique()'` can no longer be created or changed through a nested document. A string reference still links +it. + +`authorizeCreate()` and `authorizeUpdate()` give the related documents they check the same IDs, on a copy. A +placeholder related document is judged as new, so it needs create permission on its collection. While the toggle is +on, `updateDocuments()` refuses a related document given as a document with `Exception\Relationship`, because one +generated ID cannot serve every document the update changes. Related documents given by ID still link. `Mirror` +forwards the toggle to its source and destination. + +```php +$database->setReplacePlaceholders(true); + +$database->createDocument('authors', new Document([ + '$id' => Id::PLACEHOLDER, + 'name' => 'Ada', + 'books' => [ + ['$id' => Id::PLACEHOLDER, 'title' => 'Notes'], // created under a generated ID + 'unique()', // links the stored document 'unique()' + ], +])); +``` + +## Grant check on related documents + +Opt-in. By default 8.1 never compares the permissions a write grants with the roles the caller holds, as in 8.0. + +With `setCheckGrants(true)`, the check covers related documents written through a parent: every related document +given as a document to `createDocument()`, `createDocuments()`, `updateDocument()`, `upsertDocument()`, +`upsertDocuments()`, `authorizeCreate()` or `authorizeUpdate()`, as deep as the write relates them. The document the +write is given is not checked. For each related document whose `$permissions` is not null: + +- Malformed permissions throw `Exception\Structure` with the `Validator\Permissions` description. +- The permissions it adds must name roles the caller holds (`Authorization::hasRole()`). Otherwise the write throws + `Exception\Authorization` (`Permissions must be one of: ()`). The added permissions are + `Permission::aggregate()` of the sent permissions minus those the document already has. A new document has none. + Permissions sent back unchanged pass, and so does an empty list. +- Permissions a related document inherits from its parent are never checked. +- Both refusals happen before anything is written. + +The check is skipped while authorization is disabled (`$authorization->skip()` or `setStatus(false)`). To write +without it while authorization stays enabled, use `withCheckGrants(false, $callback)`. + +A related document's current permissions are read whether or not the caller can read the document: + +- `createDocument()` and `updateDocument()` read them locked, inside the write's transaction. +- `createDocuments()`, `upsertDocument()` and `upsertDocuments()` write related documents before the transaction of + the documents that relate them. They read the permissions without a lock, so a permission change committed between + the check and the related write is not seen. +- `authorizeCreate()` and `authorizeUpdate()` take them from `$resolve($collection, $id)` when it returns a document, + and from the stored document when it returns `null`. A document `$resolve` returns empty (staged for deletion) + is new. + +While the toggle is on, `updateDocuments()` refuses a related document given as a document with +`Exception\Relationship`, because it is not checked there. `Mirror` checks on its source only. The destination replays +what the source allowed with the check off, so a destination that is behind never refuses it. + +Because the current permissions are read regardless of the caller's read permission, the error can reveal one fact to +a caller who cannot read a related document: whether a permission it sends is already on that document. If it is, this +check passes and the relationship write refuses the document for its read or update reason instead. Both refusals are +`Exception\Authorization`. If that difference matters to you, answer every `Exception\Authorization` with the same +response. + +## Fractional operands on integer attributes + +Opt-in. By default 8.1 refuses fractions on integer attributes exactly as 8.0.2 and 8.0.3 do: +`increaseDocumentAttribute()` and `decreaseDocumentAttribute()` throw `Exception\Type`, and fractional operator limits +throw `Exception\Structure`. + +With `setRoundFractions(true)`, fractions on single-valued integer and big integer attributes are stored as the +adapter's engine stored them on 7.4.1, which handed them to the engine. This covers: + +- the change value and the `max` or `min` of `increaseDocumentAttribute()` and `decreaseDocumentAttribute()`; +- the amount and limit of `Operator::increment()`, `decrement()`, `multiply()`, `divide()`, `modulo()` and `power()` + given to `updateDocument()`, `updateDocuments()`, `upsertDocument()` and `upsertDocuments()`; +- fractional items added to an integer array by `arrayAppend()`, `arrayPrepend()` or `arrayInsert()`, which are cut + toward zero, as every engine read them back. + +Float and double attributes are unchanged. + +Each fractional operator is resolved from the stored value into the whole-number increment or decrement that stores +the engine's result. The stored value is read locked, inside the write's transaction. `updateDocuments()` and the +upserts read each row locked inside its batch's transaction, then send one update per group of rows with the same +resolved change. A row an upsert creates is resolved from 0. A write without a fractional operator on an integer +attribute takes the 8.0 path. Big integers stay exact beyond 2^53: the stored value never passes through a float. + +`Adapter::arithmetic()` declares each engine's rules: + +| Adapter | Counter change | Divide by a whole number | Power | Other operators | Refused | +|---|---|---|---|---|---| +| MariaDB, MySQL | Half to even | Exact, half away from zero | Half to even | Computed in `DOUBLE`, half to even | — | +| PostgreSQL | Refused | Exact, half away from zero | Double precision, half to even | `NUMERIC`, half away from zero; multiply, divide and modulo exact | A fractional counter change; a fractional amount of increment, decrement, multiply or divide without a limit | +| MongoDB | Toward zero | Toward zero | Toward zero | Toward zero | — | +| SQLite, Memory, Redis | Toward zero | Toward zero | Toward zero | Toward zero; a fractional modulus is cut toward zero first | A modulus that cuts to zero (`modulo(0.5)`) | + +An adapter that did not exist in 7.4.1 answers with the MariaDB and MySQL rules unless it overrides `arithmetic()`. +`Pool` asks the pooled adapter once per pool, and `ReadWritePool` answers wherever reads go. + +**Operator limits.** A limit is compared with the exact result, as given, and is never rounded. A result past the limit +leaves the stored value unchanged, as on 7.4.1. So does a power that has no value under a limit: 0 to a negative power, +or a negative base to a fractional power. Because the engine rounded after comparing, a stored whole number can pass a +fractional limit: on MariaDB, `increment(1.5, 13.5)` on 12 stores 14. + +**Counters.** + +- A whole change is an integer, integer text, numeric text with a whole value such as `'1e3'`, or a float without a + fraction. With a whole change, a fractional `max` is rounded down and a fractional `min` rounded up, reading text + exactly. Then the change runs as in 8.0. On PostgreSQL, a whole change with a fractional bound is refused as 7.4.1 + refused it: `Exception\Limit` if the exact result passes the bound, otherwise `Exception\Type` (`Max must be an + integer.` or `Min must be an integer.`). +- With a fractional change, the bound is used as given. The stored value is read locked. If the exact result passes + the bound, the call throws `Exception\Limit` before writing. Otherwise it writes the whole-number change the engine + would have stored, and the returned document carries the exact result as a float. On MariaDB, 5 + 1.5 with + `max` 6.5 returns 6.5 and stores 6, and 5 − 1.5 with `min` 3.5 returns 3.5 and stores 4. +- A whole-number change of 0 still writes. On MariaDB, 4 + 0.5 stores 4, moves `$updatedAt`, purges the caches and + fires the events. +- On PostgreSQL a fractional change throws `Exception\Type` (`Change value must be an integer.`), with or without a + bound. A bound the exact result passes throws `Exception\Limit` first. +- A change of 0 or less still throws `Exception\Type` (`Value must be numeric and greater than 0`). + +**Refusals.** An operator an engine refused, or one whose amount or limit is infinite or not a number, comes back +unresolved. Validation then refuses it with `Exception\Structure`, as by default. So does an amount or limit on +PostgreSQL's exact operators that is too long to compute exactly (more than 128 significant digits). A counter change +that is infinite or not a number throws `Exception\Type`. + +**MongoDB.** 7.4.1 stored the fractional double, every read cut it toward zero, and later operations continued from the +double. 8.1 stores the whole number after each operation, which is what every 7.4.1 read returned. This approximates +7.4.1 rather than reproducing it: two increments of 0.5 from 1 stored 2.0 on 7.4.1 and store 1 on 8.1. **SQLite.** +7.4.1 compared a fractional operator limit as text, so a fractional maximum never held and a fractional minimum always +held. 8.1 does not reproduce this: limits hold as on Memory and Redis. + +**Mirror.** `Mirror` forwards the toggle to its source and destination. + +- `increaseDocumentAttribute()`, `decreaseDocumentAttribute()`, `updateDocuments()` and the upserts are replayed on + the destination with the caller's arguments. The destination resolves them with its own adapter's rules and its own + stored values. If it refuses one, for example a fractional counter on a PostgreSQL destination, the failure only + reaches `onError()`. +- `updateDocument()` replicates the values the source stored. + +`Validator\Operator` and `Validator\Structure` (and `Validator\PartialStructure`) take a trailing +`bool $fractions = false` for code that validates operators itself. When it is set, a finite fractional amount or limit +within the attribute's range passes, and the caller resolves it before writing, for example with +`Arithmetic\Fractional::operator()`. + +```php +$database->setRoundFractions(true); + +// MariaDB: stores 6 and returns 6.5. PostgreSQL: throws Exception\Type, as 7.4.1 did. +$document = $database->increaseDocumentAttribute('posts', 'p1', 'likes', 1.5, max: 6.5); +``` + +## Cursor from an ID + +Additive. `Database::resolveCursor(string $collection, string $id, CursorDirection $direction, array $queries = []): +Document` returns the cursor a list pages from when a client names its last row by document ID. Pass the list's +`$queries` (its cursor queries are left out) and put the result in the list's `cursorAfter()` or `cursorBefore()`. + +- The document is read with authorization skipped, so a list can page past a document its caller cannot read. +- The cursor holds only `$id`, `$sequence`, `$collection`, the values the list orders by and the `alias.$id` + tie-breaks `find()` adds for a join. An order on a joined attribute takes its value from the joined rows the caller + can read that the list pairs with the document. A page after the document starts behind the last of those rows, and + a page before it ends ahead of the first. With no such row, the value is null. +- It returns an empty `Document` when no document has the ID. It throws `Exception\NotFound` for a missing collection, + and `Exception\Query` for invalid queries or joins. + +The caller chooses the orders, and the values come from a document it may not be allowed to read. Never return the +cursor, or any value in it, to an end user. Only pass it to `cursorAfter()` or `cursorBefore()`. + +```php +$cursor = $database->resolveCursor('posts', $lastId, CursorDirection::After, $queries); +if ($cursor->isEmpty()) { + // no such document +} +$page = $database->find('posts', [...$queries, Query::cursorAfter($cursor)]); +``` + +## One read for documents and aggregates + +Additive. `find()` still refuses aggregate and `groupBy()` queries, as 8.0 decided. A caller that serves both through +one list can use `Database::read(string $collection, array $queries = [], PermissionType $forPermission = +PermissionType::Read): array`. It returns a list of `Document`: + +- aggregate or `groupBy()` queries return the rows of `aggregate()`, each wrapped in a `Document`. Aggregates are read + under the read permission only, so another `$forPermission` throws `Exception\Query`. +- any other queries return what `find($collection, $queries, $forPermission)` returns. + +`ParsedQuery::isAggregation(): bool` (from `Query::groupByType($queries)`) tells which of the two a list is. + +## Joining on a to-many relationship + +Changed for joins that 8.0 refused. Nothing else changes. The parent side of a one-to-many relationship, and the child +side of a two-way many-to-one relationship, hold no column: the related documents store this document's ID under the +two-way key. 8.0 refused a join on such an attribute (`Cannot join on virtual relationship attribute: `). 8.1 +reads it when all of the following hold: + +- the condition is `Query::on(, '$id')` or `Query::on(, '.$id')` in the + join's ON list; +- the joined collection is the relationship's related collection; +- the relationship has a two-way key. + +The condition then compares this document's `$id` with `.`, keeping its operator and alias. The +caller's queries are not changed. `find()`, `count()`, `sum()`, `aggregate()`, `getDocument()` and `resolveCursor()` +all do this. + +The parent side of a many-to-one, one-to-one and many-to-many relationships, a one-way many-to-one child side, a join +of another collection and a column of an earlier join are refused as in 8.0. The read rule for joined collections is +unchanged (see [Joins](#joins)). + +```php +// authors.books is the parent side of a one-to-many relationship with books.author +$database->find('authors', [ + Query::join('books', 'b', [Query::on('books', '$id')]), // reads as $id = b.author + Query::select(['name', 'b.title']), +]); +``` + +## Spatial support behind a pool + +No change. 8.0.3 already answers correctly, and 8.1 adds tests for it. Ask +`$adapter->hasFeature(Feature\Spatial::class)` or `$database->profile()->hasFeature(Feature\Spatial::class)`, never +`instanceof Feature\Spatial`. `Pool` and `ReadWritePool` do not implement the feature interfaces; they answer +`hasFeature()` for the adapter they pool. A `Mirror`'s `getAdapter()` is its source's adapter, so it answers for the +source. Creating a `point` attribute fails exactly when `hasFeature(Feature\Spatial::class)` is false. + +## `State\Snapshot` + +`State\Snapshot` is `@internal`, as `Database::snapshot()` and `withSnapshot()` already were. Its constructor takes +three more required arguments, `replacePlaceholders`, `checkGrants` and `roundFractions`, which `Database::snapshot()` +fills in. Code that builds a `Snapshot` itself has to pass them. + # Upgrading from 7.x to 8.0 This guide lists the changes you may need to make when you move from utopia-php/database 7.x (last release 7.4.1) From fb945b110251fb047df08d609d997f34b37680f8 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 18:56:16 +1300 Subject: [PATCH 36/55] refactor(joins): keep refusing joins on relationship attributes without a column 8.0.3 refuses a join that compares a one-to-many parent attribute, or the child side of a two-way many-to-one, with the related $id: the attribute has no column to match. Rewriting such a join onto the related collection's two-way key reads rows 8.0.3 refused, a changed default the 8.1.0 brief never asked for. With the rewrite gone getDocument(), find(), aggregate(), count(), sum() and resolveCursor() answer these joins as 8.0.3 did, and JoinInternalColumnsTest is back to its 8.0.3 assertions. The aggregate join read-rule coverage stays, in a trait named for what it covers. Co-Authored-By: Claude Opus 5.5 --- src/Database/Trait/Cursors.php | 4 +- src/Database/Trait/Documents.php | 4 - src/Database/Trait/Joins.php | 97 ----- tests/e2e/Adapter/Base.php | 4 +- tests/e2e/Adapter/Scopes/CursorTests.php | 70 ---- .../e2e/Adapter/Scopes/JoinReadRuleTests.php | 148 +++++++ .../Adapter/Scopes/JoinRelationshipTests.php | 396 ------------------ tests/unit/Documents/ResolveCursorTest.php | 26 -- tests/unit/Joins/JoinInternalColumnsTest.php | 13 +- tests/unit/Joins/RelationshipJoinTest.php | 339 --------------- 10 files changed, 155 insertions(+), 946 deletions(-) create mode 100644 tests/e2e/Adapter/Scopes/JoinReadRuleTests.php delete mode 100644 tests/e2e/Adapter/Scopes/JoinRelationshipTests.php delete mode 100644 tests/unit/Joins/RelationshipJoinTest.php diff --git a/src/Database/Trait/Cursors.php b/src/Database/Trait/Cursors.php index 5fd25c80d..855232c12 100644 --- a/src/Database/Trait/Cursors.php +++ b/src/Database/Trait/Cursors.php @@ -49,10 +49,10 @@ public function resolveCursor(string $collection, string $id, CursorDirection $d $definition = $this->silent(fn (): Collection => $this->getCollection($collection)); $this->checkQueryTypes($queries); - $queries = $this->rewriteRelationshipJoins($definition, \array_values(\array_filter( + $queries = \array_values(\array_filter( $queries, static fn (Query $query): bool => $query->getMethod() !== Method::CursorAfter && $query->getMethod() !== Method::CursorBefore, - ))); + )); $joinedById = $this->resolveJoinedCollections($queries); if ($this->validation()->get()) { diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 54d5e0d4d..633c2bb08 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -494,7 +494,6 @@ private function readDocument(Collection $collection, string $id, array $queries $attributes = $collection->attributes(); $this->checkQueryTypes($queries); - $queries = $this->rewriteRelationshipJoins($collection, $queries); $joinedCollections = null; @@ -3985,7 +3984,6 @@ private function fetch(Collection $collection, array $queries, PermissionType $f $queryCacheQueries = $queries; $this->checkQueryTypes($queries); - $queries = $this->rewriteRelationshipJoins($collection, $queries); $joinedCollectionsById = null; @@ -4356,7 +4354,6 @@ public function count(string $collection, array $queries = [], ?int $max = null) $collection = $this->silent(fn () => $this->getCollection($collection)); $this->checkQueryTypes($queries); - $queries = $this->rewriteRelationshipJoins($collection, $queries); $joinedCollections = null; @@ -4413,7 +4410,6 @@ public function sum(string $collection, string $attribute, array $queries = [], $collection = $this->silent(fn () => $this->getCollection($collection)); $this->checkQueryTypes($queries); - $queries = $this->rewriteRelationshipJoins($collection, $queries); $joinedCollections = null; diff --git a/src/Database/Trait/Joins.php b/src/Database/Trait/Joins.php index 5d326c365..4cd6e8cdf 100644 --- a/src/Database/Trait/Joins.php +++ b/src/Database/Trait/Joins.php @@ -10,110 +10,13 @@ use Utopia\Database\Exception\Query as QueryException; use Utopia\Database\PermissionType; use Utopia\Database\Query; -use Utopia\Database\RelationshipSide; -use Utopia\Database\RelationshipType; use Utopia\Database\Validator\Authorization\Input; use Utopia\Database\Validator\Query\Join as JoinValidator; use Utopia\Query\Method; -use Utopia\Query\Query as BaseQuery; use Utopia\Query\Schema\IndexType; trait Joins { - /** - * A one-to-many parent attribute, or the child side of a two-way many-to-one, has no column: the related - * collection stores this document's ID under the two-way key. A join comparing such an attribute with the related - * `$id` compares this document's `$id` with that key instead. Rewritten joins are copies of the caller's. - * - * @param array $queries - * @return array - */ - private function rewriteRelationshipJoins(Collection $collection, array $queries): array - { - $relationships = null; - - foreach ($queries as $index => $query) { - if (! $query->getMethod()->isJoin()) { - continue; - } - - $relationships ??= self::joinableRelationships($collection); - if ($relationships === []) { - return $queries; - } - - $joined = $query->getAttribute(); - $alias = $query->getAlias(); - $rewritten = null; - - foreach ($query->getJoinOnQueries() as $position => $condition) { - $column = self::relationshipJoinColumn($condition, $joined, $alias, $relationships); - if ($column === null) { - continue; - } - - $rewritten ??= clone $query; - $rewritten->getJoinOnQueries()[$position]->setValues([Document::ID, $condition->getValues()[1] ?? '=', $alias.'.'.$column]); - } - - if ($rewritten !== null) { - $queries[$index] = $rewritten; - } - } - - return $queries; - } - - /** - * @return array - */ - private static function joinableRelationships(Collection $collection): array - { - $relationships = []; - foreach (self::relationshipAttributes($collection) as $attribute) { - $relationship = $attribute->relationship; - if ($relationship === null || $relationship->twoWayKey === null || $relationship->twoWayKey === '') { - continue; - } - - $storedByRelated = match ($relationship->type) { - RelationshipType::OneToMany => $attribute->side === RelationshipSide::Parent, - RelationshipType::ManyToOne => $attribute->side === RelationshipSide::Child && $relationship->twoWay, - RelationshipType::OneToOne, RelationshipType::ManyToMany => false, - }; - - if ($storedByRelated) { - $relationships[$attribute->key] = $attribute; - } - } - - return $relationships; - } - - /** - * @param array $relationships - */ - private static function relationshipJoinColumn(BaseQuery $condition, string $joined, string $alias, array $relationships): ?string - { - if ($condition->getMethod() !== Method::On) { - return null; - } - - $values = $condition->getValues(); - $left = $values[0] ?? null; - $right = $values[2] ?? null; - if (! \is_string($left) || ($right !== Document::ID && $right !== $alias.'.'.Document::ID)) { - return null; - } - - $relationship = ($relationships[$left] ?? null)?->relationship; - if ($relationship === null || $relationship->relatedCollection !== $joined) { - return null; - } - - return $relationship->twoWayKey; - } - /** * The collections the join queries name, each loaded once, by the id the join names it with. A * read resolves them once and hands them to validation, authorization, the adapter and decoding. diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index ec98cfe90..e8820d99b 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -16,7 +16,7 @@ use Tests\E2E\Adapter\Scopes\GeneralTests; use Tests\E2E\Adapter\Scopes\IndexTests; use Tests\E2E\Adapter\Scopes\JoinComboTests; -use Tests\E2E\Adapter\Scopes\JoinRelationshipTests; +use Tests\E2E\Adapter\Scopes\JoinReadRuleTests; use Tests\E2E\Adapter\Scopes\JoinTests; use Tests\E2E\Adapter\Scopes\MetadataCacheTests; use Tests\E2E\Adapter\Scopes\ObjectAttributeTests; @@ -60,7 +60,7 @@ abstract class Base extends TestCase use GeneralTests; use IndexTests; use JoinComboTests; - use JoinRelationshipTests; + use JoinReadRuleTests; use JoinTests; use MetadataCacheTests; use ObjectAttributeTests; diff --git a/tests/e2e/Adapter/Scopes/CursorTests.php b/tests/e2e/Adapter/Scopes/CursorTests.php index 056016507..f8f7d0844 100644 --- a/tests/e2e/Adapter/Scopes/CursorTests.php +++ b/tests/e2e/Adapter/Scopes/CursorTests.php @@ -2,7 +2,6 @@ namespace Tests\E2E\Adapter\Scopes; -use Utopia\Database\Adapter\Feature; use Utopia\Database\Attribute; use Utopia\Database\Capability; use Utopia\Database\Collection; @@ -10,7 +9,6 @@ use Utopia\Database\Document; use Utopia\Database\Permission; use Utopia\Database\Query; -use Utopia\Database\Relationship; use Utopia\Database\Role; use Utopia\Query\CursorDirection; @@ -101,74 +99,6 @@ public function testResolveCursorTakesAJoinedOrderFromTheRowsTheCallerCanRead(): } } - public function testResolveCursorOverAJoinOnAOneToManyRelationshipPagesEveryRowOnce(): void - { - $database = $this->getDatabase(); - if (! $database->getAdapter()->supports(Capability::Joins) || ! $database->getAdapter()->hasFeature(Feature\Relationships::class)) { - $this->expectNotToPerformAssertions(); - - return; - } - - $this->seedCursorCollections($database); - - try { - $database->createRelationship(self::CURSOR_CUSTOMERS, Relationship::oneToMany(relatedCollection: self::CURSOR_ORDERS, key: 'orders', twoWayKey: 'customer')); - - foreach (['alice' => 'Alice', 'bob' => 'Bob', 'carol' => 'Carol'] as $id => $name) { - $this->createCursorDocument($database, self::CURSOR_CUSTOMERS, $id, ['name' => $name], readable: true); - } - foreach ([ - ['a-first', 'alice', 10, true], - ['a-second', 'alice', 20, true], - ['b-first', 'bob', 30, true], - ['b-second', 'bob', 40, true], - ['c-hidden', 'carol', 5, false], - ['c-only', 'carol', 50, true], - ] as [$id, $customer, $amount, $readable]) { - $this->createCursorDocument($database, self::CURSOR_ORDERS, $id, ['customer' => $customer, 'amount' => $amount, 'status' => 'paid'], $readable); - } - - $queries = [Query::join(self::CURSOR_ORDERS, 'ord', [Query::on('orders', '$id')]), Query::orderAsc('ord.amount')]; - $all = $this->cursorRows($database->find(self::CURSOR_CUSTOMERS, [...$queries, Query::limit(100)])); - $this->assertSame([['alice', 'a-first'], ['alice', 'a-second'], ['bob', 'b-first'], ['bob', 'b-second'], ['carol', 'c-only']], $all); - - $forward = []; - $cursor = null; - for ($page = 0; $page <= \count($all); $page++) { - $rows = $this->cursorRows($database->find(self::CURSOR_CUSTOMERS, [ - ...$queries, - Query::limit(2), - ...($cursor === null ? [] : [Query::cursorAfter($database->resolveCursor(self::CURSOR_CUSTOMERS, $cursor, CursorDirection::After, $queries))]), - ])); - if ($rows === []) { - break; - } - \array_push($forward, ...$rows); - $cursor = $rows[\count($rows) - 1][0]; - } - $this->assertSame($all, $forward, 'paging forward by the ID of each page\'s last row returns every row once'); - - $backward = [['carol', 'c-only']]; - $cursor = 'carol'; - for ($page = 0; $page <= \count($all); $page++) { - $rows = $this->cursorRows($database->find(self::CURSOR_CUSTOMERS, [ - ...$queries, - Query::limit(2), - Query::cursorBefore($database->resolveCursor(self::CURSOR_CUSTOMERS, $cursor, CursorDirection::Before, $queries)), - ])); - if ($rows === []) { - break; - } - $backward = [...$rows, ...$backward]; - $cursor = $rows[0][0]; - } - $this->assertSame($all, $backward, 'paging backward by the ID of each page\'s first row returns every row once'); - } finally { - $this->deleteCursorCollections($database); - } - } - private function seedCursorCollections(Database $database): void { $this->deleteCursorCollections($database); diff --git a/tests/e2e/Adapter/Scopes/JoinReadRuleTests.php b/tests/e2e/Adapter/Scopes/JoinReadRuleTests.php new file mode 100644 index 000000000..1e1270673 --- /dev/null +++ b/tests/e2e/Adapter/Scopes/JoinReadRuleTests.php @@ -0,0 +1,148 @@ +getDatabase(); + if (! $database->getAdapter()->supports(Capability::Joins) || ! $database->getAdapter()->supports(Capability::Aggregations)) { + $this->expectNotToPerformAssertions(); + + return; + } + + [$main, $joined] = $this->seedJoinReadRule($database, 'jrule_deny', joinedPermissions: [Permission::create(Role::any())], joinedDocumentSecurity: false); + + try { + $queries = [Query::join($joined, 'entry', [Query::on('$id', 'mainId')]), Query::count('*', 'rows')]; + + $this->withJoinReadRuleRoles($database, [Role::any()->toString()], function () use ($database, $main, $queries): void { + try { + $database->aggregate($main, $queries); + $this->fail('aggregate() must refuse a joined collection readable neither at collection nor at document level'); + } catch (AuthorizationException $exception) { + $this->assertStringContainsString('jrule_deny_j', $exception->getMessage()); + } + }); + + $rows = $database->getAuthorization()->skip(fn (): array => $database->aggregate($main, $queries)); + $this->assertSame(2, self::joinReadRuleInteger($rows[0]['rows']), 'with authorization disabled every joined row is read'); + } finally { + $this->deleteJoinReadRuleCollections($database, $main, $joined); + } + } + + public function testJoinReadRuleAggregateFiltersAJoinedCollectionPerDocument(): void + { + $database = $this->getDatabase(); + if (! $database->getAdapter()->supports(Capability::Joins) || ! $database->getAdapter()->supports(Capability::Aggregations)) { + $this->expectNotToPerformAssertions(); + + return; + } + + [$main, $joined] = $this->seedJoinReadRule($database, 'jrule_owned', joinedPermissions: [Permission::create(Role::any())], joinedDocumentSecurity: true); + + try { + $this->withJoinReadRuleRoles($database, [Role::any()->toString(), Role::user('alice')->toString()], function () use ($database, $main, $joined): void { + $joinedRows = $database->aggregate($joined, [Query::count('*', 'rows'), Query::sum('amount', 'total')]); + $rows = $database->aggregate($main, [ + Query::join($joined, 'entry', [Query::on('$id', 'mainId')]), + Query::count('*', 'rows'), + Query::sum('entry.amount', 'total'), + ]); + + $this->assertSame([1, 5], [self::joinReadRuleInteger($joinedRows[0]['rows']), self::joinReadRuleInteger($joinedRows[0]['total'])], 'a direct aggregate reads only the rows the caller holds read on'); + $this->assertSame([1, 5], [self::joinReadRuleInteger($rows[0]['rows']), self::joinReadRuleInteger($rows[0]['total'])], 'an aggregate over the join reads the same rows'); + }); + } finally { + $this->deleteJoinReadRuleCollections($database, $main, $joined); + } + } + + /** + * A main collection every caller reads, with one document, and a joined collection holding two entries for it: + * one readable by user alice and one by user bob. + * + * @param list $joinedPermissions + * @return array{string, string} + */ + private function seedJoinReadRule(Database $database, string $prefix, array $joinedPermissions, bool $joinedDocumentSecurity): array + { + $main = $prefix.'_m'; + $joined = $prefix.'_j'; + $this->deleteJoinReadRuleCollections($database, $main, $joined); + + $database->createCollection(Collection::create( + id: $main, + attributes: [Attribute::string(key: 'name', size: 16)], + permissions: [Permission::create(Role::any()), Permission::read(Role::any())], + )); + $database->createCollection(Collection::create( + id: $joined, + attributes: [Attribute::string(key: 'mainId', size: 36), Attribute::integer(key: 'amount')], + permissions: $joinedPermissions, + documentSecurity: $joinedDocumentSecurity, + )); + + $database->createDocument($main, new Document(['$id' => 'customer', 'name' => 'Customer', '$permissions' => [Permission::read(Role::any())]])); + $database->getAuthorization()->skip(function () use ($database, $joined): void { + $database->createDocument($joined, new Document(['$id' => 'alice-entry', 'mainId' => 'customer', 'amount' => 5, '$permissions' => [Permission::read(Role::user('alice'))]])); + $database->createDocument($joined, new Document(['$id' => 'bob-entry', 'mainId' => 'customer', 'amount' => 7, '$permissions' => [Permission::read(Role::user('bob'))]])); + }); + + return [$main, $joined]; + } + + private function deleteJoinReadRuleCollections(Database $database, string ...$collections): void + { + foreach ($collections as $collection) { + if ($database->collectionExists($collection)) { + $database->deleteCollection($collection); + } + } + } + + /** + * @param list $roles + * @param callable(): void $callback + */ + private function withJoinReadRuleRoles(Database $database, array $roles, callable $callback): void + { + $authorization = $database->getAuthorization(); + $previous = $authorization->getRoles(); + $authorization->cleanRoles(); + foreach ($roles as $role) { + $authorization->addRole($role); + } + + try { + $callback(); + } finally { + $authorization->cleanRoles(); + foreach ($previous as $role) { + $authorization->addRole($role); + } + } + } + + private static function joinReadRuleInteger(mixed $value): ?int + { + return \is_numeric($value) ? (int) $value : null; + } +} diff --git a/tests/e2e/Adapter/Scopes/JoinRelationshipTests.php b/tests/e2e/Adapter/Scopes/JoinRelationshipTests.php deleted file mode 100644 index f3c6e9b0f..000000000 --- a/tests/e2e/Adapter/Scopes/JoinRelationshipTests.php +++ /dev/null @@ -1,396 +0,0 @@ -getDatabase(); - if (! $this->supportsJoinRelationships($database)) { - $this->expectNotToPerformAssertions(); - - return; - } - - $this->seedJoinRelationships($database); - - try { - $this->assertSame( - ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4', 'c3/-'], - $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, Query::leftJoin(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id')]), 'purchase'), - 'a left join keeps the customer without orders', - ); - $this->assertSame( - ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], - $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', 'purchase.$id')]), 'purchase'), - ); - $this->assertSame( - ['c1/o3', 'c1/o4', 'c2/o1', 'c2/o2', 'c3/o1', 'c3/o2', 'c3/o3', 'c3/o4'], - $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id', '!=')]), 'purchase'), - 'the operator of the condition is kept', - ); - $this->assertSame( - ['c1/o1', 'c2/o4'], - $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [ - Query::on('orders', '$id'), - Query::on('region', 'region'), - Query::equal('purchase.status', ['paid']), - ]), 'purchase'), - 'the other conditions of the ON list compare what they named', - ); - - $rows = $database->find(self::JOIN_RELATIONSHIP_CUSTOMERS, [ - Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id')]), - Query::select(['name', 'purchase.amount']), - Query::orderAsc('purchase.amount'), - ]); - $this->assertSame([10, 20, 30, 40], \array_map(static fn (Document $row): ?int => self::joinRelationshipInteger($row->getAttribute('purchase.amount')), $rows)); - $this->assertSame(['Ada', 'Ada', 'Bob', 'Bob'], \array_map(static fn (Document $row): mixed => $row->getAttribute('name'), $rows)); - } finally { - $this->deleteJoinRelationshipCollections($database, ...self::JOIN_RELATIONSHIP_COLLECTIONS); - } - } - - public function testJoinRelationshipOnTheChildSideOfATwoWayManyToOneMatchesTheParentKey(): void - { - $database = $this->getDatabase(); - if (! $this->supportsJoinRelationships($database)) { - $this->expectNotToPerformAssertions(); - - return; - } - - $this->seedJoinRelationships($database); - - try { - $this->assertSame( - ['r1/c1', 'r1/c3', 'r2/c2'], - $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_REGIONS, Query::join(self::JOIN_RELATIONSHIP_CUSTOMERS, 'resident', [Query::on('customers', '$id')]), 'resident'), - ); - $this->assertSame( - ['c1/r1', 'c2/r2', 'c3/r1'], - $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, Query::join(self::JOIN_RELATIONSHIP_REGIONS, 'area', [Query::on('region', '$id')]), 'area'), - 'the parent side holds the related ID in its own column', - ); - } finally { - $this->deleteJoinRelationshipCollections($database, ...self::JOIN_RELATIONSHIP_COLLECTIONS); - } - } - - public function testJoinRelationshipIsMatchedByEveryRead(): void - { - $database = $this->getDatabase(); - if (! $this->supportsJoinRelationships($database)) { - $this->expectNotToPerformAssertions(); - - return; - } - - $this->seedJoinRelationships($database); - - try { - $join = Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id')]); - - $this->assertSame(4, $database->count(self::JOIN_RELATIONSHIP_CUSTOMERS, [$join])); - $this->assertSame(100, (int) $database->sum(self::JOIN_RELATIONSHIP_CUSTOMERS, 'purchase.amount', [$join])); - $this->assertSame( - [['Ada', 30], ['Bob', 70]], - \array_map( - static fn (array $row): array => [$row['name'], self::joinRelationshipInteger($row['total'])], - $database->aggregate(self::JOIN_RELATIONSHIP_CUSTOMERS, [$join, Query::sum('purchase.amount', 'total'), Query::groupBy(['name']), Query::orderAsc('name')]), - ), - ); - $this->assertSame( - 20, - self::joinRelationshipInteger($database->getDocument(self::JOIN_RELATIONSHIP_CUSTOMERS, 'c1', [ - Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id'), Query::equal('purchase.status', ['open'])]), - Query::select(['name', 'purchase.amount']), - ])->getAttribute('purchase.amount')), - ); - } finally { - $this->deleteJoinRelationshipCollections($database, ...self::JOIN_RELATIONSHIP_COLLECTIONS); - } - } - - public function testJoinRelationshipKeepsRefusingAttributesWithoutAColumnToMatch(): void - { - $database = $this->getDatabase(); - if (! $this->supportsJoinRelationships($database)) { - $this->expectNotToPerformAssertions(); - - return; - } - - $this->seedJoinRelationships($database); - - try { - $database->createCollection(Collection::create( - id: self::JOIN_RELATIONSHIP_WAREHOUSES, - attributes: [Attribute::string(key: 'name', size: 16)], - permissions: [Permission::create(Role::any()), Permission::read(Role::any())], - )); - $database->createRelationship(self::JOIN_RELATIONSHIP_ORDERS, Relationship::manyToOne(relatedCollection: self::JOIN_RELATIONSHIP_WAREHOUSES, key: 'warehouse', twoWayKey: 'stock')); - - $refusals = [ - 'the child side of a one-way many-to-one' => [self::JOIN_RELATIONSHIP_WAREHOUSES, Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('stock', '$id')]), 'stock'], - 'a join of another collection than the related one' => [self::JOIN_RELATIONSHIP_CUSTOMERS, Query::join(self::JOIN_RELATIONSHIP_REGIONS, 'area', [Query::on('orders', '$id')]), 'orders'], - 'a comparison with another column than the related $id' => [self::JOIN_RELATIONSHIP_CUSTOMERS, Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', 'status')]), 'orders'], - ]; - - foreach ($refusals as $case => [$collection, $join, $attribute]) { - try { - $database->find($collection, [$join]); - $this->fail("{$case} must be refused"); - } catch (QueryException $exception) { - $this->assertStringContainsString('Cannot join on virtual relationship attribute: '.$attribute, $exception->getMessage(), $case); - } - } - } finally { - $this->deleteJoinRelationshipCollections($database, ...self::JOIN_RELATIONSHIP_COLLECTIONS); - } - } - - public function testJoinRelationshipAggregateRefusesAJoinedCollectionWithoutGrantOrDocumentSecurity(): void - { - $database = $this->getDatabase(); - if (! $database->getAdapter()->supports(Capability::Joins) || ! $database->getAdapter()->supports(Capability::Aggregations)) { - $this->expectNotToPerformAssertions(); - - return; - } - - [$main, $joined] = $this->seedJoinRelationshipReadRule($database, 'jrel_deny', joinedPermissions: [Permission::create(Role::any())], joinedDocumentSecurity: false); - - try { - $queries = [Query::join($joined, 'entry', [Query::on('$id', 'mainId')]), Query::count('*', 'rows')]; - - $this->withJoinRelationshipRoles($database, [Role::any()->toString()], function () use ($database, $main, $queries): void { - try { - $database->aggregate($main, $queries); - $this->fail('aggregate() must refuse a joined collection readable neither at collection nor at document level'); - } catch (AuthorizationException $exception) { - $this->assertStringContainsString('jrel_deny_j', $exception->getMessage()); - } - }); - - $rows = $database->getAuthorization()->skip(fn (): array => $database->aggregate($main, $queries)); - $this->assertSame(2, self::joinRelationshipInteger($rows[0]['rows']), 'with authorization disabled every joined row is read'); - } finally { - $this->deleteJoinRelationshipCollections($database, $main, $joined); - } - } - - public function testJoinRelationshipAggregateFiltersAJoinedCollectionPerDocument(): void - { - $database = $this->getDatabase(); - if (! $database->getAdapter()->supports(Capability::Joins) || ! $database->getAdapter()->supports(Capability::Aggregations)) { - $this->expectNotToPerformAssertions(); - - return; - } - - [$main, $joined] = $this->seedJoinRelationshipReadRule($database, 'jrel_owned', joinedPermissions: [Permission::create(Role::any())], joinedDocumentSecurity: true); - - try { - $this->withJoinRelationshipRoles($database, [Role::any()->toString(), Role::user('alice')->toString()], function () use ($database, $main, $joined): void { - $joinedRows = $database->aggregate($joined, [Query::count('*', 'rows'), Query::sum('amount', 'total')]); - $rows = $database->aggregate($main, [ - Query::join($joined, 'entry', [Query::on('$id', 'mainId')]), - Query::count('*', 'rows'), - Query::sum('entry.amount', 'total'), - ]); - - $this->assertSame([1, 5], [self::joinRelationshipInteger($joinedRows[0]['rows']), self::joinRelationshipInteger($joinedRows[0]['total'])], 'a direct aggregate reads only the rows the caller holds read on'); - $this->assertSame([1, 5], [self::joinRelationshipInteger($rows[0]['rows']), self::joinRelationshipInteger($rows[0]['total'])], 'an aggregate over the join reads the same rows'); - }); - } finally { - $this->deleteJoinRelationshipCollections($database, $main, $joined); - } - } - - private function supportsJoinRelationships(Database $database): bool - { - $adapter = $database->getAdapter(); - - return $adapter->supports(Capability::Joins) - && $adapter->supports(Capability::Aggregations) - && $adapter->hasFeature(Feature\Relationships::class); - } - - /** - * Customers c1 (region r1, orders o1 and o2), c2 (region r2, orders o3 and o4) and c3 (region r1, no orders). - */ - private function seedJoinRelationships(Database $database): void - { - $this->deleteJoinRelationshipCollections($database, ...self::JOIN_RELATIONSHIP_COLLECTIONS); - - $permissions = [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())]; - $database->createCollection(Collection::create( - id: self::JOIN_RELATIONSHIP_CUSTOMERS, - attributes: [Attribute::string(key: 'name', size: 16)], - permissions: $permissions, - )); - $database->createCollection(Collection::create( - id: self::JOIN_RELATIONSHIP_ORDERS, - attributes: [ - Attribute::integer(key: 'amount'), - Attribute::string(key: 'status', size: 16), - Attribute::string(key: 'region', size: 16), - ], - permissions: $permissions, - )); - $database->createCollection(Collection::create( - id: self::JOIN_RELATIONSHIP_REGIONS, - attributes: [Attribute::string(key: 'name', size: 16)], - permissions: $permissions, - )); - - $database->createRelationship(self::JOIN_RELATIONSHIP_CUSTOMERS, Relationship::oneToMany(relatedCollection: self::JOIN_RELATIONSHIP_ORDERS, key: 'orders', twoWayKey: 'customer')); - $database->createRelationship(self::JOIN_RELATIONSHIP_CUSTOMERS, Relationship::manyToOne(relatedCollection: self::JOIN_RELATIONSHIP_REGIONS, key: 'region', twoWay: true, twoWayKey: 'customers')); - - $documents = [ - [self::JOIN_RELATIONSHIP_REGIONS, 'r1', ['name' => 'north']], - [self::JOIN_RELATIONSHIP_REGIONS, 'r2', ['name' => 'south']], - [self::JOIN_RELATIONSHIP_ORDERS, 'o1', ['amount' => 10, 'status' => 'paid', 'region' => 'r1']], - [self::JOIN_RELATIONSHIP_ORDERS, 'o2', ['amount' => 20, 'status' => 'open', 'region' => 'r1']], - [self::JOIN_RELATIONSHIP_ORDERS, 'o3', ['amount' => 30, 'status' => 'paid', 'region' => 'r1']], - [self::JOIN_RELATIONSHIP_ORDERS, 'o4', ['amount' => 40, 'status' => 'paid', 'region' => 'r2']], - [self::JOIN_RELATIONSHIP_CUSTOMERS, 'c1', ['name' => 'Ada', 'region' => 'r1', 'orders' => ['o1', 'o2']]], - [self::JOIN_RELATIONSHIP_CUSTOMERS, 'c2', ['name' => 'Bob', 'region' => 'r2', 'orders' => ['o3', 'o4']]], - [self::JOIN_RELATIONSHIP_CUSTOMERS, 'c3', ['name' => 'Cy', 'region' => 'r1']], - ]; - - foreach ($documents as [$collection, $id, $attributes]) { - $database->createDocument($collection, new Document([ - '$id' => $id, - '$permissions' => [Permission::read(Role::any())], - ...$attributes, - ])); - } - } - - private function deleteJoinRelationshipCollections(Database $database, string ...$collections): void - { - foreach ($collections as $collection) { - if ($database->collectionExists($collection)) { - $database->deleteCollection($collection); - } - } - } - - /** - * Each row as `
/`, sorted, with `-` for a row the outer join did not match. - * - * @return list - */ - private function joinRelationshipPairs(Database $database, string $collection, Query $join, string $alias): array - { - $pairs = \array_map( - static function (Document $row) use ($alias): string { - $joined = $row->getAttribute($alias.'.$id'); - - return $row->getId().'/'.(\is_string($joined) ? $joined : '-'); - }, - $database->find($collection, [$join, Query::limit(100)]), - ); - \sort($pairs); - - return $pairs; - } - - /** - * A main collection every caller reads, with one document, and a joined collection holding two entries for it: - * one readable by user alice and one by user bob. - * - * @param list $joinedPermissions - * @return array{string, string} - */ - private function seedJoinRelationshipReadRule(Database $database, string $prefix, array $joinedPermissions, bool $joinedDocumentSecurity): array - { - $main = $prefix.'_m'; - $joined = $prefix.'_j'; - $this->deleteJoinRelationshipCollections($database, $main, $joined); - - $database->createCollection(Collection::create( - id: $main, - attributes: [Attribute::string(key: 'name', size: 16)], - permissions: [Permission::create(Role::any()), Permission::read(Role::any())], - )); - $database->createCollection(Collection::create( - id: $joined, - attributes: [Attribute::string(key: 'mainId', size: 36), Attribute::integer(key: 'amount')], - permissions: $joinedPermissions, - documentSecurity: $joinedDocumentSecurity, - )); - - $database->createDocument($main, new Document(['$id' => 'customer', 'name' => 'Customer', '$permissions' => [Permission::read(Role::any())]])); - $database->getAuthorization()->skip(function () use ($database, $joined): void { - $database->createDocument($joined, new Document(['$id' => 'alice-entry', 'mainId' => 'customer', 'amount' => 5, '$permissions' => [Permission::read(Role::user('alice'))]])); - $database->createDocument($joined, new Document(['$id' => 'bob-entry', 'mainId' => 'customer', 'amount' => 7, '$permissions' => [Permission::read(Role::user('bob'))]])); - }); - - return [$main, $joined]; - } - - /** - * @param list $roles - * @param callable(): void $callback - */ - private function withJoinRelationshipRoles(Database $database, array $roles, callable $callback): void - { - $authorization = $database->getAuthorization(); - $previous = $authorization->getRoles(); - $authorization->cleanRoles(); - foreach ($roles as $role) { - $authorization->addRole($role); - } - - try { - $callback(); - } finally { - $authorization->cleanRoles(); - foreach ($previous as $role) { - $authorization->addRole($role); - } - } - } - - private static function joinRelationshipInteger(mixed $value): ?int - { - return \is_numeric($value) ? (int) $value : null; - } -} diff --git a/tests/unit/Documents/ResolveCursorTest.php b/tests/unit/Documents/ResolveCursorTest.php index 47aa32e07..2cabb2bcb 100644 --- a/tests/unit/Documents/ResolveCursorTest.php +++ b/tests/unit/Documents/ResolveCursorTest.php @@ -14,10 +14,8 @@ use Utopia\Database\Exception\NotFound as NotFoundException; use Utopia\Database\Exception\Query as QueryException; use Utopia\Database\Hook\Permissions; -use Utopia\Database\Hook\Relationships; use Utopia\Database\Permission; use Utopia\Database\Query; -use Utopia\Database\Relationship; use Utopia\Database\Role; use Utopia\Database\Validator\Authorization; use Utopia\Query\CursorDirection; @@ -48,7 +46,6 @@ protected function setUp(): void ->setAuthorization($authorization) ->setDatabase('resolve_cursor') ->setNamespace('resolve_cursor_'.\uniqid()); - $this->database->addHook(new Relationships()); $this->database->addHook(new Permissions()); $this->database->create(); @@ -293,27 +290,6 @@ public function testCursorOnAJoinOfTheJoinedIdNeedsNoJoinedRow(): void $this->assertSame(['a-second'], $this->ids($this->database->find(self::ORDERS, [...$queries, Query::cursorAfter($cursor)]))); } - public function testCursorOverAJoinOnAOneToManyRelationshipPagesEveryRowOnce(): void - { - $this->database->createRelationship(self::CUSTOMERS, Relationship::oneToMany(relatedCollection: self::ORDERS, key: 'orders', twoWayKey: 'customer')); - $this->customer('alice', readable: true); - $this->customer('bob', readable: true); - $this->customer('carol', readable: true); - $this->order('a-first', 'alice', 10, readable: true, customer: 'alice'); - $this->order('a-second', 'alice', 20, readable: true, customer: 'alice'); - $this->order('b-first', 'bob', 30, readable: true, customer: 'bob'); - $this->order('b-second', 'bob', 40, readable: true, customer: 'bob'); - $this->order('c-hidden', 'carol', 5, readable: false, customer: 'carol'); - $this->order('c-only', 'carol', 50, readable: true, customer: 'carol'); - - $queries = [Query::join(self::ORDERS, 'ord', [Query::on('orders', '$id')]), Query::orderAsc('ord.amount')]; - - $this->assertSame(20, $this->database->resolveCursor(self::CUSTOMERS, 'alice', CursorDirection::After, $queries)->getAttribute('ord.amount')); - $this->assertSame(50, $this->database->resolveCursor(self::CUSTOMERS, 'carol', CursorDirection::Before, $queries)->getAttribute('ord.amount'), 'the hidden order never pairs with carol'); - $this->assertWalksEveryRowOnce($queries, ['alice', 'alice', 'bob', 'bob', 'carol'], 2); - $this->assertCursorSplitsTheList($queries); - } - public function testCursorCarriesOnlyTheValuesTheListPagesBy(): void { $this->customer('alice', readable: true); @@ -486,7 +462,6 @@ private function order( string $status = 'paid', ?string $placedAt = null, ?string $productId = null, - ?string $customer = null, ): void { $this->create(self::ORDERS, $id, [ 'customerId' => $customerId, @@ -494,7 +469,6 @@ private function order( 'status' => $status, 'amount' => $amount, 'placedAt' => $placedAt, - ...($customer === null ? [] : ['customer' => $customer]), ], $readable); } diff --git a/tests/unit/Joins/JoinInternalColumnsTest.php b/tests/unit/Joins/JoinInternalColumnsTest.php index f69bee478..7cf423df5 100644 --- a/tests/unit/Joins/JoinInternalColumnsTest.php +++ b/tests/unit/Joins/JoinInternalColumnsTest.php @@ -237,17 +237,10 @@ public function testJoinConditionOverARelationshipNamesItsColumn(): void $this->assertSame([['rows' => 2]], $this->database->aggregate('persons', [Query::join('books', 'book', [Query::on('$id', 'owner')]), Query::count('*', 'rows')]), 'the child side of a one-to-many relationship holds a column'); $this->assertSame(2, $this->database->count('books', [Query::join('persons', 'person', [Query::on('owner', '$id')])])); - $owned = $this->database->find('persons', [ - Query::join('books', 'book', [Query::on('books', '$id')]), - Query::select(['name', 'book.title']), - Query::orderAsc('book.title'), - ]); - $this->assertSame( - [['Ada', 'One'], ['Ada', 'Two']], - \array_map(static fn (Document $person): array => [$person->getAttribute('name'), $person->getAttribute('book.title')], $owned), - 'the parent side of a one-to-many relationship joins on the two-way key its children store', + $this->assertInvalidQuery( + 'Invalid query: Cannot join on virtual relationship attribute: books', + fn (): mixed => $this->database->find('persons', [Query::join('books', 'book', [Query::on('books', '$id')])]), ); - $this->assertInvalidQuery( 'Invalid query: Cannot join on virtual relationship attribute: person', fn (): mixed => $this->database->find('persons', [Query::join('libraries', 'lib', [Query::on('$id', 'person')])]), diff --git a/tests/unit/Joins/RelationshipJoinTest.php b/tests/unit/Joins/RelationshipJoinTest.php deleted file mode 100644 index 62877338a..000000000 --- a/tests/unit/Joins/RelationshipJoinTest.php +++ /dev/null @@ -1,339 +0,0 @@ -database = new Database(new SQLite(new PDO('sqlite::memory:')), new Cache(new NoCache())); - $this->database - ->setDatabase('relationship_join') - ->setNamespace('relationship_join_'.\uniqid()) - ->setAuthorization(new Authorization()); - $this->database->addHook(new Relationships()); - $this->database->addHook(new Permissions()); - $this->database->create(); - - $this->createCollection('customers', [Attribute::string(key: 'name', size: 16)]); - $this->createCollection('orders', [ - Attribute::integer(key: 'amount'), - Attribute::string(key: 'status', size: 16), - Attribute::string(key: 'region', size: 16), - ]); - $this->createCollection('regions', [Attribute::string(key: 'name', size: 16)]); - - $this->database->createRelationship('customers', Relationship::oneToMany(relatedCollection: 'orders', key: 'orders', twoWayKey: 'customer')); - $this->database->createRelationship('customers', Relationship::manyToOne(relatedCollection: 'regions', key: 'region', twoWay: true, twoWayKey: 'customers')); - - $this->createDocument('regions', 'r1', ['name' => 'north']); - $this->createDocument('regions', 'r2', ['name' => 'south']); - - foreach ([ - 'o1' => [10, 'paid', 'r1'], - 'o2' => [20, 'open', 'r1'], - 'o3' => [30, 'paid', 'r1'], - 'o4' => [40, 'paid', 'r2'], - ] as $id => [$amount, $status, $region]) { - $this->createDocument('orders', $id, ['amount' => $amount, 'status' => $status, 'region' => $region]); - } - - $this->createDocument('customers', 'c1', ['name' => 'Ada', 'region' => 'r1', 'orders' => ['o1', 'o2']]); - $this->createDocument('customers', 'c2', ['name' => 'Bob', 'region' => 'r2', 'orders' => ['o3', 'o4']]); - $this->createDocument('customers', 'c3', ['name' => 'Cy', 'region' => 'r1']); - } - - public function testLeftJoinOnAOneToManyRelationshipPairsEachCustomerWithItsOrders(): void - { - $this->assertSame( - ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4', 'c3/-'], - $this->pairs('customers', [Query::leftJoin('orders', 'ord', [Query::on('orders', '$id')])], 'ord'), - ); - } - - public function testJoinOnAOneToManyRelationshipMatchesTheMainIdAgainstTheRelatedKey(): void - { - $this->assertSame( - ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], - $this->pairs('customers', [Query::join('orders', 'ord', [Query::on('orders', '$id')])], 'ord'), - 'a one-to-many relationship is stored on the related side, in its two-way key', - ); - } - - public function testJoinNamingTheRelatedIdUnderItsAliasIsRewritten(): void - { - $this->assertSame( - ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], - $this->pairs('customers', [Query::join('orders', 'ord', [Query::on('orders', 'ord.$id')])], 'ord'), - ); - } - - public function testRewrittenJoinKeepsItsAlias(): void - { - $rows = $this->database->find('customers', [ - Query::join('orders', 'purchase', [Query::on('orders', '$id')]), - Query::select(['name', 'purchase.amount']), - Query::orderAsc('purchase.amount'), - ]); - - $this->assertSame([10, 20, 30, 40], \array_map(static fn (Document $row): mixed => $row->getAttribute('purchase.amount'), $rows)); - $this->assertSame(['Ada', 'Ada', 'Bob', 'Bob'], \array_map(static fn (Document $row): mixed => $row->getAttribute('name'), $rows)); - } - - public function testOnlyTheRelationshipConditionOfTheOnListIsRewritten(): void - { - $this->assertSame( - ['c1/o1', 'c2/o4'], - $this->pairs('customers', [Query::join('orders', 'ord', [ - Query::on('orders', '$id'), - Query::on('region', 'region'), - Query::equal('ord.status', ['paid']), - ])], 'ord'), - 'the many-to-one column and the ON filter keep comparing what they named', - ); - } - - public function testOperatorOfTheRewrittenConditionIsKept(): void - { - $this->assertSame( - ['c1/o3', 'c1/o4', 'c2/o1', 'c2/o2', 'c3/o1', 'c3/o2', 'c3/o3', 'c3/o4'], - $this->pairs('customers', [Query::join('orders', 'ord', [Query::on('orders', '$id', '!=')])], 'ord'), - ); - } - - public function testConditionOnAnEarlierJoinsColumnIsNotTheMainRelationship(): void - { - $this->expectException(QueryException::class); - $this->expectExceptionMessage(self::VIRTUAL.'peer.orders'); - - $this->database->find('customers', [ - Query::join('customers', 'peer', [Query::on('$id', '$id')]), - Query::join('orders', 'sub', [Query::on('peer.orders', '$id')]), - ]); - } - - public function testOneToManyWithoutATwoWayKeyIsNotRewritten(): void - { - $this->createCollection('ledgers', [Attribute::relationship('orders', Relationship::oneToMany(relatedCollection: 'orders'), RelationshipSide::Parent)]); - - $this->expectException(QueryException::class); - $this->expectExceptionMessage(self::VIRTUAL.'orders'); - - $this->database->find('ledgers', [Query::join('orders', 'ord', [Query::on('orders', '$id')])]); - } - - public function testRelationshipDeclaredInTheCollectionDefinitionIsRewritten(): void - { - $this->createCollection('areas', [ - Attribute::string(key: 'name', size: 16), - Attribute::relationship('orders', Relationship::oneToMany(relatedCollection: 'orders', twoWayKey: 'region'), RelationshipSide::Parent), - ]); - $this->createDocument('areas', 'r1', ['name' => 'north']); - $this->createDocument('areas', 'r2', ['name' => 'south']); - - $this->assertSame( - ['r1/o1', 'r1/o2', 'r1/o3', 'r2/o4'], - $this->pairs('areas', [Query::join('orders', 'ord', [Query::on('orders', '$id')])], 'ord'), - ); - } - - public function testJoinOnAManyToOneRelationshipKeepsItsColumns(): void - { - $this->assertSame( - ['c1/r1', 'c2/r2', 'c3/r1'], - $this->pairs('customers', [Query::join('regions', 'reg', [Query::on('region', '$id')])], 'reg'), - 'the parent side of a many-to-one holds the related ID in its own column', - ); - } - - public function testJoinOnTheChildSideOfATwoWayManyToOneMatchesTheMainIdAgainstTheParentKey(): void - { - $this->assertSame( - ['r1/c1', 'r1/c3', 'r2/c2'], - $this->pairs('regions', [Query::leftJoin('customers', 'resident', [Query::on('customers', '$id')])], 'resident'), - ); - } - - public function testJoinOnTheChildSideOfAOneWayManyToOneIsRefused(): void - { - $this->createCollection('warehouses', [Attribute::string(key: 'name', size: 16)]); - $this->database->createRelationship('orders', Relationship::manyToOne(relatedCollection: 'warehouses', key: 'warehouse', twoWayKey: 'stock')); - - $this->expectException(QueryException::class); - $this->expectExceptionMessage(self::VIRTUAL.'stock'); - - $this->database->find('warehouses', [Query::join('orders', 'ord', [Query::on('stock', '$id')])]); - } - - public function testJoinOfAnotherCollectionThanTheRelatedOneIsRefused(): void - { - $this->expectException(QueryException::class); - $this->expectExceptionMessage(self::VIRTUAL.'orders'); - - $this->database->find('customers', [Query::join('regions', 'reg', [Query::on('orders', '$id')])]); - } - - public function testRelationshipComparedWithAnotherColumnThanTheRelatedIdIsRefused(): void - { - $this->expectException(QueryException::class); - $this->expectExceptionMessage(self::VIRTUAL.'orders'); - - $this->database->find('customers', [Query::join('orders', 'ord', [Query::on('orders', 'status')])]); - } - - public function testJoinOnAManyToManyRelationshipIsRefused(): void - { - $this->createCollection('tags', [Attribute::string(key: 'name', size: 16)]); - $this->database->createRelationship('customers', Relationship::manyToMany(relatedCollection: 'tags', key: 'tags', twoWay: true, twoWayKey: 'customers')); - - $this->expectException(QueryException::class); - $this->expectExceptionMessage(self::VIRTUAL.'tags'); - - $this->database->find('customers', [Query::join('tags', 'tag', [Query::on('tags', '$id')])]); - } - - public function testJoinOnTheChildSideOfAOneWayOneToOneIsRefused(): void - { - $this->createCollection('profiles', [Attribute::string(key: 'name', size: 16)]); - $this->database->createRelationship('customers', Relationship::oneToOne(relatedCollection: 'profiles', key: 'profile', twoWayKey: 'owner')); - - $this->expectException(QueryException::class); - $this->expectExceptionMessage(self::VIRTUAL.'owner'); - - $this->database->find('profiles', [Query::join('customers', 'customer', [Query::on('owner', '$id')])]); - } - - /** - * @return iterable - */ - public static function reads(): iterable - { - $join = Query::join('orders', 'ord', [Query::on('orders', '$id')]); - - yield 'count' => [static fn (Database $database): int => $database->count('customers', [$join]), 4]; - yield 'sum' => [static fn (Database $database): int => (int) $database->sum('customers', 'ord.amount', [$join]), 100]; - yield 'aggregate' => [ - static fn (Database $database): array => \array_map( - static fn (array $row): array => [$row['name'], self::integer($row['total'])], - $database->aggregate('customers', [$join, Query::sum('ord.amount', 'total'), Query::groupBy(['name']), Query::orderAsc('name')]), - ), - [['Ada', 30], ['Bob', 70]], - ]; - yield 'getDocument' => [ - static fn (Database $database): mixed => $database->getDocument('customers', 'c1', [ - Query::join('orders', 'ord', [Query::on('orders', '$id'), Query::equal('ord.status', ['open'])]), - Query::select(['name', 'ord.amount']), - ])->getAttribute('ord.amount'), - 20, - ]; - } - - /** - * @param Closure(Database): mixed $read - */ - #[DataProvider('reads')] - public function testEveryReadOfARelationshipJoinMatchesTheRelatedKey(Closure $read, mixed $expected): void - { - $this->assertSame($expected, $read($this->database)); - } - - public function testRelationshipJoinIsRewrittenWithoutValidation(): void - { - $pairs = $this->database->skipValidation(fn (): array => $this->pairs('customers', [Query::join('orders', 'ord', [Query::on('orders', '$id')])], 'ord')); - - $this->assertSame(['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], $pairs); - } - - public function testCallersJoinIsLeftAsGivenForTheNextRead(): void - { - $condition = Query::on('orders', '$id'); - $join = Query::join('orders', 'ord', [$condition]); - - $this->assertSame(4, $this->database->count('customers', [$join])); - $this->assertSame(['orders', '=', '$id'], $condition->getValues()); - $this->assertSame($condition, $join->getJoinOnQueries()[0]); - $this->assertSame(['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], $this->pairs('customers', [$join], 'ord')); - } - - /** - * Each row as `
/`, sorted, with `-` for a row the outer join did not match. - * - * @param list $queries - * @return list - */ - private function pairs(string $collection, array $queries, string $alias): array - { - $pairs = \array_map( - static function (Document $row) use ($alias): string { - $joined = $row->getAttribute($alias.'.$id'); - - return $row->getId().'/'.(\is_string($joined) ? $joined : '-'); - }, - $this->database->find($collection, [...$queries, Query::limit(100)]), - ); - \sort($pairs); - - return $pairs; - } - - /** - * @param list $attributes - */ - private function createCollection(string $id, array $attributes): void - { - $this->database->createCollection(Collection::create( - id: $id, - attributes: $attributes, - permissions: [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], - )); - } - - /** - * @param array $attributes - */ - private function createDocument(string $collection, string $id, array $attributes): void - { - $this->database->createDocument($collection, new Document([ - '$id' => $id, - '$permissions' => [Permission::read(Role::any())], - ...$attributes, - ])); - } - - private static function integer(mixed $value): ?int - { - return \is_numeric($value) ? (int) $value : null; - } -} From 82dafb34c3bccf635d720076452a10e1778541cf Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 18:56:34 +1300 Subject: [PATCH 37/55] refactor(documents): keep the join helpers where 8.0.3 has them Moving the join helpers into a Joins trait was a pure refactor the 8.1.0 brief did not ask for; without the relationship join rewrite nothing needs them apart. Back at their 8.0.3 place and text, the documents trait shows no change in them against 8.0.3, and Trait\Cursors keeps calling them on the database. Co-Authored-By: Claude Opus 5.5 --- src/Database/Database.php | 1 - src/Database/Trait/Documents.php | 349 +++++++++++++++++++++++++++++ src/Database/Trait/Joins.php | 367 ------------------------------- 3 files changed, 349 insertions(+), 368 deletions(-) delete mode 100644 src/Database/Trait/Joins.php diff --git a/src/Database/Database.php b/src/Database/Database.php index 63258b4a9..44da534b3 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -58,7 +58,6 @@ class Database use Trait\Documents; use Trait\Fractions; use Trait\Indexes; - use Trait\Joins; use Trait\Relationships; use Trait\Transactions; diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 633c2bb08..41e97dd9d 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -55,6 +55,7 @@ use Utopia\Database\Validator\Queries\Documents as DocumentsValidator; use Utopia\Database\Validator\Queries\Narrow; use Utopia\Database\Validator\Query\Aggregate; +use Utopia\Database\Validator\Query\Join as JoinValidator; use Utopia\Database\Validator\Query\Joined\Collection as JoinedCollection; use Utopia\Database\Validator\Structure; use Utopia\Database\Validator\UID; @@ -5037,6 +5038,354 @@ private function validateDocumentsQueries(Document $collection, array $queries, } } + /** + * The collections the join queries name, each loaded once, by the id the join names it with. A + * read resolves them once and hands them to validation, authorization, the adapter and decoding. + * + * @param array $queries + * @return array + * + * @throws QueryException + */ + private function resolveJoinedCollections(array $queries): array + { + $collections = []; + + foreach ($queries as $query) { + if (! $query->getMethod()->isJoin()) { + continue; + } + + $id = $query->getAttribute(); + if ($id === '' || isset($collections[$id])) { + continue; + } + + $collection = $this->silent(fn () => $this->findCollection($id)); + if ($collection === null) { + throw new QueryException("Joined collection '{$id}' not found"); + } + + $collections[$id] = $collection; + } + + return $collections; + } + + /** + * @param array $queries + * @param list $relationships + * @param array|null $joinedCollections The collection each join names, by its id + * @return array{0: Document, 1: array, 2: bool}|null + */ + private function prepareFilterJoinQueries( + Document $collection, + array $queries, + array $relationships, + bool $collectionGranted, + ?array $joinedCollections = null, + ): ?array { + $parsed = Query::groupByType($queries); + $filters = $parsed->filters; + $joins = $parsed->joins; + + if (! empty($joins)) { + if (! $this->adapter->supports(Capability::Joins)) { + throw new QueryException('Join queries are not supported by this adapter'); + } + + $this->assertJoinCount($joins); + + $joinedCollections ??= $this->resolveJoinedCollections($joins); + $collection = $this->withJoinIndexed($this->withJoinAuthorization( + $collection, + $this->authorizeJoins($joins, PermissionType::Read, $joinedCollections), + $collectionGranted, + ), $joins, $joinedCollections); + } + + $queries = \array_merge($filters, $joins); + if ($queries !== []) { + $queries = $this->convertQueries($collection, $queries, $this->joinedCollectionsByAlias($joins, $joinedCollections)); + } + + $convertedQueries = $this->relationshipHook !== null + ? $this->relationshipHook->convertQueries($relationships, $queries, $collection) + : $queries; + + if ($convertedQueries === null) { + return null; + } + + return [$collection, $convertedQueries, $collectionGranted && empty($joins)]; + } + + /** + * The join cap the Join validator enforces holds without validation too: every join is one + * more table the engine plans and the permission filters check. + * + * @param array $joins + * + * @throws QueryException + */ + private function assertJoinCount(array $joins): void + { + $validator = new JoinValidator(); + if (! $validator->isValidCount(\count($joins))) { + throw new QueryException($validator->getDescription()); + } + } + + /** + * Set on the collection handed to the adapter for a join read when the caller + * holds the collection-level permission, so its rows are not filtered per document. + */ + public const string COLLECTION_GRANTED = 'collectionGranted'; + + /** + * Maps each joined table to whether the adapter filters its rows per document. + */ + public const string JOIN_DOCUMENT_SECURITY = 'joinDocumentSecurity'; + + /** + * Each joined collection is authorized once, however many joins read it. + * + * @param array $joins + * @param array|null $joinedCollections The collection each join names, by its id + * @return array + */ + private function authorizeJoins(array $joins, PermissionType $forPermission, ?array $joinedCollections = null): array + { + $joinedCollections ??= $this->resolveJoinedCollections($joins); + $joinDocumentSecurity = []; + $authorized = []; + + foreach ($joins as $joinQuery) { + $joinCollectionId = $joinQuery->getAttribute(); + if (isset($authorized[$joinCollectionId])) { + continue; + } + $authorized[$joinCollectionId] = true; + + $joinCollection = $joinedCollections[$joinCollectionId] ?? null; + + if ($joinCollection === null) { + throw new QueryException("Joined collection '{$joinCollectionId}' not found"); + } + + $granted = $this->authorization->isValid(new Input($forPermission, $joinCollection->getPermissionsByType($forPermission))); + $documentSecurity = (bool) $joinCollection->getAttribute('documentSecurity', false); + + if (! $granted && ! $documentSecurity) { + throw new AuthorizationException("Unauthorized access to joined collection '{$joinCollectionId}'"); + } + + foreach ($this->joinDocumentSecurityKeys($joinCollectionId, $joinCollection) as $key) { + $joinDocumentSecurity[$key] = ! $granted; + } + } + + return $joinDocumentSecurity; + } + + /** + * @return list + */ + private function joinDocumentSecurityKeys(string $joinCollectionId, Document $joinCollection): array + { + $keys = [ + $joinCollectionId, + $joinCollection->getId(), + $this->adapter->filter($joinCollectionId), + $this->adapter->filter($joinCollection->getId()), + ]; + + return \array_values(\array_unique(\array_filter( + $keys, + static fn (string $key): bool => $key !== '', + ))); + } + + /** + * @param array $joinDocumentSecurity + */ + private function withJoinAuthorization(Document $collection, array $joinDocumentSecurity, bool $collectionGranted): Document + { + if ($joinDocumentSecurity === []) { + return $collection; + } + + $adapterCollection = clone $collection; + $adapterCollection->setAttribute(self::COLLECTION_GRANTED, $collectionGranted); + $adapterCollection->setAttribute(self::JOIN_DOCUMENT_SECURITY, $joinDocumentSecurity); + + return $adapterCollection; + } + + /** + * Maps each joined collection, as its join query names it, to the attributes a join without a + * select returns under the join's alias. + */ + public const string JOIN_ATTRIBUTES = 'joinAttributes'; + + /** + * Relationship attributes are left out: only some sides of a relationship have a column, and a + * join does not populate related documents. + * + * @param array $joins + * @param array|null $joinedCollections The collection each join names, by its id + */ + private function withJoinAttributes(Document $collection, array $joins, ?array $joinedCollections = null): Document + { + if ($joins === []) { + return $collection; + } + + $joinedCollections ??= $this->resolveJoinedCollections($joins); + + $joinAttributes = []; + foreach ($joins as $join) { + $joinCollectionId = $join->getAttribute(); + if (isset($joinAttributes[$joinCollectionId])) { + continue; + } + + $keys = []; + if (isset($joinedCollections[$joinCollectionId])) { + foreach (Collection::fromDocument($joinedCollections[$joinCollectionId])->attributes() as $attribute) { + if ($attribute->relationship === null) { + $keys[] = $attribute->key; + } + } + } + $joinAttributes[$joinCollectionId] = $keys; + } + + $adapterCollection = clone $collection; + $adapterCollection->setAttribute(self::JOIN_ATTRIBUTES, $joinAttributes); + + return $adapterCollection; + } + + /** + * Under shared tables, maps each joined collection, as its join query names it, to the attributes one of its key + * or unique indexes leads with: the ones a join can look its rows up by. + */ + public const string JOIN_INDEXED = 'joinIndexed'; + + private const array LOOKUP_INDEX_TYPES = [IndexType::Key, IndexType::Index, IndexType::Unique]; + + /** + * @param array $joins + * @param array|null $joinedCollections The collection each join names, by its id + */ + private function withJoinIndexed(Document $collection, array $joins, ?array $joinedCollections = null): Document + { + if ($joins === [] || ! $this->adapter->hasSharedTables()) { + return $collection; + } + + $joinedCollections ??= $this->resolveJoinedCollections($joins); + + $joinIndexed = []; + foreach ($joins as $join) { + $joinCollectionId = $join->getAttribute(); + if (isset($joinIndexed[$joinCollectionId]) || ! isset($joinedCollections[$joinCollectionId])) { + continue; + } + + $leads = []; + foreach (Collection::fromDocument($joinedCollections[$joinCollectionId])->indexes() as $index) { + if (\in_array($index->type, self::LOOKUP_INDEX_TYPES, true) && isset($index->attributes[0])) { + $leads[] = $index->attributes[0]; + } + } + $joinIndexed[$joinCollectionId] = \array_values(\array_unique($leads)); + } + + $adapterCollection = clone $collection; + $adapterCollection->setAttribute(self::JOIN_INDEXED, $joinIndexed); + + return $adapterCollection; + } + + /** + * The collection each join reads, by the alias its values come back under. + * + * @param array $joins + * @param array|null $joinedCollections The collection each join names, by its id + * @return array + */ + private function joinedCollectionsByAlias(array $joins, ?array $joinedCollections = null): array + { + if ($joins === []) { + return []; + } + + $joinedCollections ??= $this->resolveJoinedCollections($joins); + + $collections = []; + foreach ($joins as $join) { + $collections[$join->getAlias()] = $joinedCollections[$join->getAttribute()] ?? new Document(); + } + + return $collections; + } + + /** + * The `alias.$id` of each join whose attributes a select names without it or `alias.*`, when an + * outer join can leave a joined row unmatched: the joined `$id` is what tells an unmatched row + * from a matched one when the row is decoded, so it is selected for that and left out of the + * result. + * + * @param array $selects + * @param array $joins + * @param array $joinedCollections The collection each join alias reads + * @return list + */ + private function outerJoinIdSelections(array $selects, array $joins, array $joinedCollections): array + { + if ($selects === [] || $joinedCollections === []) { + return []; + } + + $outer = false; + foreach ($joins as $join) { + if (\in_array($join->getMethod(), [Method::LeftJoin, Method::RightJoin, Method::FullOuterJoin], true)) { + $outer = true; + break; + } + } + if (! $outer) { + return []; + } + + $selectedAliases = []; + foreach ($selects as $select) { + foreach ($select->getValues() as $value) { + if (! \is_string($value)) { + continue; + } + if ($value === '*') { + return []; + } + $dot = \strpos($value, '.'); + if ($dot !== false) { + $selectedAliases[\substr($value, 0, $dot)][\substr($value, $dot + 1)] = true; + } + } + } + + $ids = []; + foreach ($selectedAliases as $alias => $attributes) { + if (isset($joinedCollections[$alias]) && ! isset($attributes[Document::ID]) && ! isset($attributes['*'])) { + $ids[] = $alias.'.'.Document::ID; + } + } + + return $ids; + } + /** * @param array $queries * @return array diff --git a/src/Database/Trait/Joins.php b/src/Database/Trait/Joins.php deleted file mode 100644 index 4cd6e8cdf..000000000 --- a/src/Database/Trait/Joins.php +++ /dev/null @@ -1,367 +0,0 @@ - $queries - * @return array - * - * @throws QueryException - */ - private function resolveJoinedCollections(array $queries): array - { - $collections = []; - - foreach ($queries as $query) { - if (! $query->getMethod()->isJoin()) { - continue; - } - - $id = $query->getAttribute(); - if ($id === '' || isset($collections[$id])) { - continue; - } - - $collection = $this->silent(fn () => $this->findCollection($id)); - if ($collection === null) { - throw new QueryException("Joined collection '{$id}' not found"); - } - - $collections[$id] = $collection; - } - - return $collections; - } - - /** - * @param array $queries - * @param list $relationships - * @param array|null $joinedCollections The collection each join names, by its id - * @return array{0: Document, 1: array, 2: bool}|null - */ - private function prepareFilterJoinQueries( - Document $collection, - array $queries, - array $relationships, - bool $collectionGranted, - ?array $joinedCollections = null, - ): ?array { - $parsed = Query::groupByType($queries); - $filters = $parsed->filters; - $joins = $parsed->joins; - - if (! empty($joins)) { - if (! $this->adapter->supports(Capability::Joins)) { - throw new QueryException('Join queries are not supported by this adapter'); - } - - $this->assertJoinCount($joins); - - $joinedCollections ??= $this->resolveJoinedCollections($joins); - $collection = $this->withJoinIndexed($this->withJoinAuthorization( - $collection, - $this->authorizeJoins($joins, PermissionType::Read, $joinedCollections), - $collectionGranted, - ), $joins, $joinedCollections); - } - - $queries = \array_merge($filters, $joins); - if ($queries !== []) { - $queries = $this->convertQueries($collection, $queries, $this->joinedCollectionsByAlias($joins, $joinedCollections)); - } - - $convertedQueries = $this->relationshipHook !== null - ? $this->relationshipHook->convertQueries($relationships, $queries, $collection) - : $queries; - - if ($convertedQueries === null) { - return null; - } - - return [$collection, $convertedQueries, $collectionGranted && empty($joins)]; - } - - /** - * The join cap the Join validator enforces holds without validation too: every join is one - * more table the engine plans and the permission filters check. - * - * @param array $joins - * - * @throws QueryException - */ - private function assertJoinCount(array $joins): void - { - $validator = new JoinValidator(); - if (! $validator->isValidCount(\count($joins))) { - throw new QueryException($validator->getDescription()); - } - } - - /** - * Set on the collection handed to the adapter for a join read when the caller - * holds the collection-level permission, so its rows are not filtered per document. - */ - public const string COLLECTION_GRANTED = 'collectionGranted'; - - /** - * Maps each joined table to whether the adapter filters its rows per document. - */ - public const string JOIN_DOCUMENT_SECURITY = 'joinDocumentSecurity'; - - /** - * Each joined collection is authorized once, however many joins read it. - * - * @param array $joins - * @param array|null $joinedCollections The collection each join names, by its id - * @return array - */ - private function authorizeJoins(array $joins, PermissionType $forPermission, ?array $joinedCollections = null): array - { - $joinedCollections ??= $this->resolveJoinedCollections($joins); - $joinDocumentSecurity = []; - $authorized = []; - - foreach ($joins as $joinQuery) { - $joinCollectionId = $joinQuery->getAttribute(); - if (isset($authorized[$joinCollectionId])) { - continue; - } - $authorized[$joinCollectionId] = true; - - $joinCollection = $joinedCollections[$joinCollectionId] ?? null; - - if ($joinCollection === null) { - throw new QueryException("Joined collection '{$joinCollectionId}' not found"); - } - - $granted = $this->authorization->isValid(new Input($forPermission, $joinCollection->getPermissionsByType($forPermission))); - $documentSecurity = (bool) $joinCollection->getAttribute('documentSecurity', false); - - if (! $granted && ! $documentSecurity) { - throw new AuthorizationException("Unauthorized access to joined collection '{$joinCollectionId}'"); - } - - foreach ($this->joinDocumentSecurityKeys($joinCollectionId, $joinCollection) as $key) { - $joinDocumentSecurity[$key] = ! $granted; - } - } - - return $joinDocumentSecurity; - } - - /** - * @return list - */ - private function joinDocumentSecurityKeys(string $joinCollectionId, Document $joinCollection): array - { - $keys = [ - $joinCollectionId, - $joinCollection->getId(), - $this->adapter->filter($joinCollectionId), - $this->adapter->filter($joinCollection->getId()), - ]; - - return \array_values(\array_unique(\array_filter( - $keys, - static fn (string $key): bool => $key !== '', - ))); - } - - /** - * @param array $joinDocumentSecurity - */ - private function withJoinAuthorization(Document $collection, array $joinDocumentSecurity, bool $collectionGranted): Document - { - if ($joinDocumentSecurity === []) { - return $collection; - } - - $adapterCollection = clone $collection; - $adapterCollection->setAttribute(self::COLLECTION_GRANTED, $collectionGranted); - $adapterCollection->setAttribute(self::JOIN_DOCUMENT_SECURITY, $joinDocumentSecurity); - - return $adapterCollection; - } - - /** - * Maps each joined collection, as its join query names it, to the attributes a join without a - * select returns under the join's alias. - */ - public const string JOIN_ATTRIBUTES = 'joinAttributes'; - - /** - * Relationship attributes are left out: only some sides of a relationship have a column, and a - * join does not populate related documents. - * - * @param array $joins - * @param array|null $joinedCollections The collection each join names, by its id - */ - private function withJoinAttributes(Document $collection, array $joins, ?array $joinedCollections = null): Document - { - if ($joins === []) { - return $collection; - } - - $joinedCollections ??= $this->resolveJoinedCollections($joins); - - $joinAttributes = []; - foreach ($joins as $join) { - $joinCollectionId = $join->getAttribute(); - if (isset($joinAttributes[$joinCollectionId])) { - continue; - } - - $keys = []; - if (isset($joinedCollections[$joinCollectionId])) { - foreach (Collection::fromDocument($joinedCollections[$joinCollectionId])->attributes() as $attribute) { - if ($attribute->relationship === null) { - $keys[] = $attribute->key; - } - } - } - $joinAttributes[$joinCollectionId] = $keys; - } - - $adapterCollection = clone $collection; - $adapterCollection->setAttribute(self::JOIN_ATTRIBUTES, $joinAttributes); - - return $adapterCollection; - } - - /** - * Under shared tables, maps each joined collection, as its join query names it, to the attributes one of its key - * or unique indexes leads with: the ones a join can look its rows up by. - */ - public const string JOIN_INDEXED = 'joinIndexed'; - - private const array LOOKUP_INDEX_TYPES = [IndexType::Key, IndexType::Index, IndexType::Unique]; - - /** - * @param array $joins - * @param array|null $joinedCollections The collection each join names, by its id - */ - private function withJoinIndexed(Document $collection, array $joins, ?array $joinedCollections = null): Document - { - if ($joins === [] || ! $this->adapter->hasSharedTables()) { - return $collection; - } - - $joinedCollections ??= $this->resolveJoinedCollections($joins); - - $joinIndexed = []; - foreach ($joins as $join) { - $joinCollectionId = $join->getAttribute(); - if (isset($joinIndexed[$joinCollectionId]) || ! isset($joinedCollections[$joinCollectionId])) { - continue; - } - - $leads = []; - foreach (Collection::fromDocument($joinedCollections[$joinCollectionId])->indexes() as $index) { - if (\in_array($index->type, self::LOOKUP_INDEX_TYPES, true) && isset($index->attributes[0])) { - $leads[] = $index->attributes[0]; - } - } - $joinIndexed[$joinCollectionId] = \array_values(\array_unique($leads)); - } - - $adapterCollection = clone $collection; - $adapterCollection->setAttribute(self::JOIN_INDEXED, $joinIndexed); - - return $adapterCollection; - } - - /** - * The collection each join reads, by the alias its values come back under. - * - * @param array $joins - * @param array|null $joinedCollections The collection each join names, by its id - * @return array - */ - private function joinedCollectionsByAlias(array $joins, ?array $joinedCollections = null): array - { - if ($joins === []) { - return []; - } - - $joinedCollections ??= $this->resolveJoinedCollections($joins); - - $collections = []; - foreach ($joins as $join) { - $collections[$join->getAlias()] = $joinedCollections[$join->getAttribute()] ?? new Document(); - } - - return $collections; - } - - /** - * The `alias.$id` of each join whose attributes a select names without it or `alias.*`, when an - * outer join can leave a joined row unmatched: the joined `$id` is what tells an unmatched row - * from a matched one when the row is decoded, so it is selected for that and left out of the - * result. - * - * @param array $selects - * @param array $joins - * @param array $joinedCollections The collection each join alias reads - * @return list - */ - private function outerJoinIdSelections(array $selects, array $joins, array $joinedCollections): array - { - if ($selects === [] || $joinedCollections === []) { - return []; - } - - $outer = false; - foreach ($joins as $join) { - if (\in_array($join->getMethod(), [Method::LeftJoin, Method::RightJoin, Method::FullOuterJoin], true)) { - $outer = true; - break; - } - } - if (! $outer) { - return []; - } - - $selectedAliases = []; - foreach ($selects as $select) { - foreach ($select->getValues() as $value) { - if (! \is_string($value)) { - continue; - } - if ($value === '*') { - return []; - } - $dot = \strpos($value, '.'); - if ($dot !== false) { - $selectedAliases[\substr($value, 0, $dot)][\substr($value, $dot + 1)] = true; - } - } - } - - $ids = []; - foreach ($selectedAliases as $alias => $attributes) { - if (isset($joinedCollections[$alias]) && ! isset($attributes[Document::ID]) && ! isset($attributes['*'])) { - $ids[] = $alias.'.'.Document::ID; - } - } - - return $ids; - } -} From d3ac1cce906611af2f45a7b6cc358714a3a35b15 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 18:56:42 +1300 Subject: [PATCH 38/55] refactor(documents): keep the 8.0.3 aggregation checks in fetch() and nextPageCheck() ParsedQuery::isAggregation() exists for read(); rewriting the existing checks in fetch() and nextPageCheck() onto it changed 8.0.3 code the 8.1.0 brief did not ask to touch. Co-Authored-By: Claude Opus 5.5 --- src/Database/Trait/Documents.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 41e97dd9d..932e519d7 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -4020,7 +4020,7 @@ private function fetch(Collection $collection, array $queries, PermissionType $f $cursor = $parsed->cursor; $cursorDirection = $parsed->cursorDirection ?? CursorDirection::After; - $isAggregation = $parsed->isAggregation(); + $isAggregation = ! empty($aggregations) || ! empty($groupByAttrs); if ($isAggregation !== $aggregate) { throw new QueryException($aggregate @@ -4850,7 +4850,7 @@ private function nextPageCheck(string $collection, array $queries): Closure $parsed = Query::groupByType($queries); $joins = $parsed->joins; $distinct = $parsed->distinct; - if (($joins === [] && ! $distinct) || $parsed->isAggregation()) { + if (($joins === [] && ! $distinct) || $parsed->aggregations !== [] || $parsed->groupBy !== []) { return static function (Document $cursor): void { }; } From 6d85c8dccb6872ee2d4dceae37a6af436a2bdc93 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 18:56:49 +1300 Subject: [PATCH 39/55] refactor(documents): keep the 8.0.3 counter path while fractions are not rounded increaseDocumentAttribute() and decreaseDocumentAttribute() renamed $currentVal and $currentDecVal and sent the integer checks of the default path through prepareIntegerChange(). Only the round-fractions mode needs new code: with it off the counters now read exactly as in 8.0.3, and prepareIntegerChange() serves the rounding mode alone. Co-Authored-By: Claude Opus 5.5 --- src/Database/Trait/Documents.php | 42 +++++++++++++++++++++----------- src/Database/Trait/Fractions.php | 13 ++++------ 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 932e519d7..ae14b8ddf 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -2670,7 +2670,14 @@ public function increaseDocumentAttribute( $exact = false; if ($this->isDeclaredInteger($numericAttribute ?? $this->declaredAttribute($collection, $attribute))) { - [$exact, $max] = $this->prepareIntegerChange($value, $max, true); + if ($this->fractionRounding()->get()) { + [$exact, $max] = $this->prepareIntegerChange($value, $max, true); + } else { + $this->assertIntegerChange($value); + if ($max !== null) { + $max = $this->integerBound($max, 'Max'); + } + } } $cacheTarget = $collection->getId() === self::METADATA @@ -2696,22 +2703,22 @@ public function increaseDocumentAttribute( } $attributeExists = $document->offsetExists($attribute); - $current = $document->getAttribute($attribute); + $currentVal = $document->getAttribute($attribute); $returned = null; if ($exact) { - [$value, $returned] = $this->exactChange($current, $value, $max, true); + [$value, $returned] = $this->exactChange($currentVal, $value, $max, true); $max = null; } if ($numericAttribute instanceof Attribute) { - $result = $this->getNumericResult($numericAttribute, $current, $value, true); + $result = $this->getNumericResult($numericAttribute, $currentVal, $value, true); } else { if (! $attributeExists) { - $current = 0; + $currentVal = 0; } - if (! \is_int($current) && ! \is_float($current)) { + if (! \is_int($currentVal) && ! \is_float($currentVal)) { throw new TypeException('Attribute value must be numeric.'); } - $result = $current + $this->getNativeNumber($value); + $result = $currentVal + $this->getNativeNumber($value); } $exceedsMaximum = ! \is_null($max) && ( $numericAttribute?->isInteger() === true @@ -2797,7 +2804,14 @@ public function decreaseDocumentAttribute( $exact = false; if ($this->isDeclaredInteger($numericAttribute ?? $this->declaredAttribute($collection, $attribute))) { - [$exact, $min] = $this->prepareIntegerChange($value, $min, false); + if ($this->fractionRounding()->get()) { + [$exact, $min] = $this->prepareIntegerChange($value, $min, false); + } else { + $this->assertIntegerChange($value); + if ($min !== null) { + $min = $this->integerBound($min, 'Min'); + } + } } $cacheTarget = $collection->getId() === self::METADATA @@ -2823,22 +2837,22 @@ public function decreaseDocumentAttribute( } $attributeExists = $document->offsetExists($attribute); - $current = $document->getAttribute($attribute); + $currentDecVal = $document->getAttribute($attribute); $returned = null; if ($exact) { - [$value, $returned] = $this->exactChange($current, $value, $min, false); + [$value, $returned] = $this->exactChange($currentDecVal, $value, $min, false); $min = null; } if ($numericAttribute instanceof Attribute) { - $result = $this->getNumericResult($numericAttribute, $current, $value, false); + $result = $this->getNumericResult($numericAttribute, $currentDecVal, $value, false); } else { if (! $attributeExists) { - $current = 0; + $currentDecVal = 0; } - if (! \is_int($current) && ! \is_float($current)) { + if (! \is_int($currentDecVal) && ! \is_float($currentDecVal)) { throw new TypeException('Attribute value must be numeric.'); } - $result = $current - $this->getNativeNumber($value); + $result = $currentDecVal - $this->getNativeNumber($value); } $belowMinimum = ! \is_null($min) && ( $numericAttribute?->isInteger() === true diff --git a/src/Database/Trait/Fractions.php b/src/Database/Trait/Fractions.php index 5a3427a87..577a89213 100644 --- a/src/Database/Trait/Fractions.php +++ b/src/Database/Trait/Fractions.php @@ -29,8 +29,7 @@ trait Fractions */ private function prepareIntegerChange(int|float|string $value, int|float|string|null $bound, bool $increase): array { - $round = $this->fractionRounding()->get(); - if ($round && $this->isExactChange($value, $bound)) { + if ($this->isExactChange($value, $bound)) { if ($bound !== null) { $this->assertNumericBound($bound, $increase ? 'Max' : 'Min'); } @@ -40,7 +39,7 @@ private function prepareIntegerChange(int|float|string $value, int|float|string| $this->assertIntegerChange($value); - return [false, $bound === null ? null : $this->integerLimit($bound, $increase, $round)]; + return [false, $bound === null ? null : $this->integerLimit($bound, $increase)]; } /** @@ -66,13 +65,11 @@ private function isExactChange(int|float|string $value, int|float|string|null $b /** * @throws TypeException */ - private function integerLimit(int|float|string $bound, bool $maximum, bool $round): int|string + private function integerLimit(int|float|string $bound, bool $maximum): int|string { - if ($round) { - $bound = ($maximum ? Fractional::maximum($bound) : Fractional::minimum($bound)) ?? $bound; - } + $whole = ($maximum ? Fractional::maximum($bound) : Fractional::minimum($bound)) ?? $bound; - return $this->integerBound($bound, $maximum ? 'Max' : 'Min'); + return $this->integerBound($whole, $maximum ? 'Max' : 'Min'); } /** From 779040ab3ccefafe985a79fcb77266b93b8a6e18 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 18:56:50 +1300 Subject: [PATCH 40/55] refactor(validator): read operator limit bounds where 8.0.3 reads them isValidLimit() read the attribute's bounds before every limit check to serve the fractional branch. The default path reads them at its 8.0.3 place again, and the fractional branch reads them only for its own message. The fractional check no longer narrows the limit when it is false, since a limit that is not a fraction can still be a float. Co-Authored-By: Claude Opus 5.5 --- src/Database/Validator/Operator.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Database/Validator/Operator.php b/src/Database/Validator/Operator.php index 47a8b8aa6..6947d2e14 100644 --- a/src/Database/Validator/Operator.php +++ b/src/Database/Validator/Operator.php @@ -96,7 +96,7 @@ private function isNumericValueInBounds(mixed $value, Attribute $attribute): boo } /** - * @phpstan-assert-if-true float|numeric-string $value + * @phpstan-assert-if-true =float|numeric-string $value */ private function isFraction(mixed $value, Attribute $attribute): bool { @@ -116,14 +116,14 @@ private function isFractionInBounds(int|float|string $value, Attribute $attribut private function isValidLimit(mixed $limit, Attribute $attribute, DatabaseOperator $operator): bool { $methodName = $operator->getMethod()->value; - $bounds = $attribute->bounds(); - if ($bounds !== null && $this->isFraction($limit, $attribute)) { + if ($this->isFraction($limit, $attribute)) { if ($this->isFractionInBounds($limit, $attribute)) { return true; } - $this->message = "Cannot apply {$methodName} operator: max/min limit must be between {$bounds->min} and {$bounds->max}"; + $bounds = $attribute->bounds(); + $this->message = "Cannot apply {$methodName} operator: max/min limit must be between {$bounds?->min} and {$bounds?->max}"; return false; } @@ -148,6 +148,7 @@ private function isValidLimit(mixed $limit, Attribute $attribute, DatabaseOperat return false; } + $bounds = $attribute->bounds(); if ($bounds === null) { $this->message = "Cannot apply {$methodName} operator: max/min limit must be numeric, got ".\gettype($limit); From 79384252c89b3dfc856f2e5f90921f8e20868bdd Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 19:06:38 +1300 Subject: [PATCH 41/55] feat(hook): replace unique() placeholders with a nested hook 'unique()' placeholders are an appwrite feature, not a library one, so the library goes back to treating 'unique()' exactly as 8.0.3 did: a literal ID at every depth. Id.php and both ID generation sites are restored to 8.0.3, and setReplacePlaceholders()/withReplacePlaceholders()/isReplacingPlaceholders(), the snapshot field and the Mirror overrides are gone. What appwrite needs instead is a way to rewrite nested related documents before the write relates them. None of the existing hooks can: decorators run after the write and are silenced for nested writes, lifecycle hooks return void, write hooks run per row after IDs are fixed and never in the pre-checks. Hook\Nested plugs into the one walk that already reaches every nested document at every depth (the nested pre-pass), so it runs before the grant check and before relating, in writes and in authorizeCreate/authorizeUpdate. Running it before the grant check means a rewritten ID that is not stored is judged new and needs create. A Mirror registers it on itself, for its pre-checks, and on the source only: the destination replicates the documents the source already prepared, so preparing them again would give it IDs the source does not have. The bulk update refusal stays, gated on the grant check alone: updateDocuments() relates nested documents without the pre-pass, so with grants checked an unheld role could otherwise be granted there. Nested hooks never run in it, because one rewritten document would serve every matched row. Co-Authored-By: Claude Opus 5.5 --- src/Database/Database.php | 62 +++++------------------ src/Database/Hook/Nested.php | 39 +++++++++++++++ src/Database/Hook/Preparation.php | 4 +- src/Database/Hook/Relationships.php | 19 +++----- src/Database/Id.php | 10 ---- src/Database/Mirror.php | 30 +++++------- src/Database/State/Snapshot.php | 2 - src/Database/Trait/Documents.php | 35 ++++++------- tests/unit/MirrorReplicationTest.php | 65 +++++++++++++++++++++++++ tests/unit/State/CoroutineStateTest.php | 1 - 10 files changed, 154 insertions(+), 113 deletions(-) create mode 100644 src/Database/Hook/Nested.php diff --git a/src/Database/Database.php b/src/Database/Database.php index 63258b4a9..3ff76f34a 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -236,6 +236,11 @@ class Database */ protected array $decorators = []; + /** + * @var list + */ + protected array $nestedHooks = []; + /** @var Value|null Whether every lifecycle hook is silenced. */ private ?Value $silenced = null; @@ -269,9 +274,6 @@ class Database /** @var Value|null */ private ?Value $ignoringDuplicates = null; - /** @var Value|null */ - private ?Value $placeholderReplacement = null; - /** @var Value|null */ private ?Value $grantChecking = null; @@ -1162,41 +1164,6 @@ public function withPreserveSequence(bool $preserve, callable $callback): mixed return $this->sequencePreservation()->with($preserve, $callback); } - /** - * Set whether a document whose ID is Id::PLACEHOLDER gets a generated ID, as one whose ID is empty does. This - * covers the documents given to createDocument(), createDocuments(), upsertDocument() and upsertDocuments(), and - * every related document they or updateDocument() are given as a document, as deep as the write relates them. A - * related document given as the string Id::PLACEHOLDER stays a reference to the document stored under that ID. - * While it is on, updateDocuments() refuses a related document given as a document with a relationship - * exception: one generated ID could not serve every document it updates. - * - * @return $this - */ - public function setReplacePlaceholders(bool $replace): static - { - $this->placeholderReplacement()->set($replace); - - return $this; - } - - public function isReplacingPlaceholders(): bool - { - return $this->placeholderReplacement()->get(); - } - - /** - * Run the callback with placeholder replacement on or off for the calling coroutine and the coroutines it starts. - * - * @template T - * - * @param callable(): T $callback - * @return T - */ - public function withReplacePlaceholders(bool $replace, callable $callback): mixed - { - return $this->placeholderReplacement()->with($replace, $callback); - } - /** * Set whether, while authorization is enabled, a related document given as a document to createDocument(), * createDocuments(), updateDocument(), upsertDocument(), upsertDocuments(), authorizeCreate() or authorizeUpdate() @@ -1445,6 +1412,7 @@ public function skipValidation(callable $callback): mixed * {@see Named} one replaces the lifecycle hook registered under its name * - {@see Hook\Decorator} — document transformation on read/write results * - {@see Hook\Relationships} — relationship resolution and mutation + * - {@see Hook\Nested} — preparation of each nested related document before a write relates it, run in registration order * - {@see Hook\Write} — row-level write interception (permissions, tenant) * - {@see Hook\Transform} — raw SQL transformation before execution * - {@see Invalidator} — the query cache invalidation, replacing the one setQueryCache() made @@ -1460,6 +1428,7 @@ public function addHook(\Utopia\Query\Hook $hook): static && ! $hook instanceof Invalidator && ! $hook instanceof Hook\Decorator && ! $hook instanceof Relationships + && ! $hook instanceof Hook\Nested && ! $hook instanceof Hook\Write && ! $hook instanceof Transform ) { @@ -1486,6 +1455,10 @@ public function addHook(\Utopia\Query\Hook $hook): static $this->relationshipHook = $hook; } + if ($hook instanceof Hook\Nested) { + $this->nestedHooks[] = $hook; + } + if ($hook instanceof Hook\Write) { $this->adapter->addWriteHook($hook); } @@ -1514,6 +1487,7 @@ public function removeHook(\Utopia\Query\Hook|string $hook): static $this->lifecycleHooks = \array_values(\array_filter($this->lifecycleHooks, $kept)); $this->decorators = \array_values(\array_filter($this->decorators, $kept)); + $this->nestedHooks = \array_values(\array_filter($this->nestedHooks, $kept)); if ($this->invalidator !== null && ! $kept($this->invalidator)) { $this->invalidator = null; @@ -1680,7 +1654,6 @@ public function snapshot(): Snapshot preserveSequence: $this->sequencePreservation()->get(), ignoreDuplicates: $this->ignoringDuplicates()->get(), requestTimestamp: $this->requestTimestamp()->get(), - replacePlaceholders: $this->placeholderReplacement()->get(), checkGrants: $this->grantChecking()->get(), roundFractions: $this->fractionRounding()->get(), ); @@ -1725,8 +1698,7 @@ private function withToggles(Snapshot $snapshot, callable $callback): mixed { $rounded = fn (): mixed => $this->fractionRounding()->with($snapshot->roundFractions, $callback); $checked = fn (): mixed => $this->grantChecking()->with($snapshot->checkGrants, $rounded); - $replaced = fn (): mixed => $this->placeholderReplacement()->with($snapshot->replacePlaceholders, $checked); - $timestamped = fn (): mixed => $this->requestTimestamp()->with($snapshot->requestTimestamp, $replaced); + $timestamped = fn (): mixed => $this->requestTimestamp()->with($snapshot->requestTimestamp, $checked); $deduplicated = fn (): mixed => $this->ignoringDuplicates()->with($snapshot->ignoreDuplicates, $timestamped); $sequenced = fn (): mixed => $this->sequencePreservation()->with($snapshot->preserveSequence, $deduplicated); $dated = fn (): mixed => $this->datePreservation()->with($snapshot->preserveDates, $sequenced); @@ -1805,14 +1777,6 @@ private function ignoringDuplicates(): Value return $this->ignoringDuplicates ??= new Value(false); } - /** - * @return Value - */ - private function placeholderReplacement(): Value - { - return $this->placeholderReplacement ??= new Value(false); - } - /** * @return Value */ diff --git a/src/Database/Hook/Nested.php b/src/Database/Hook/Nested.php new file mode 100644 index 000000000..27a3bdf41 --- /dev/null +++ b/src/Database/Hook/Nested.php @@ -0,0 +1,39 @@ + $hooks The hooks each related document is given to, in order, before it is checked * @param bool $check Whether a related document may only add permissions for roles the caller holds * @param bool $lock Whether stored permissions are read locked, holding them until the write's transaction ends * @param (Closure(string, string): ?Document)|null $resolve The staged view of a related document by collection and ID; null reads the stored one */ public function __construct( - public bool $replace, + public array $hooks, public bool $check, public bool $lock = false, public ?Closure $resolve = null, diff --git a/src/Database/Hook/Relationships.php b/src/Database/Hook/Relationships.php index c36bdc955..2fe24abe0 100644 --- a/src/Database/Hook/Relationships.php +++ b/src/Database/Hook/Relationships.php @@ -20,7 +20,6 @@ use Utopia\Database\Exception\Relationship as RelationshipException; use Utopia\Database\Exception\Restricted as RestrictedException; use Utopia\Database\Exception\Structure as StructureException; -use Utopia\Database\Id; use Utopia\Database\Operator; use Utopia\Database\OperatorType; use Utopia\Database\Permission; @@ -3169,10 +3168,11 @@ private function authorizeLink(Document $collection, Document $related): void /** * Prepare the related documents $document holds as documents, as deep as the write relates them, replacing its - * relationship values with prepared copies so the caller's related documents are not changed. A related document - * that does not exist yet has no permissions to keep, so all of its permissions are checked; any other keeps - * those of its staged view, else of the stored one. A write nested in another relationship write is left as - * given: the outermost write prepared it. + * relationship values with prepared copies so the caller's related documents are not changed. Each copy is first + * given to the {@see Nested} hooks, in order, parent before children, and the document they return is the one + * whose grants are checked and that is related. A related document that does not exist yet has no permissions + * to keep, so all of its permissions are checked; any other keeps those of its staged view, else of the stored + * one. A write nested in another relationship write is left as given: the outermost write prepared it. * * @throws AuthorizationException When a related document adds a permission for a role the caller does not hold * @throws RelationshipException When a related document given as an array is not a valid document @@ -3250,15 +3250,12 @@ private function prepareNestedValues(Document $collection, Document $document, a */ private function prepareNestedDocument(Document $collection, Document $document, array $writeStack, Preparation $preparation): Document { - $id = $document->getId(); - $new = $id === ''; - if ($preparation->replace && ($new || Id::isPlaceholder($id))) { - $document->setAttribute(Document::ID, Id::unique()); - $new = true; + foreach ($preparation->hooks as $hook) { + $document = $hook->prepare($collection, $document); } if ($preparation->check) { - $this->authorizeGrants($collection, $document, $new, $preparation); + $this->authorizeGrants($collection, $document, $document->getId() === '', $preparation); } foreach ($this->prepareNestedValues($collection, $document, $writeStack, $preparation) as $key => $value) { diff --git a/src/Database/Id.php b/src/Database/Id.php index da34edd0c..ed63cecef 100644 --- a/src/Database/Id.php +++ b/src/Database/Id.php @@ -10,11 +10,6 @@ */ class Id { - /** - * The ID a caller sends to ask for a generated one, replaced only where the database replaces placeholders. - */ - public const string PLACEHOLDER = 'unique()'; - /** * Create a new unique ID using uniqid with optional random padding. * @@ -49,9 +44,4 @@ public static function custom(string $id): string { return $id; } - - public static function isPlaceholder(string $id): bool - { - return $id === self::PLACEHOLDER; - } } diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php index c8bb9ea87..62afb5ec4 100644 --- a/src/Database/Mirror.php +++ b/src/Database/Mirror.php @@ -16,6 +16,7 @@ use Utopia\Database\Exception\Limit; use Utopia\Database\Filter\Registry; use Utopia\Database\Hook\Lifecycle; +use Utopia\Database\Hook\Nested; use Utopia\Database\Hook\Relationships; use Utopia\Database\Hook\Write; use Utopia\Database\Mirror\Failure; @@ -482,14 +483,6 @@ public function setPreserveSequence(bool $preserve): static return parent::setPreserveSequence($preserve); } - #[\Override] - public function setReplacePlaceholders(bool $replace): static - { - $this->delegate(__FUNCTION__, \func_get_args()); - - return parent::setReplacePlaceholders($replace); - } - /** * Only the source checks grants: the destination replays what the source already allowed, and judging it against * the destination's own copies would refuse writes the source made whenever the destination is behind. @@ -558,15 +551,6 @@ public function withPreserveSequence(bool $preserve, callable $callback): mixed return parent::withPreserveSequence($preserve, fn (): mixed => $this->source->withPreserveSequence($preserve, $scoped)); } - #[\Override] - public function withReplacePlaceholders(bool $replace, callable $callback): mixed - { - $destination = $this->destination; - $scoped = $destination === null ? $callback : fn (): mixed => $destination->withReplacePlaceholders($replace, $callback); - - return parent::withReplacePlaceholders($replace, fn (): mixed => $this->source->withReplacePlaceholders($replace, $scoped)); - } - #[\Override] public function withCheckGrants(bool $check, callable $callback): mixed { @@ -1864,7 +1848,9 @@ public function setAuthorization(Authorization $authorization): static /** * A relationships hook is the mirror's own, and the source and the destination each attach a copy of it, so * both sides relate documents exactly as it is configured to. A write hook also intercepts the destination's - * writes. + * writes. A nested hook runs on the mirror, for its pre-checks, and on the source, but never on the destination: + * the destination replicates the related documents the source already prepared, so preparing them again would + * give it IDs the source does not have. */ #[\Override] public function addHook(\Utopia\Query\Hook $hook): static @@ -1878,6 +1864,10 @@ public function addHook(\Utopia\Query\Hook $hook): static parent::addHook($hook); } + if ($hook instanceof Nested) { + $this->source->addHook($hook); + } + if ($hook instanceof Relationships) { $this->source->addHook(clone $hook); $this->destination?->addHook(clone $hook); @@ -1902,6 +1892,10 @@ public function removeHook(\Utopia\Query\Hook|string $hook): static $this->source->removeHook($hook); } + if ($hook instanceof Nested || (\is_string($hook) && \is_a($hook, Nested::class, true))) { + $this->source->removeHook($hook); + } + if ($hook instanceof Relationships || (\is_string($hook) && \is_a($hook, Relationships::class, true))) { $this->source->removeHook(Relationships::class); $this->destination?->removeHook(Relationships::class); diff --git a/src/Database/State/Snapshot.php b/src/Database/State/Snapshot.php index 45fe2f50f..71a2fb221 100644 --- a/src/Database/State/Snapshot.php +++ b/src/Database/State/Snapshot.php @@ -29,7 +29,6 @@ * @param bool $preserveSequence Whether writes keep the sequences they are given * @param bool $ignoreDuplicates Whether creating a document that exists is skipped instead of failing * @param DateTime|null $requestTimestamp The time an update conflicts after - * @param bool $replacePlaceholders Whether a related document sent with an Id::PLACEHOLDER ID gets a generated one * @param bool $checkGrants Whether a related document may only be given permissions for roles the caller holds * @param bool $roundFractions Whether a fractional amount or limit on an integer attribute is stored as the engine rounded it */ @@ -49,7 +48,6 @@ public function __construct( public bool $preserveSequence, public bool $ignoreDuplicates, public ?DateTime $requestTimestamp, - public bool $replacePlaceholders, public bool $checkGrants, public bool $roundFractions, ) { diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 54d5e0d4d..4e4ca5421 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -1049,7 +1049,8 @@ public function applySelectFiltersToDocuments(array $documents, array $selectQue * document needs create on its collection; linking a stored one, by ID or as a document, needs update on it * (on its collection, or on the document under document security); a stored document the caller cannot read is * refused. For staging a write that runs later, such as an operation of a transaction committed by another - * request; createDocument() runs the same checks when it writes. + * request; createDocument() runs the same checks when it writes. Related documents are first prepared by the + * registered {@see \Utopia\Database\Hook\Nested} hooks, as createDocument() prepares them. * * Writes staged before the later one change what it finds. With $resolve, every related document is judged as * the later write would find it: $resolve(collection, ID) is given the collection identifier and the document ID @@ -1084,7 +1085,8 @@ public function authorizeCreate(string $collection, Document $document, ?callabl * authorizeCreate() checks for it. A related document $old already links needs no link check, only update when * it is sent as a document with attributes to change. Unlike updateDocument(), which lets an update that changes * nothing through with read permission, this always needs update. $resolve gives the staged view of related - * documents as for authorizeCreate(); $old is the document as the staged writes leave it. + * documents as for authorizeCreate(); $old is the document as the staged writes leave it. Related documents are + * first prepared by the registered {@see \Utopia\Database\Hook\Nested} hooks, as updateDocument() prepares them. * * @param string $collection The collection identifier * @param Document $old The stored document the update changes, as the caller reads it @@ -1236,9 +1238,9 @@ public function createPrepared(array $documents): void }); } - private function generatesId(string $id): bool + private function checksGrants(): bool { - return empty($id) || (Id::isPlaceholder($id) && $this->placeholderReplacement()->get()); + return $this->grantChecking()->get() && $this->authorization->getStatus(); } /** @@ -1248,13 +1250,12 @@ private function generatesId(string $id): bool */ private function nestedPreparation(?callable $resolve = null, bool $lock = false): ?Preparation { - $replace = $this->placeholderReplacement()->get(); - $check = $this->grantChecking()->get() && $this->authorization->getStatus(); - if ((! $replace && ! $check) || ! $this->relationshipHook?->isEnabled()) { + $check = $this->checksGrants(); + if (($this->nestedHooks === [] && ! $check) || ! $this->relationshipHook?->isEnabled()) { return null; } - return new Preparation($replace, $check, $lock, $resolve === null ? null : $resolve(...)); + return new Preparation($this->nestedHooks, $check, $lock, $resolve === null ? null : $resolve(...)); } /** @@ -1346,7 +1347,7 @@ private function prepareDocument(Document $collection, Document $document): Docu $id = $document->getId(); $document - ->setAttribute(Document::ID, $this->generatesId($id) ? Id::unique() : $id) + ->setAttribute(Document::ID, empty($id) ? Id::unique() : $id) ->setAttribute(Document::COLLECTION, $collection->getId()) ->setAttribute(Document::CREATED_AT, ($createdAt === null || ! $this->datePreservation()->get()) ? $time : $createdAt) ->setAttribute(Document::UPDATED_AT, ($updatedAt === null || ! $this->datePreservation()->get()) ? $time : $updatedAt); @@ -1465,7 +1466,7 @@ public function createDocuments( $updatedAt = $document->getUpdatedAt(); $document - ->setAttribute(Document::ID, $this->generatesId($document->getId()) ? Id::unique() : $document->getId()) + ->setAttribute(Document::ID, empty($document->getId()) ? Id::unique() : $document->getId()) ->setAttribute(Document::COLLECTION, $collection->getId()) ->setAttribute(Document::CREATED_AT, ($createdAt === null || ! $this->datePreservation()->get()) ? $time : $createdAt) ->setAttribute(Document::UPDATED_AT, ($updatedAt === null || ! $this->datePreservation()->get()) ? $time : $updatedAt); @@ -1905,8 +1906,8 @@ public function updateDocuments( $collection = $this->silent(fn () => $this->getCollection($collection)); - if ($this->nestedPreparation() !== null && self::holdsRelatedDocument($collection, $updates)) { - throw new RelationshipException('Invalid relationship value. A bulk update takes related documents by ID while placeholders are replaced or grants are checked.'); + if ($this->checksGrants() && $this->relationshipHook?->isEnabled() && self::holdsRelatedDocument($collection, $updates)) { + throw new RelationshipException('Invalid relationship value. A bulk update takes related documents by ID while grants are checked.'); } $documentSecurity = $collection->getAttribute('documentSecurity', false); @@ -2273,16 +2274,10 @@ private function upsert( $operatorIds = []; $seenIds = []; $hasRelationships = self::relationshipAttributes($collection) !== []; - $replacing = $this->placeholderReplacement()->get(); $preparation = $hasRelationships ? $this->nestedPreparation() : null; - if ($replacing || $preparation !== null) { + if ($preparation !== null) { foreach ($documents as $key => $document) { - if ($replacing && Id::isPlaceholder($document->getId())) { - $document->setAttribute(Document::ID, Id::unique()); - } - if ($preparation !== null) { - $documents[$key] = $this->prepareNested($collection, $document, $preparation); - } + $documents[$key] = $this->prepareNested($collection, $document, $preparation); } } $internalKeys = \array_flip(\array_map( diff --git a/tests/unit/MirrorReplicationTest.php b/tests/unit/MirrorReplicationTest.php index fdf2b1720..cd62d884b 100644 --- a/tests/unit/MirrorReplicationTest.php +++ b/tests/unit/MirrorReplicationTest.php @@ -24,7 +24,9 @@ use Utopia\Database\Document; use Utopia\Database\Event; use Utopia\Database\Exception as DatabaseException; +use Utopia\Database\Hook\Nested; use Utopia\Database\Hook\Relationships; +use Utopia\Database\Id; use Utopia\Database\Index; use Utopia\Database\Mirror; use Utopia\Database\Mirror\Failure; @@ -53,6 +55,8 @@ final class MirrorReplicationTest extends TestCase public const string INCREASED = 'increased'; + public const string MARKER = 'marker'; + private Authorization $authorization; private Mirror $mirror; @@ -433,6 +437,62 @@ public function testTheGrantCheckRunsOnTheSourceOnly(): void $this->assertSame([true, true, false], [$this->mirror->isCheckingGrants(), $source->isCheckingGrants(), $this->destination->isCheckingGrants()]); } + public function testANestedHookRunsOnTheSourceAndTheDestinationKeepsItsIds(): void + { + $this->relate(); + $hook = new class () implements Nested { + /** + * @var list + */ + public array $seen = []; + + #[\Override] + public function prepare(Document $collection, Document $document): Document + { + $this->seen[] = [$collection->getId(), $document->getId()]; + if ($document->getId() === MirrorReplicationTest::MARKER) { + $document->setAttribute(Document::ID, Id::unique()); + } + + return $document; + } + }; + $this->mirror->addHook($hook); + $child = static fn (string $title): Document => new Document([Document::ID => self::MARKER, 'title' => $title]); + $parent = static fn (string $id): Document => new Document([Document::ID => $id, 'title' => $id, 'children' => [$child($id.' child')]]); + + $this->mirror->createDocument(self::PARENTS, $parent('created')); + $this->mirror->createDocuments(self::PARENTS, [$parent('batched')]); + $this->mirror->updateDocument(self::PARENTS, 'created', new Document(['children' => [$child('updated child')]])); + $this->mirror->upsertDocument(self::PARENTS, $parent('upserted')); + $this->mirror->upsertDocuments(self::PARENTS, [$parent('upserts')]); + $this->mirror->authorizeCreate(self::PARENTS, $parent('checked')); + + $this->assertSame([], $this->errors); + $titles = ['created child', 'batched child', 'updated child', 'upserted child', 'upserts child']; + $ids = []; + foreach ($titles as $title) { + $stored = $this->childTitled($this->mirror->getSource(), $title); + $this->assertFalse($stored->isEmpty(), $title); + $this->assertNotSame(self::MARKER, $stored->getId(), $title); + $this->assertSame($stored->getId(), $this->childTitled($this->destination, $title)->getId(), $title); + $ids[] = $stored->getId(); + } + $this->assertCount(\count($titles), \array_unique($ids)); + $this->assertTrue($this->childTitled($this->mirror->getSource(), 'checked child')->isEmpty()); + foreach ([$this->mirror->getSource(), $this->destination] as $database) { + $this->assertTrue($this->authorization->skip(static fn (): Document => $database->getDocument(self::CHILDREN, self::MARKER))->isEmpty()); + } + $this->assertSame(\array_fill(0, \count($titles) + 1, [self::CHILDREN, self::MARKER]), $hook->seen); + + $this->mirror->removeHook(Nested::class); + $this->mirror->createDocument(self::PARENTS, $parent('unprepared')); + + foreach ([$this->mirror->getSource(), $this->destination] as $database) { + $this->assertSame(self::MARKER, $this->childTitled($database, 'unprepared child')->getId()); + } + } + /** * @return iterable */ @@ -732,6 +792,11 @@ private function relate(): void }); } + private function childTitled(Database $database, string $title): Document + { + return $this->authorization->skip(static fn (): Document => $database->findOne(self::CHILDREN, [Query::equal('title', [$title])])); + } + /** * @return array{mixed, mixed}|null The title and views a database stores for the note, or null when it has none */ diff --git a/tests/unit/State/CoroutineStateTest.php b/tests/unit/State/CoroutineStateTest.php index a809fdfe7..e9bdb01d5 100644 --- a/tests/unit/State/CoroutineStateTest.php +++ b/tests/unit/State/CoroutineStateTest.php @@ -258,7 +258,6 @@ private function snapshot(bool $population): Snapshot preserveSequence: false, ignoreDuplicates: false, requestTimestamp: null, - replacePlaceholders: false, checkGrants: false, roundFractions: false, ); From 99bb61d94803f10e9e245a9d44ef9eca91954cc2 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 19:06:47 +1300 Subject: [PATCH 42/55] fix(state): keep snapshot constructor compatible with 8.0.3 8.1.0 is a minor release, so a Snapshot built with the 8.0.3 arguments must keep working. The new checkGrants and roundFractions parameters now default to false, the modes' own defaults, and the @internal tag this release added is dropped so the class stays as public as it was in 8.0.3. CoroutineStateTest is back to its 8.0.3 text and passes unchanged, which is the proof. Co-Authored-By: Claude Opus 5.5 --- src/Database/State/Snapshot.php | 6 ++---- tests/unit/State/CoroutineStateTest.php | 2 -- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/Database/State/Snapshot.php b/src/Database/State/Snapshot.php index 71a2fb221..aa45b9391 100644 --- a/src/Database/State/Snapshot.php +++ b/src/Database/State/Snapshot.php @@ -8,8 +8,6 @@ * The scoped state a database handle's reads depend on, as one coroutine sees it. Taken with * Database::snapshot() and applied with Database::withSnapshot(), it lets work started in another coroutine run * under its caller's state, however long the caller keeps that state. - * - * @internal */ final readonly class Snapshot { @@ -48,8 +46,8 @@ public function __construct( public bool $preserveSequence, public bool $ignoreDuplicates, public ?DateTime $requestTimestamp, - public bool $checkGrants, - public bool $roundFractions, + public bool $checkGrants = false, + public bool $roundFractions = false, ) { } } diff --git a/tests/unit/State/CoroutineStateTest.php b/tests/unit/State/CoroutineStateTest.php index e9bdb01d5..5e89d68c8 100644 --- a/tests/unit/State/CoroutineStateTest.php +++ b/tests/unit/State/CoroutineStateTest.php @@ -258,8 +258,6 @@ private function snapshot(bool $population): Snapshot preserveSequence: false, ignoreDuplicates: false, requestTimestamp: null, - checkGrants: false, - roundFractions: false, ); } From 2d1584ce5a97f6cd72ecb054e052e2df7a5339e4 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 19:06:47 +1300 Subject: [PATCH 43/55] docs(arithmetic): mark decimal and fractional internal Decimal and Fractional implement the opt-in fractional mode; they were never approved as public API, so they are marked @internal to keep them free to change without a major release. Co-Authored-By: Claude Opus 5.5 --- src/Database/Arithmetic/Decimal.php | 2 ++ src/Database/Arithmetic/Fractional.php | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/Database/Arithmetic/Decimal.php b/src/Database/Arithmetic/Decimal.php index c0f340bad..2a90173ec 100644 --- a/src/Database/Arithmetic/Decimal.php +++ b/src/Database/Arithmetic/Decimal.php @@ -6,6 +6,8 @@ /** * A number read exactly from its text, at any size, without passing through a float. + * + * @internal */ final readonly class Decimal { diff --git a/src/Database/Arithmetic/Fractional.php b/src/Database/Arithmetic/Fractional.php index c11c4a84c..63a1c5c17 100644 --- a/src/Database/Arithmetic/Fractional.php +++ b/src/Database/Arithmetic/Fractional.php @@ -12,6 +12,8 @@ * Fractional amounts and limits on an integer column, answered as database 7.4.1 answered them: it handed them to * the engine, which stored the result rounded its own way. Each is resolved from the stored value into the * whole-number change that stores the same result. + * + * @internal */ final readonly class Fractional { From 7d8181d3f074846862abc1d6e8faa475d48d72a5 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 19:10:21 +1300 Subject: [PATCH 44/55] test(relationships): pin the nested hook in writes and pre-checks The placeholder toggle is gone: the library never interprets 'unique()', and an application that wants generated nested IDs registers a Hook\Nested. These tests pin that contract on Memory and SQLite for every relationship type, in both directions, two and three levels down: createDocument, createDocuments, updateDocument, upsertDocument(s) and authorizeCreate/authorizeUpdate give each related document sent as a document or an array to the hooks, parent before child and in registration order, as a copy; the top-level document, string references and bulk updates are never given to them; removing a hook by instance or class stops it; a hook exception writes nothing. The grant check runs on the rewritten document, so a rewritten ID that is not stored needs create and has every permission checked, and a resolver is only asked about rewritten IDs. The bulk refusal now fires only while grants are checked, with the new message. The grant tests that are kept run against their unchanged fixtures; the hook tests use separate two-way fixtures. Co-Authored-By: Claude Opus 5.5 --- .../unit/Relationships/NestedPreCheckTest.php | 246 +++++-- .../Relationships/NestedPreparationTest.php | 598 +++++++++++++----- 2 files changed, 641 insertions(+), 203 deletions(-) diff --git a/tests/unit/Relationships/NestedPreCheckTest.php b/tests/unit/Relationships/NestedPreCheckTest.php index 0efb6dee8..cb013c7b0 100644 --- a/tests/unit/Relationships/NestedPreCheckTest.php +++ b/tests/unit/Relationships/NestedPreCheckTest.php @@ -16,6 +16,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization as AuthorizationException; +use Utopia\Database\Hook\Nested; use Utopia\Database\Hook\Permissions; use Utopia\Database\Hook\Relationships; use Utopia\Database\Id; @@ -28,9 +29,9 @@ /** * Database::authorizeCreate() and authorizeUpdate() prepare the related documents they are given as the write - * would: a placeholder ID is generated, so the related document is judged new, and with grants checked a related - * document may only add permissions for roles the caller holds, its current permissions being those of the staged - * view when a resolver gives one. + * would: the registered Hook\Nested hooks run first, so a related document they give an ID that is not stored is + * judged new, and with grants checked a related document may only add permissions for roles the caller holds, its + * current permissions being those of the staged view when a resolver gives one. */ final class NestedPreCheckTest extends TestCase { @@ -44,10 +45,28 @@ final class NestedPreCheckTest extends TestCase private const string STORED = 'stored'; + private const string LEAVES = 'leaves'; + + private const string NEXT = 'next'; + + private const string BACK = 'back'; + + private const string MARKER = 'marker'; + private Authorization $authorization; private Database $database; + /** + * @var list The collection and ID of each related document the hook was given, in order + */ + private array $seen = []; + + /** + * @var list The collection and ID of each related document resolved, in order + */ + private array $resolved = []; + /** * @return iterable */ @@ -122,32 +141,83 @@ public function testADocumentStagedForDeletionIsNew(Closure $adapter, Relationsh * @param Closure(): Adapter $adapter */ #[DataProvider('types')] - public function testAPlaceholderIsJudgedNewWithItsGrantsChecked(Closure $adapter, RelationshipType $type): void + public function testANestedHookRewritesInThePreChecks(Closure $adapter, RelationshipType $type): void { - $this->seed($adapter(), $type, [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], null); - $this->database->setReplacePlaceholders(true)->setCheckGrants(true); - $placeholder = static fn (string $role): Document => new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'new', Document::PERMISSIONS => [Permission::read(Role::parse($role))]]); - $parent = $this->parent($type, $placeholder('user:u2')); - - $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $this->database->authorizeCreate('parents', $parent))); - $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $this->database->authorizeUpdate('parents', $this->storedParent(), new Document([self::RELATED => $parent->getAttribute(self::RELATED)])))); - $this->assertNull($this->refusal(fn () => $this->database->authorizeCreate('parents', $this->parent($type, $placeholder('user:'.self::USER))))); + $this->chain($adapter(), $type, [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())]); + $old = $this->storedParent(); + $this->database->addHook($this->marker()); + $resolve = $this->resolveAsStored(...); + $leaf = new Document([Document::ID => self::MARKER, 'name' => 'leaf']); + $related = new Document([Document::ID => self::MARKER, 'name' => 'related', self::NEXT => self::value($type, $leaf)]); + $parent = $this->parent($type, $related); + + $this->database->authorizeCreate('parents', $parent, $resolve); + $this->database->authorizeUpdate('parents', $old, new Document([self::RELATED => self::value($type, $related)]), $resolve); + $this->database->authorizeCreate(self::LEAVES, new Document([ + Document::ID => 'leaf', + self::BACK => self::reverse($type, new Document([ + Document::ID => self::MARKER, + 'name' => 'inverse related', + self::BACK => self::reverse($type, new Document([Document::ID => self::MARKER])), + ])), + ]), $resolve); + + $forward = [[self::RELATED, self::MARKER], [self::LEAVES, self::MARKER]]; + $this->assertSame([...$forward, ...$forward, [self::RELATED, self::MARKER], ['parents', self::MARKER]], $this->seen); + $this->assertSame([self::RELATED, self::LEAVES, self::RELATED, self::LEAVES, self::RELATED, 'parents'], \array_column($this->resolved, 0), 'Each related document is resolved once, by its rewritten ID'); + $ids = \array_column($this->resolved, 1); + $this->assertNotContains(self::MARKER, $ids); + $this->assertNotContains('', $ids); + $this->assertCount(\count($ids), \array_unique($ids)); + $this->assertSame([self::MARKER], self::ids($parent->getAttribute(self::RELATED)), 'The document checked is not changed'); + $this->assertSame(self::MARKER, $related->getId()); + $this->assertSame(self::MARKER, $leaf->getId()); $this->assertSame([], $this->relatedNames()); - $this->assertSame([Id::PLACEHOLDER], self::ids($parent->getAttribute(self::RELATED)), 'The document checked is not changed'); + $this->assertSame([], $this->leafNames()); } /** * @param Closure(): Adapter $adapter */ #[DataProvider('types')] - public function testAPlaceholderNeedsCreateWhereALiteralDocumentNeedsUpdate(Closure $adapter, RelationshipType $type): void + public function testARewrittenIdNeedsCreateWhereTheMarkedDocumentNeedsUpdate(Closure $adapter, RelationshipType $type): void { - $this->seed($adapter(), $type, [Permission::read(Role::any()), Permission::update(Role::any())], null); - $this->authorization->skip(fn () => $this->database->createDocument(self::RELATED, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'Literal']))); - $parent = $this->parent($type, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'Changed'])); + $this->chain($adapter(), $type, [Permission::read(Role::any()), Permission::update(Role::any())]); + $this->authorization->skip(fn () => $this->database->createDocument(self::RELATED, new Document([Document::ID => self::MARKER, 'name' => 'Literal']))); + $parent = $this->parent($type, new Document([Document::ID => self::MARKER, 'name' => 'Changed'])); - $this->assertNull($this->refusal(fn () => $this->database->authorizeCreate('parents', $parent)), 'The literal document is linked and changed, which needs update'); - $this->assertSame(self::NO_CREATE, $this->refusal(fn () => $this->database->withReplacePlaceholders(true, fn () => $this->database->authorizeCreate('parents', $parent))), 'A generated ID is new, which needs create'); + $this->assertNull($this->refusal(fn () => $this->database->authorizeCreate('parents', $parent)), 'The marked document is stored, so it is linked and changed, which needs update'); + $this->database->addHook($this->marker()); + $this->assertSame(self::NO_CREATE, $this->refusal(fn () => $this->database->authorizeCreate('parents', $parent)), 'A rewritten ID is new, which needs create'); + } + + /** + * @param Closure(): Adapter $adapter + */ + #[DataProvider('types')] + public function testTheGrantCheckRunsAfterTheRewrite(Closure $adapter, RelationshipType $type): void + { + $foreign = [Permission::read(Role::user('u2'))]; + $this->chain($adapter(), $type, [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())]); + $this->authorization->skip(fn () => $this->database->createDocument(self::RELATED, new Document([Document::ID => self::MARKER, 'name' => 'Stored', Document::PERMISSIONS => $foreign]))); + $old = $this->storedParent(); + $this->database->setCheckGrants(true); + $related = new Document([Document::ID => self::MARKER, 'name' => 'Stored', Document::PERMISSIONS => $foreign]); + $parent = $this->parent($type, $related); + $changes = new Document([self::RELATED => self::value($type, $related)]); + $staged = static fn (string $collection, string $id): ?Document => $collection === self::RELATED && $id === self::MARKER + ? new Document([Document::ID => self::MARKER, Document::COLLECTION => self::RELATED, 'name' => 'Stored', Document::PERMISSIONS => $foreign]) + : null; + + $this->assertNull($this->refusal(fn () => $this->database->authorizeCreate('parents', $parent)), 'Judged as the marked document, the role is sent back unchanged'); + $this->assertNull($this->refusal(fn () => $this->database->authorizeUpdate('parents', $old, $changes))); + + $this->database->addHook($this->marker()); + + $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $this->database->authorizeCreate('parents', $parent)), 'Judged after the rewrite, the document is new and every role is granted anew'); + $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $this->database->authorizeUpdate('parents', $old, $changes))); + $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $this->database->authorizeCreate('parents', $parent, $staged)), 'A staged view of the marker is not the rewritten document'); + $this->assertSame([], $this->relatedNames()); } /** @@ -155,6 +225,40 @@ public function testAPlaceholderNeedsCreateWhereALiteralDocumentNeedsUpdate(Clos * @param list|null $storedPermissions The related document's permissions; null stores none */ private function seed(Adapter $adapter, RelationshipType $type, array $collectionPermissions, ?array $storedPermissions): void + { + $this->connect($adapter); + + $this->authorization->skip(function () use ($type, $collectionPermissions, $storedPermissions): void { + $this->createCollections([self::RELATED], $collectionPermissions); + $this->database->createRelationship('parents', match ($type) { + RelationshipType::OneToOne => Relationship::oneToOne(self::RELATED, self::RELATED, onDelete: RelationshipDeleteAction::SetNull), + RelationshipType::ManyToOne => Relationship::manyToOne(self::RELATED, self::RELATED, onDelete: RelationshipDeleteAction::SetNull), + RelationshipType::OneToMany => Relationship::oneToMany(self::RELATED, self::RELATED, twoWay: true, twoWayKey: self::BACK, onDelete: RelationshipDeleteAction::SetNull), + RelationshipType::ManyToMany => Relationship::manyToMany(self::RELATED, self::RELATED, twoWay: true, twoWayKey: self::BACK, onDelete: RelationshipDeleteAction::SetNull), + }); + if ($storedPermissions !== null) { + $this->database->createDocument(self::RELATED, new Document([Document::ID => self::STORED, 'name' => 'Stored', Document::PERMISSIONS => $storedPermissions])); + } + }); + } + + /** + * Parents relate RELATED, which relate LEAVES, the same way, each related back by BACK. + * + * @param list $relatedPermissions The permissions of the RELATED and LEAVES collections + */ + private function chain(Adapter $adapter, RelationshipType $type, array $relatedPermissions): void + { + $this->connect($adapter); + + $this->authorization->skip(function () use ($type, $relatedPermissions): void { + $this->createCollections([self::RELATED, self::LEAVES], $relatedPermissions); + $this->database->createRelationship('parents', self::twoWayRelationship($type, self::RELATED, self::RELATED)); + $this->database->createRelationship(self::RELATED, self::twoWayRelationship($type, self::LEAVES, self::NEXT)); + }); + } + + private function connect(Adapter $adapter): void { $this->authorization = new Authorization(); $this->authorization->addRole(Role::users()->toString()); @@ -165,30 +269,77 @@ private function seed(Adapter $adapter, RelationshipType $type, array $collectio ->setNamespace('nested_pre_check_'.\uniqid()); $this->database->addHook(new Permissions()); $this->database->addHook(new Relationships()); + } - $this->authorization->skip(function () use ($type, $collectionPermissions, $storedPermissions): void { - $this->database->create(); - $this->database->createCollection(Collection::create( - 'parents', - attributes: [Attribute::string('name', size: 64)], - permissions: [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], - )); + /** + * Creates the parents collection and the $related collections, which have $permissions. + * + * @param list $related + * @param list $permissions + */ + private function createCollections(array $related, array $permissions): void + { + $this->database->create(); + $this->database->createCollection(Collection::create( + 'parents', + attributes: [Attribute::string('name', size: 64)], + permissions: [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], + )); + foreach ($related as $collection) { $this->database->createCollection(Collection::create( - self::RELATED, + $collection, attributes: [Attribute::string('name', size: 64)], - permissions: $collectionPermissions, + permissions: $permissions, documentSecurity: false, )); - $this->database->createRelationship('parents', match ($type) { - RelationshipType::OneToOne => Relationship::oneToOne(self::RELATED, self::RELATED, onDelete: RelationshipDeleteAction::SetNull), - RelationshipType::ManyToOne => Relationship::manyToOne(self::RELATED, self::RELATED, onDelete: RelationshipDeleteAction::SetNull), - RelationshipType::OneToMany => Relationship::oneToMany(self::RELATED, self::RELATED, twoWay: true, twoWayKey: 'back', onDelete: RelationshipDeleteAction::SetNull), - RelationshipType::ManyToMany => Relationship::manyToMany(self::RELATED, self::RELATED, twoWay: true, twoWayKey: 'back', onDelete: RelationshipDeleteAction::SetNull), - }); - if ($storedPermissions !== null) { - $this->database->createDocument(self::RELATED, new Document([Document::ID => self::STORED, 'name' => 'Stored', Document::PERMISSIONS => $storedPermissions])); + } + } + + private static function twoWayRelationship(RelationshipType $type, string $related, string $key): Relationship + { + return match ($type) { + RelationshipType::OneToOne => Relationship::oneToOne($related, $key, twoWay: true, twoWayKey: self::BACK, onDelete: RelationshipDeleteAction::SetNull), + RelationshipType::ManyToOne => Relationship::manyToOne($related, $key, twoWay: true, twoWayKey: self::BACK, onDelete: RelationshipDeleteAction::SetNull), + RelationshipType::OneToMany => Relationship::oneToMany($related, $key, twoWay: true, twoWayKey: self::BACK, onDelete: RelationshipDeleteAction::SetNull), + RelationshipType::ManyToMany => Relationship::manyToMany($related, $key, twoWay: true, twoWayKey: self::BACK, onDelete: RelationshipDeleteAction::SetNull), + }; + } + + /** + * A hook that records each related document it is given and replaces a MARKER ID with a generated one. + */ + private function marker(): Nested + { + return new class (self::MARKER, $this->record(...)) implements Nested { + /** + * @param Closure(string, string): void $record + */ + public function __construct( + private readonly string $marker, + private readonly Closure $record, + ) { } - }); + + #[\Override] + public function prepare(Document $collection, Document $document): Document + { + ($this->record)($collection->getId(), $document->getId()); + + return $document->getId() === $this->marker ? $document->setAttribute(Document::ID, Id::unique()) : $document; + } + }; + } + + private function record(string $collection, string $id): void + { + $this->seen[] = [$collection, $id]; + } + + private function resolveAsStored(string $collection, string $id): ?Document + { + $this->resolved[] = [$collection, $id]; + + return null; } private function parent(RelationshipType $type, Document $related): Document @@ -221,6 +372,27 @@ private static function value(RelationshipType $type, Document $related): Docume return \in_array($type, [RelationshipType::OneToMany, RelationshipType::ManyToMany], true) ? [$related] : $related; } + /** + * The value of BACK, the inverse side of a relationship of $type, holding $related. + * + * @return Document|list + */ + private static function reverse(RelationshipType $type, Document $related): Document|array + { + return \in_array($type, [RelationshipType::ManyToOne, RelationshipType::ManyToMany], true) ? [$related] : $related; + } + + /** + * @return list + */ + private function leafNames(): array + { + return $this->authorization->skip(fn (): array => \array_values(\array_map( + static fn (Document $document): string => \is_string($name = $document->getAttribute('name')) ? $name : '', + $this->database->find(self::LEAVES), + ))); + } + /** * @return list */ diff --git a/tests/unit/Relationships/NestedPreparationTest.php b/tests/unit/Relationships/NestedPreparationTest.php index 87baf9509..3b5fb0730 100644 --- a/tests/unit/Relationships/NestedPreparationTest.php +++ b/tests/unit/Relationships/NestedPreparationTest.php @@ -6,6 +6,7 @@ use PDO; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; +use RuntimeException; use Swoole\Coroutine; use Swoole\Coroutine\Channel; use Swoole\Runtime; @@ -24,6 +25,7 @@ use Utopia\Database\Exception\Authorization as AuthorizationException; use Utopia\Database\Exception\Relationship as RelationshipException; use Utopia\Database\Exception\Structure as StructureException; +use Utopia\Database\Hook\Nested; use Utopia\Database\Hook\Permissions; use Utopia\Database\Hook\Relationships; use Utopia\Database\Id; @@ -37,10 +39,10 @@ use function Swoole\Coroutine\run; /** - * With placeholders replaced, a related document sent as a document whose ID is empty or Id::PLACEHOLDER gets a - * generated ID at every depth a write relates, while a string 'unique()' stays a reference to the document stored - * under it. With grants checked, a related document sent with permissions may only add permissions for roles the - * caller holds. + * A registered Hook\Nested prepares every related document a write is given as a document, at every depth the write + * relates it, before the grant check and before relating it, while the top-level document and related documents + * given by ID are left as given. With grants checked, a related document sent with permissions may only add + * permissions for roles the caller holds. */ final class NestedPreparationTest extends TestCase { @@ -56,14 +58,23 @@ final class NestedPreparationTest extends TestCase private const string PREVIOUS = 'previous'; + private const string MARKER = 'marker'; + private const string UNHELD_REFUSAL = 'Permissions must be one of: (any, users, user:u1)'; private const string MALFORMED = 'Wildcard permission "*" has been replaced. Use "any" instead.'; - private const string BULK_REFUSAL = 'Invalid relationship value. A bulk update takes related documents by ID while placeholders are replaced or grants are checked.'; + private const string BULK_REFUSAL = 'Invalid relationship value. A bulk update takes related documents by ID while grants are checked.'; + + private const string HOOK_FAILURE = 'Refused by the nested hook'; private Authorization $authorization; + /** + * @var list The collection and ID of each related document the hooks were given, in order + */ + private array $seen = []; + /** * @return iterable */ @@ -89,192 +100,350 @@ public static function adapters(): iterable * @param Closure(): Adapter $adapter */ #[DataProvider('types')] - public function testPlaceholdersGetGeneratedIdsAtEveryDepth(Closure $adapter, RelationshipType $type): void + public function testANestedHookRewritesEveryDepthOnCreate(Closure $adapter, RelationshipType $type): void { - $database = $this->database($adapter(), $type)->setReplacePlaceholders(true); - $grandchild = new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'grandchild']); - $child = new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'child', self::NEXT => self::value($type, $grandchild)]); - $root = new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'root', self::NEXT => self::value($type, $child)]); + $database = $this->twoWayDatabase($adapter(), $type)->addHook($this->marker()); + $grandchild = new Document([Document::ID => self::MARKER, 'name' => 'grandchild']); + $child = new Document([Document::ID => self::MARKER, 'name' => 'child', self::NEXT => self::value($type, $grandchild)]); - $created = $database->createDocument(self::FIRST, $root); + $database->createDocument(self::FIRST, new Document([Document::ID => 'root', 'name' => 'root', self::NEXT => self::value($type, $child)])); - $rootId = $this->assertGeneratedId($database, self::FIRST, 'root'); $childId = $this->assertGeneratedId($database, self::SECOND, 'child'); $grandchildId = $this->assertGeneratedId($database, self::THIRD, 'grandchild'); - $this->assertSame($rootId, $created->getId()); - $this->assertSame([$childId], self::ids($database->getDocument(self::FIRST, $rootId)->getAttribute(self::NEXT))); + $this->assertSame([$childId], self::ids($database->getDocument(self::FIRST, 'root')->getAttribute(self::NEXT))); $this->assertSame([$grandchildId], self::ids($database->getDocument(self::SECOND, $childId)->getAttribute(self::NEXT))); - $this->assertSame(Id::PLACEHOLDER, $child->getId(), 'The caller\'s nested document is not changed'); - $this->assertSame(Id::PLACEHOLDER, $grandchild->getId(), 'The caller\'s nested document is not changed'); + $this->assertSame(['root'], self::ids($database->getDocument(self::SECOND, $childId)->getAttribute(self::PREVIOUS))); + $this->assertSame([$childId], self::ids($database->getDocument(self::THIRD, $grandchildId)->getAttribute(self::PREVIOUS))); + $this->assertSame(self::MARKER, $child->getId(), 'The caller\'s nested document is not changed'); + $this->assertSame(self::MARKER, $grandchild->getId(), 'The caller\'s nested document is not changed'); + $this->assertSame([[self::SECOND, self::MARKER], [self::THIRD, self::MARKER]], $this->seen, 'The parent is given before its child'); + } + + /** + * @param Closure(): Adapter $adapter + */ + #[DataProvider('types')] + public function testANestedHookRewritesTheInverseSideTwoLevelsDown(Closure $adapter, RelationshipType $type): void + { + $database = $this->twoWayDatabase($adapter(), $type)->addHook($this->marker()); + + $database->createDocument(self::THIRD, new Document([ + Document::ID => 'leaf', + 'name' => 'leaf', + self::PREVIOUS => self::reverse($type, new Document([ + Document::ID => self::MARKER, + 'name' => 'child', + self::PREVIOUS => self::reverse($type, new Document([Document::ID => self::MARKER, 'name' => 'root'])), + ])), + ])); + + $childId = $this->assertGeneratedId($database, self::SECOND, 'child'); + $rootId = $this->assertGeneratedId($database, self::FIRST, 'root'); + $this->assertSame([$childId], self::ids($database->getDocument(self::THIRD, 'leaf')->getAttribute(self::PREVIOUS))); + $this->assertSame([$rootId], self::ids($database->getDocument(self::SECOND, $childId)->getAttribute(self::PREVIOUS))); + $this->assertSame(['leaf'], self::ids($database->getDocument(self::SECOND, $childId)->getAttribute(self::NEXT))); + $this->assertSame([[self::SECOND, self::MARKER], [self::FIRST, self::MARKER]], $this->seen); } /** * @param Closure(): Adapter $adapter */ #[DataProvider('adapters')] - public function testAPlaceholderThreeLevelsDownGetsAGeneratedId(Closure $adapter): void + public function testANestedHookReachesThreeLevelsDownInBothDirections(Closure $adapter): void { - $database = $this->database($adapter(), RelationshipType::ManyToMany)->setReplacePlaceholders(true); + $database = $this->twoWayDatabase($adapter(), RelationshipType::ManyToMany)->addHook($this->marker()); $database->createDocument(self::FIRST, new Document([ Document::ID => 'root', + 'name' => 'root', self::NEXT => [new Document([ - Document::ID => Id::PLACEHOLDER, + Document::ID => self::MARKER, 'name' => 'child', self::NEXT => [new Document([ - Document::ID => Id::PLACEHOLDER, + Document::ID => self::MARKER, 'name' => 'grandchild', - self::PREVIOUS => [new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'great-grandchild'])], + self::PREVIOUS => [new Document([Document::ID => self::MARKER, 'name' => 'great-grandchild'])], + ])], + ])], + ])); + $database->createDocument(self::THIRD, new Document([ + Document::ID => 'leaf', + 'name' => 'leaf', + self::PREVIOUS => [new Document([ + Document::ID => self::MARKER, + 'name' => 'inverse child', + self::PREVIOUS => [new Document([ + Document::ID => self::MARKER, + 'name' => 'inverse grandchild', + self::NEXT => [new Document([Document::ID => self::MARKER, 'name' => 'inverse great-grandchild'])], ])], ])], ])); - $this->assertGeneratedId($database, self::SECOND, 'child'); - $this->assertGeneratedId($database, self::SECOND, 'great-grandchild'); - $this->assertGeneratedId($database, self::THIRD, 'grandchild'); + $child = $this->assertGeneratedId($database, self::SECOND, 'child'); + $grandchild = $this->assertGeneratedId($database, self::THIRD, 'grandchild'); + $greatGrandchild = $this->assertGeneratedId($database, self::SECOND, 'great-grandchild'); + $inverseChild = $this->assertGeneratedId($database, self::SECOND, 'inverse child'); + $inverseGrandchild = $this->assertGeneratedId($database, self::FIRST, 'inverse grandchild'); + $inverseGreatGrandchild = $this->assertGeneratedId($database, self::SECOND, 'inverse great-grandchild'); + $this->assertEqualsCanonicalizing([$child, $greatGrandchild], self::ids($database->getDocument(self::THIRD, $grandchild)->getAttribute(self::PREVIOUS))); + $this->assertEqualsCanonicalizing([$inverseChild, $inverseGreatGrandchild], self::ids($database->getDocument(self::FIRST, $inverseGrandchild)->getAttribute(self::NEXT))); + $this->assertSame([ + [self::SECOND, self::MARKER], + [self::THIRD, self::MARKER], + [self::SECOND, self::MARKER], + [self::SECOND, self::MARKER], + [self::FIRST, self::MARKER], + [self::SECOND, self::MARKER], + ], $this->seen); } /** * @param Closure(): Adapter $adapter */ #[DataProvider('types')] - public function testAnEmptyIdIsGeneratedAndAnExplicitIdIsKept(Closure $adapter, RelationshipType $type): void + public function testANestedHookIsGivenARelatedDocumentSentAsAnArray(Closure $adapter, RelationshipType $type): void { - $database = $this->database($adapter(), $type)->setReplacePlaceholders(true); + $database = $this->twoWayDatabase($adapter(), $type)->addHook($this->marker()); $database->createDocument(self::FIRST, new Document([ Document::ID => 'root', - self::NEXT => self::value($type, new Document([ - Document::ID => 'chosen', - 'name' => 'child', - self::NEXT => self::value($type, new Document(['name' => 'grandchild'])), - ])), + 'name' => 'root', + self::NEXT => self::value($type, [Document::ID => self::MARKER, 'name' => 'child']), ])); - $this->assertSame('child', $database->getDocument(self::SECOND, 'chosen')->getAttribute('name')); - $this->assertGeneratedId($database, self::THIRD, 'grandchild'); + $this->assertSame([$this->assertGeneratedId($database, self::SECOND, 'child')], self::ids($database->getDocument(self::FIRST, 'root')->getAttribute(self::NEXT))); + $this->assertSame([[self::SECOND, self::MARKER]], $this->seen); } /** * @param Closure(): Adapter $adapter */ #[DataProvider('types')] - public function testAStringPlaceholderStaysAReference(Closure $adapter, RelationshipType $type): void + public function testANestedHookRewritesOnUpdate(Closure $adapter, RelationshipType $type): void { - $database = $this->database($adapter(), $type); - $database->createDocument(self::SECOND, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'literal'])); - $database->setReplacePlaceholders(true); + $database = $this->twoWayDatabase($adapter(), $type); + $database->createDocument(self::FIRST, new Document([Document::ID => 'root', 'name' => 'root'])); + $database->addHook($this->marker()); + $child = new Document([ + Document::ID => self::MARKER, + 'name' => 'child', + self::NEXT => self::value($type, new Document([Document::ID => self::MARKER, 'name' => 'grandchild'])), + ]); - $database->createDocument(self::FIRST, new Document([Document::ID => 'linked', self::NEXT => self::value($type, Id::PLACEHOLDER)])); + $database->updateDocument(self::FIRST, 'root', new Document([self::NEXT => self::value($type, $child)])); - $this->assertSame([Id::PLACEHOLDER], self::ids($database->getDocument(self::FIRST, 'linked')->getAttribute(self::NEXT))); - $this->assertSame(['literal'], self::names($database, self::SECOND), 'Linking by the string creates nothing'); + $childId = $this->assertGeneratedId($database, self::SECOND, 'child'); + $grandchildId = $this->assertGeneratedId($database, self::THIRD, 'grandchild'); + $this->assertSame([$childId], self::ids($database->getDocument(self::FIRST, 'root')->getAttribute(self::NEXT))); + $this->assertSame([$grandchildId], self::ids($database->getDocument(self::SECOND, $childId)->getAttribute(self::NEXT))); + $this->assertSame(self::MARKER, $child->getId(), 'The caller\'s nested document is not changed'); + $this->assertSame([[self::SECOND, self::MARKER], [self::THIRD, self::MARKER]], $this->seen); } /** * @param Closure(): Adapter $adapter */ #[DataProvider('types')] - public function testAStringPlaceholderWithoutALiteralDocumentLinksNothing(Closure $adapter, RelationshipType $type): void + public function testANestedHookRewritesOnCreateDocumentsAndUpserts(Closure $adapter, RelationshipType $type): void { - $database = $this->database($adapter(), $type)->setReplacePlaceholders(true); + $database = $this->twoWayDatabase($adapter(), $type)->addHook($this->marker()); + $root = static fn (string $id): Document => new Document([ + Document::ID => $id, + 'name' => $id, + self::NEXT => self::value($type, new Document([ + Document::ID => self::MARKER, + 'name' => "{$id} child", + self::NEXT => self::value($type, new Document([Document::ID => self::MARKER, 'name' => "{$id} grandchild"])), + ])), + ]); + $roots = ['one', 'two']; - $database->createDocument(self::FIRST, new Document([Document::ID => 'linked', self::NEXT => self::value($type, Id::PLACEHOLDER)])); + $database->createDocuments(self::FIRST, [$root('one'), $root('two')]); + if ($database->getAdapter()->hasFeature(Upserts::class)) { + $database->upsertDocument(self::FIRST, $root('three')); + $database->upsertDocuments(self::FIRST, [$root('four')]); + \array_push($roots, 'three', 'four'); + } - $this->assertSame([], self::ids($database->getDocument(self::FIRST, 'linked')->getAttribute(self::NEXT))); - $this->assertSame([], self::names($database, self::SECOND)); + $generated = []; + foreach ($roots as $id) { + $childId = $this->assertGeneratedId($database, self::SECOND, "{$id} child"); + $grandchildId = $this->assertGeneratedId($database, self::THIRD, "{$id} grandchild"); + $this->assertSame([$childId], self::ids($database->getDocument(self::FIRST, $id)->getAttribute(self::NEXT)), $id); + $this->assertSame([$grandchildId], self::ids($database->getDocument(self::SECOND, $childId)->getAttribute(self::NEXT)), $id); + \array_push($generated, $childId, $grandchildId); + } + + $this->assertCount(\count($generated), \array_unique($generated), 'Every write gets its own IDs'); + $this->assertSame(\array_merge(...\array_fill(0, \count($roots), [[self::SECOND, self::MARKER], [self::THIRD, self::MARKER]])), $this->seen); } /** * @param Closure(): Adapter $adapter */ #[DataProvider('types')] - public function testEveryWriteGeneratesNewIds(Closure $adapter, RelationshipType $type): void + public function testTheTopLevelDocumentAndStringValuesAreNotGivenToTheHook(Closure $adapter, RelationshipType $type): void { - $database = $this->database($adapter(), $type)->setReplacePlaceholders(true); - $root = static fn (string $id): Document => new Document([ - Document::ID => $id, - self::NEXT => self::value($type, new Document([Document::ID => Id::PLACEHOLDER, 'name' => $id])), - ]); + $database = $this->twoWayDatabase($adapter(), $type)->addHook($this->marker()); + $database->createDocument(self::SECOND, new Document([Document::ID => self::MARKER, 'name' => 'literal'])); - $database->createDocument(self::FIRST, $root('one')); - $database->createDocuments(self::FIRST, [$root('two'), $root('three')]); + $database->createDocument(self::FIRST, new Document([Document::ID => self::MARKER, 'name' => 'root', self::NEXT => self::value($type, self::MARKER)])); - $one = $this->assertGeneratedId($database, self::SECOND, 'one'); - $two = $this->assertGeneratedId($database, self::SECOND, 'two'); - $three = $this->assertGeneratedId($database, self::SECOND, 'three'); - $this->assertCount(3, \array_unique([$one, $two, $three])); - $this->assertSame([$two], self::ids($database->getDocument(self::FIRST, 'two')->getAttribute(self::NEXT))); + $this->assertSame('root', $database->getDocument(self::FIRST, self::MARKER)->getAttribute('name'), 'The top-level ID is stored as given'); + $this->assertSame([self::MARKER], self::ids($database->getDocument(self::FIRST, self::MARKER)->getAttribute(self::NEXT)), 'The string links the document stored under it'); + $this->assertSame(['literal'], self::names($database, self::SECOND), 'Linking by the string creates nothing'); + $this->assertSame([], $this->seen); } /** * @param Closure(): Adapter $adapter */ - #[DataProvider('adapters')] - public function testTopLevelPlaceholdersAreGenerated(Closure $adapter): void + #[DataProvider('types')] + public function testAnEmptyIdIsStillGeneratedAndAnExplicitIdIsKeptWithAHook(Closure $adapter, RelationshipType $type): void { - $database = $this->database($adapter(), RelationshipType::ManyToOne)->setReplacePlaceholders(true); + $database = $this->twoWayDatabase($adapter(), $type)->addHook($this->marker()); - $database->createDocument(self::SECOND, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'single'])); - $database->createDocuments(self::SECOND, [new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'batched'])]); - if ($database->getAdapter()->hasFeature(Upserts::class)) { - $upserted = $database->upsertDocument(self::SECOND, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'upserted'])); - $database->upsertDocuments(self::SECOND, [new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'upserted in bulk'])]); - $this->assertSame($upserted->getId(), $this->assertGeneratedId($database, self::SECOND, 'upserted')); - $this->assertGeneratedId($database, self::SECOND, 'upserted in bulk'); - } + $database->createDocument(self::FIRST, new Document([ + Document::ID => 'root', + 'name' => 'root', + self::NEXT => self::value($type, new Document([ + Document::ID => 'chosen', + 'name' => 'child', + self::NEXT => self::value($type, new Document(['name' => 'grandchild'])), + ])), + ])); - $this->assertGeneratedId($database, self::SECOND, 'single'); - $this->assertGeneratedId($database, self::SECOND, 'batched'); + $this->assertSame('child', $database->getDocument(self::SECOND, 'chosen')->getAttribute('name')); + $grandchildId = $this->assertGeneratedId($database, self::THIRD, 'grandchild'); + $this->assertSame([$grandchildId], self::ids($database->getDocument(self::SECOND, 'chosen')->getAttribute(self::NEXT))); + $this->assertSame([[self::SECOND, 'chosen'], [self::THIRD, '']], $this->seen); } /** * @param Closure(): Adapter $adapter */ #[DataProvider('types')] - public function testAnUpdateGeneratesIdsForNestedPlaceholders(Closure $adapter, RelationshipType $type): void + public function testHooksRunInRegistrationOrder(Closure $adapter, RelationshipType $type): void { - $database = $this->database($adapter(), $type); - $database->createDocument(self::FIRST, new Document([Document::ID => 'root', 'name' => 'root'])); - $database->setReplacePlaceholders(true); + $database = $this->twoWayDatabase($adapter(), $type) + ->addHook($this->hook(static fn (string $id): string => $id === self::MARKER ? 'renamed' : $id)) + ->addHook($this->hook(static fn (string $id): string => $id === 'renamed' ? 'final' : $id)); - $database->updateDocument(self::FIRST, 'root', new Document([ - self::NEXT => self::value($type, new Document([ - Document::ID => Id::PLACEHOLDER, - 'name' => 'child', - self::NEXT => self::value($type, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'grandchild'])), - ])), + $database->createDocument(self::FIRST, new Document([ + Document::ID => 'root', + 'name' => 'root', + self::NEXT => self::value($type, new Document([Document::ID => self::MARKER, 'name' => 'child'])), ])); - $childId = $this->assertGeneratedId($database, self::SECOND, 'child'); - $this->assertGeneratedId($database, self::THIRD, 'grandchild'); - $this->assertSame([$childId], self::ids($database->getDocument(self::FIRST, 'root')->getAttribute(self::NEXT))); + $this->assertSame([[self::SECOND, self::MARKER], [self::SECOND, 'renamed']], $this->seen, 'The second hook is given the first hook\'s result'); + $this->assertSame(['final'], self::ids($database->getDocument(self::FIRST, 'root')->getAttribute(self::NEXT))); + $this->assertSame(['child'], self::names($database, self::SECOND)); } /** * @param Closure(): Adapter $adapter */ #[DataProvider('types')] - public function testAnUpsertGeneratesIdsForNestedPlaceholders(Closure $adapter, RelationshipType $type): void + public function testARemovedHookNoLongerRuns(Closure $adapter, RelationshipType $type): void { - $database = $this->database($adapter(), $type)->setReplacePlaceholders(true); - if (! $database->getAdapter()->hasFeature(Upserts::class)) { - $this->expectNotToPerformAssertions(); + $hook = $this->marker(); + $database = $this->twoWayDatabase($adapter(), $type)->addHook($hook); + $root = static fn (string $id, string $childId): Document => new Document([ + Document::ID => $id, + 'name' => $id, + self::NEXT => self::value($type, new Document([Document::ID => $childId, 'name' => "{$id} child"])), + ]); - return; - } + $database->createDocument(self::FIRST, $root('hooked', self::MARKER)); + $this->assertGeneratedId($database, self::SECOND, 'hooked child'); + $database->removeHook($hook); + $database->createDocument(self::FIRST, $root('removed', self::MARKER)); + $database->addHook($this->marker())->removeHook(Nested::class); + $database->createDocument(self::FIRST, $root('removed by class', 'explicit')); + + $this->assertSame([self::MARKER], self::ids($database->getDocument(self::FIRST, 'removed')->getAttribute(self::NEXT)), 'The removed hook leaves the ID as given'); + $this->assertSame([[self::SECOND, self::MARKER]], $this->seen); + } - $database->upsertDocument(self::FIRST, new Document([ - Document::ID => 'single', - self::NEXT => self::value($type, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'single child'])), + /** + * @param Closure(): Adapter $adapter + */ + #[DataProvider('types')] + public function testAHookExceptionWritesNothing(Closure $adapter, RelationshipType $type): void + { + $database = $this->twoWayDatabase($adapter(), $type); + $database->createDocument(self::FIRST, new Document([Document::ID => 'stored', 'name' => 'stored'])); + $database->addHook(new class (self::THIRD, self::HOOK_FAILURE) implements Nested { + public function __construct( + private readonly string $refused, + private readonly string $message, + ) { + } + + #[\Override] + public function prepare(Document $collection, Document $document): Document + { + if ($collection->getId() === $this->refused) { + throw new RuntimeException($this->message); + } + + return $document; + } + }); + $children = static fn (): Document|string|array => self::value($type, new Document([ + 'name' => 'child', + self::NEXT => self::value($type, new Document(['name' => 'grandchild'])), ])); - $database->upsertDocuments(self::FIRST, [new Document([ - Document::ID => 'bulk', - self::NEXT => self::value($type, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'bulk child'])), - ])]); - $this->assertSame([$this->assertGeneratedId($database, self::SECOND, 'single child')], self::ids($database->getDocument(self::FIRST, 'single')->getAttribute(self::NEXT))); - $this->assertSame([$this->assertGeneratedId($database, self::SECOND, 'bulk child')], self::ids($database->getDocument(self::FIRST, 'bulk')->getAttribute(self::NEXT))); + $writes = [ + 'create' => fn () => $database->createDocument(self::FIRST, new Document([Document::ID => 'created', 'name' => 'created', self::NEXT => $children()])), + 'create in bulk' => fn () => $database->createDocuments(self::FIRST, [new Document([Document::ID => 'batched', 'name' => 'batched', self::NEXT => $children()])]), + 'update' => fn () => $database->updateDocument(self::FIRST, 'stored', new Document([self::NEXT => $children()])), + ]; + if ($database->getAdapter()->hasFeature(Upserts::class)) { + $writes['upsert'] = fn () => $database->upsertDocument(self::FIRST, new Document([Document::ID => 'upserted', 'name' => 'upserted', self::NEXT => $children()])); + $writes['upsert in bulk'] = fn () => $database->upsertDocuments(self::FIRST, [new Document([Document::ID => 'upserted', 'name' => 'upserted', self::NEXT => $children()])]); + } + + foreach ($writes as $write => $callback) { + $failure = $this->failure($callback); + + $this->assertInstanceOf(RuntimeException::class, $failure, $write); + $this->assertSame(self::HOOK_FAILURE, $failure->getMessage(), $write); + } + + $this->assertSame(['stored'], self::names($database, self::FIRST)); + $this->assertSame([], self::names($database, self::SECOND)); + $this->assertSame([], self::names($database, self::THIRD)); + $this->assertSame([], self::ids($database->getDocument(self::FIRST, 'stored')->getAttribute(self::NEXT))); + } + + /** + * @param Closure(): Adapter $adapter + */ + #[DataProvider('types')] + public function testTheGrantCheckJudgesTheRewrittenDocument(Closure $adapter, RelationshipType $type): void + { + $database = $this->twoWayDatabase($adapter(), $type); + $foreign = [Permission::read(Role::user('u2'))]; + $database->createDocument(self::SECOND, new Document([Document::ID => self::MARKER, 'name' => 'stored', Document::PERMISSIONS => $foreign])); + $database->setCheckGrants(true); + $database->createDocument(self::FIRST, self::rootWithMarkedChild($type, 'unhooked', $foreign)); + $this->assertSame([self::MARKER], self::ids($database->getDocument(self::FIRST, 'unhooked')->getAttribute(self::NEXT)), 'Without the hook the stored document keeps its permissions'); + + $database->addHook($this->marker()); + $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $database->createDocument(self::FIRST, self::rootWithMarkedChild($type, 'refused', $foreign))), 'The rewritten document is new, so every permission is granted anew'); + $this->assertTrue($this->authorization->skip(fn (): Document => $database->getDocument(self::FIRST, 'refused'))->isEmpty()); + $this->assertSame(['stored'], self::names($database, self::SECOND)); + + $held = [Permission::read(Role::user(self::USER))]; + $database->createDocument(self::FIRST, self::rootWithMarkedChild($type, 'held', $held)); + $created = self::ids($database->getDocument(self::FIRST, 'held')->getAttribute(self::NEXT)); + $this->assertCount(1, $created); + $this->assertNotSame(self::MARKER, $created[0]); + $this->assertSame($held, $database->getDocument(self::SECOND, $created[0])->getPermissions()); + $this->assertSame($foreign, $this->authorization->skip(fn (): Document => $database->getDocument(self::SECOND, self::MARKER))->getPermissions()); } /** @@ -404,91 +573,77 @@ public function testDisabledAuthorizationOrTheToggleOffAllowsAnyRole(Closure $ad * @param Closure(): Adapter $adapter */ #[DataProvider('types')] - public function testAPlaceholderWithAnUnheldRoleWritesNothingAndWithAHeldRoleIsCreated(Closure $adapter, RelationshipType $type): void + public function testABulkUpdateRefusesARelatedDocumentGivenAsADocument(Closure $adapter, RelationshipType $type): void { - $database = $this->database($adapter(), $type)->setReplacePlaceholders(true)->setCheckGrants(true); - $root = fn (string $role): Document => new Document([ - Document::ID => 'root', - self::NEXT => self::value($type, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'child', Document::PERMISSIONS => [Permission::read(Role::parse($role))]])), - ]); + $database = $this->database($adapter(), $type); + $database->createDocuments(self::FIRST, [new Document([Document::ID => 'one', 'name' => 'one']), new Document([Document::ID => 'two', 'name' => 'two'])]); + $database->setCheckGrants(true); + $related = ['name' => 'child', Document::PERMISSIONS => [Permission::update(Role::team('admin'))]]; + $toMany = \in_array($type, [RelationshipType::OneToMany, RelationshipType::ManyToMany], true); + $values = [ + 'document' => self::value($type, new Document([Document::ID => self::MARKER, ...$related])), + 'array' => $toMany ? [$related] : $related, + ]; - $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $database->createDocument(self::FIRST, $root('user:u2')))); - $this->assertWroteNothing($database); + foreach ($values as $form => $value) { + $failure = $this->failure(fn () => $database->updateDocuments(self::FIRST, new Document([self::NEXT => $value]))); - $database->createDocument(self::FIRST, $root('user:'.self::USER)); - $childId = $this->assertGeneratedId($database, self::SECOND, 'child'); - $this->assertSame([Permission::read(Role::user(self::USER))], $database->getDocument(self::SECOND, $childId)->getPermissions()); + $this->assertInstanceOf(RelationshipException::class, $failure, $form); + $this->assertSame(self::BULK_REFUSAL, $failure->getMessage(), $form); + } + + $this->assertSame(['one', 'two'], self::names($database, self::FIRST)); + $this->assertSame([], self::names($database, self::SECOND)); + $this->assertSame([], self::ids($database->getDocument(self::FIRST, 'one')->getAttribute(self::NEXT))); + $this->assertSame([], self::ids($database->getDocument(self::FIRST, 'two')->getAttribute(self::NEXT))); } /** * @param Closure(): Adapter $adapter */ #[DataProvider('types')] - public function testABulkUpdateRefusesARelatedDocumentGivenAsADocument(Closure $adapter, RelationshipType $type): void + public function testABulkUpdateIsNotRefusedAndDoesNotRunTheHookWhileGrantsAreNotChecked(Closure $adapter, RelationshipType $type): void { - $toggles = [ - 'placeholders replaced' => static fn (Database $database): Database => $database->setReplacePlaceholders(true), - 'grants checked' => static fn (Database $database): Database => $database->setCheckGrants(true), - ]; - - foreach ($toggles as $toggle => $enable) { - $database = $this->database($adapter(), $type); - $database->createDocuments(self::FIRST, [new Document([Document::ID => 'one', 'name' => 'one']), new Document([Document::ID => 'two', 'name' => 'two'])]); - $enable($database); - $related = ['name' => 'child', Document::PERMISSIONS => [Permission::update(Role::team('admin'))]]; - $toMany = \in_array($type, [RelationshipType::OneToMany, RelationshipType::ManyToMany], true); - $values = [ - 'document' => self::value($type, new Document([Document::ID => Id::PLACEHOLDER, ...$related])), - 'array' => $toMany ? [$related] : $related, - ]; - - foreach ($values as $form => $value) { - $failure = $this->failure(fn () => $database->updateDocuments(self::FIRST, new Document([self::NEXT => $value]))); - - $this->assertInstanceOf(RelationshipException::class, $failure, "{$toggle}, {$form}"); - $this->assertSame(self::BULK_REFUSAL, $failure->getMessage(), "{$toggle}, {$form}"); - } + $unhooked = $this->bulkUpdate($this->twoWayDatabase($adapter(), $type), $type); + $hooked = $this->bulkUpdate($this->twoWayDatabase($adapter(), $type)->addHook($this->marker()), $type); - $this->assertSame(['one', 'two'], self::names($database, self::FIRST), $toggle); - $this->assertSame([], self::names($database, self::SECOND), $toggle); - $this->assertSame([], self::ids($database->getDocument(self::FIRST, 'one')->getAttribute(self::NEXT)), $toggle); - $this->assertSame([], self::ids($database->getDocument(self::FIRST, 'two')->getAttribute(self::NEXT)), $toggle); - } + $this->assertNotSame(RelationshipException::class.': '.self::BULK_REFUSAL, $unhooked[0]); + $this->assertSame($unhooked, $hooked, 'A registered hook changes nothing a bulk update writes'); + $this->assertSame([], $this->seen); } /** * @param Closure(): Adapter $adapter */ #[DataProvider('adapters')] - public function testABulkUpdateLinksARelatedDocumentByIdWhileBothTogglesAreOn(Closure $adapter): void + public function testABulkUpdateLinksARelatedDocumentByIdWhileGrantsAreChecked(Closure $adapter): void { $database = $this->database($adapter(), RelationshipType::ManyToOne); $database->createDocument(self::SECOND, new Document([Document::ID => 'stored', 'name' => 'stored'])); $database->createDocuments(self::FIRST, [new Document([Document::ID => 'one', 'name' => 'one']), new Document([Document::ID => 'two', 'name' => 'two'])]); - $database->setReplacePlaceholders(true)->setCheckGrants(true); + $database->setCheckGrants(true)->addHook($this->marker()); $this->assertSame(2, $database->updateDocuments(self::FIRST, new Document([self::NEXT => 'stored']))); $this->assertSame(['stored'], self::ids($database->getDocument(self::FIRST, 'one')->getAttribute(self::NEXT))); $this->assertSame(['stored'], self::ids($database->getDocument(self::FIRST, 'two')->getAttribute(self::NEXT))); + $this->assertSame([], $this->seen); } /** * @param Closure(): Adapter $adapter */ #[DataProvider('adapters')] - public function testTheTogglesAreScopedAndOffByDefault(Closure $adapter): void + public function testTheGrantCheckIsScopedAndOffByDefault(Closure $adapter): void { $database = $this->database($adapter(), RelationshipType::ManyToOne); - $toggles = static fn (): array => [$database->isReplacingPlaceholders(), $database->isCheckingGrants()]; - - $this->assertSame([false, false], $toggles()); - $this->assertSame([true, true], $database->withReplacePlaceholders(true, fn (): array => $database->withCheckGrants(true, $toggles))); - $this->assertSame([false, false], $toggles()); + $this->assertFalse($database->isCheckingGrants()); + $this->assertTrue($database->withCheckGrants(true, $database->isCheckingGrants(...))); + $this->assertFalse($database->isCheckingGrants()); } - public function testASnapshotCarriesBothTogglesIntoAChildCoroutine(): void + public function testASnapshotCarriesTheGrantCheckIntoAChildCoroutine(): void { if (! \extension_loaded('swoole')) { $this->markTestSkipped('ext-swoole is required for coroutine-scoped toggles'); @@ -503,18 +658,18 @@ public function testASnapshotCarriesBothTogglesIntoAChildCoroutine(): void $released = new Channel(1); $results = new Channel(1); - $database->withReplacePlaceholders(true, fn () => $database->withCheckGrants(true, function () use ($database, $released, $results): void { + $database->withCheckGrants(true, function () use ($database, $released, $results): void { $snapshot = $database->snapshot(); Coroutine::create(function () use ($database, $snapshot, $released, $results): void { $released->pop(); $results->push([ - 'unscoped' => $this->refusal(fn () => $database->createDocument(self::FIRST, $this->placeholderChild('unscoped', 'user:u2'))), - 'refused' => $database->withSnapshot($snapshot, fn (): ?string => $this->refusal(fn () => $database->createDocument(self::FIRST, $this->placeholderChild('refused', 'user:u2')))), - 'generated' => $database->withSnapshot($snapshot, fn (): ?string => $this->refusal(fn () => $database->createDocument(self::FIRST, $this->placeholderChild('generated', 'user:'.self::USER)))), + 'unscoped' => $this->refusal(fn () => $database->createDocument(self::FIRST, $this->rootWithChild('unscoped', 'user:u2'))), + 'refused' => $database->withSnapshot($snapshot, fn (): ?string => $this->refusal(fn () => $database->createDocument(self::FIRST, $this->rootWithChild('refused', 'user:u2')))), + 'held' => $database->withSnapshot($snapshot, fn (): ?string => $this->refusal(fn () => $database->createDocument(self::FIRST, $this->rootWithChild('held', 'user:'.self::USER)))), ]); }); - })); + }); $released->push(true); $outcomes = $results->pop(5); @@ -523,23 +678,114 @@ public function testASnapshotCarriesBothTogglesIntoAChildCoroutine(): void Runtime::setHookFlags($hookFlags); } - $this->assertSame(['unscoped' => null, 'refused' => self::UNHELD_REFUSAL, 'generated' => null], $outcomes); - $this->assertSame([Id::PLACEHOLDER], self::ids($database->getDocument(self::FIRST, 'unscoped')->getAttribute(self::NEXT)), 'Outside the snapshot both toggles are off'); + $this->assertSame(['unscoped' => null, 'refused' => self::UNHELD_REFUSAL, 'held' => null], $outcomes); + $this->assertSame(['unscoped child'], self::ids($database->getDocument(self::FIRST, 'unscoped')->getAttribute(self::NEXT)), 'Outside the snapshot grants are not checked'); $this->assertTrue($database->getDocument(self::FIRST, 'refused')->isEmpty()); - $generated = self::ids($database->getDocument(self::FIRST, 'generated')->getAttribute(self::NEXT)); - $this->assertCount(1, $generated); - $this->assertNotSame(Id::PLACEHOLDER, $generated[0]); + $this->assertSame(['held child'], self::ids($database->getDocument(self::FIRST, 'held')->getAttribute(self::NEXT))); } - private function placeholderChild(string $id, string $role): Document + /** + * @param list $permissions The permissions of the root's child, stored under MARKER + */ + private static function rootWithMarkedChild(RelationshipType $type, string $id, array $permissions): Document { return new Document([ Document::ID => $id, - self::NEXT => new Document([Document::ID => Id::PLACEHOLDER, 'name' => $id, Document::PERMISSIONS => [Permission::read(Role::parse($role))]]), + 'name' => $id, + self::NEXT => self::value($type, new Document([Document::ID => self::MARKER, 'name' => 'stored', Document::PERMISSIONS => $permissions])), ]); } + private function rootWithChild(string $id, string $role): Document + { + return new Document([ + Document::ID => $id, + self::NEXT => new Document([Document::ID => "{$id} child", 'name' => $id, Document::PERMISSIONS => [Permission::read(Role::parse($role))]]), + ]); + } + + /** + * Bulk updates the only document of FIRST to relate a MARKER document. + * + * @return array{?string, list, list, list} The failure, the names in FIRST and SECOND and the IDs the document relates + */ + private function bulkUpdate(Database $database, RelationshipType $type): array + { + $database->createDocument(self::FIRST, new Document([Document::ID => 'one', 'name' => 'one'])); + + $failure = $this->failure(fn () => $database->updateDocuments(self::FIRST, new Document([ + self::NEXT => self::value($type, new Document([Document::ID => self::MARKER, 'name' => 'child'])), + ]))); + + return [ + $failure === null ? null : $failure::class.': '.$failure->getMessage(), + self::names($database, self::FIRST), + self::names($database, self::SECOND), + self::ids($database->getDocument(self::FIRST, 'one')->getAttribute(self::NEXT)), + ]; + } + + /** + * A hook that records each related document it is given and replaces a MARKER ID with a generated one. + */ + private function marker(): Nested + { + return $this->hook(static fn (string $id): string => $id === self::MARKER ? Id::unique() : $id); + } + + /** + * @param Closure(string): string $rename The ID the related document gets, given its ID + */ + private function hook(Closure $rename): Nested + { + return new class ($rename, $this->record(...)) implements Nested { + /** + * @param Closure(string): string $rename + * @param Closure(string, string): void $record + */ + public function __construct( + private readonly Closure $rename, + private readonly Closure $record, + ) { + } + + #[\Override] + public function prepare(Document $collection, Document $document): Document + { + ($this->record)($collection->getId(), $document->getId()); + + return $document->setAttribute(Document::ID, ($this->rename)($document->getId())); + } + }; + } + + private function record(string $collection, string $id): void + { + $this->seen[] = [$collection, $id]; + } + private function database(Adapter $adapter, RelationshipType $type): Database + { + $database = $this->collections($adapter); + $database->createRelationship(self::FIRST, self::relationship($type, self::SECOND)); + $database->createRelationship(self::SECOND, self::relationship($type, self::THIRD)); + + return $database; + } + + /** + * FIRST relates SECOND, which relates THIRD, each by NEXT, and back by PREVIOUS. + */ + private function twoWayDatabase(Adapter $adapter, RelationshipType $type): Database + { + $database = $this->collections($adapter); + $database->createRelationship(self::FIRST, self::twoWayRelationship($type, self::SECOND)); + $database->createRelationship(self::SECOND, self::twoWayRelationship($type, self::THIRD)); + + return $database; + } + + private function collections(Adapter $adapter): Database { $this->authorization = new Authorization(); $this->authorization->addRole(Role::users()->toString()); @@ -562,9 +808,6 @@ private function database(Adapter $adapter, RelationshipType $type): Database )); } - $database->createRelationship(self::FIRST, self::relationship($type, self::SECOND)); - $database->createRelationship(self::SECOND, self::relationship($type, self::THIRD)); - return $database; } @@ -578,14 +821,37 @@ private static function relationship(RelationshipType $type, string $related): R }; } + private static function twoWayRelationship(RelationshipType $type, string $related): Relationship + { + return match ($type) { + RelationshipType::OneToOne => Relationship::oneToOne($related, self::NEXT, twoWay: true, twoWayKey: self::PREVIOUS, onDelete: RelationshipDeleteAction::SetNull), + RelationshipType::ManyToOne => Relationship::manyToOne($related, self::NEXT, twoWay: true, twoWayKey: self::PREVIOUS, onDelete: RelationshipDeleteAction::SetNull), + RelationshipType::OneToMany => Relationship::oneToMany($related, self::NEXT, twoWay: true, twoWayKey: self::PREVIOUS, onDelete: RelationshipDeleteAction::SetNull), + RelationshipType::ManyToMany => Relationship::manyToMany($related, self::NEXT, twoWay: true, twoWayKey: self::PREVIOUS, onDelete: RelationshipDeleteAction::SetNull), + }; + } + /** - * @return Document|string|list + * The value of NEXT holding $related. + * + * @param Document|string|array $related + * @return Document|string|array */ - private static function value(RelationshipType $type, Document|string $related): Document|string|array + private static function value(RelationshipType $type, Document|string|array $related): Document|string|array { return \in_array($type, [RelationshipType::OneToMany, RelationshipType::ManyToMany], true) ? [$related] : $related; } + /** + * The value of PREVIOUS, the inverse side of NEXT, holding $related. + * + * @return Document|list + */ + private static function reverse(RelationshipType $type, Document $related): Document|array + { + return \in_array($type, [RelationshipType::ManyToOne, RelationshipType::ManyToMany], true) ? [$related] : $related; + } + /** * The generated ID of the only document named $name in $collection. */ @@ -599,8 +865,8 @@ private function assertGeneratedId(Database $database, string $collection, strin $this->assertCount(1, $matches, "One document named {$name} in {$collection}"); $id = $matches[0]->getId(); $this->assertNotSame('', $id); - $this->assertNotSame(Id::PLACEHOLDER, $id); - $this->assertTrue($this->authorization->skip(fn (): bool => $database->getDocument($collection, Id::PLACEHOLDER)->isEmpty()), "Nothing is stored under the placeholder in {$collection}"); + $this->assertNotSame(self::MARKER, $id); + $this->assertTrue($this->authorization->skip(fn (): bool => $database->getDocument($collection, self::MARKER)->isEmpty()), "Nothing is stored under the marker in {$collection}"); return $id; } From 9fb30a034db1e2b07e00f3c07ca9f94d200787ea Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 19:10:21 +1300 Subject: [PATCH 45/55] test(relationships): cover the nested hook on every adapter Replaces the placeholder e2e tests with Hook\Nested ones so every adapter job, Mirror, Pool and shared tables included, shows a registered hook rewriting related documents of every relationship type from either side, in creates, updates, createDocuments and (behind Feature\Upserts) upserts; keeping a string marker a reference; and running in the pre-checks before the grant check. Each test removes its hook in a finally block because the database is shared across a test class. The bulk refusal test asserts the grants-only message. Co-Authored-By: Claude Opus 5.5 --- .../Relationships/NestedPreparationTests.php | 427 ++++++++++++++---- 1 file changed, 344 insertions(+), 83 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/Relationships/NestedPreparationTests.php b/tests/e2e/Adapter/Scopes/Relationships/NestedPreparationTests.php index df4cfe58b..3d94f1c27 100644 --- a/tests/e2e/Adapter/Scopes/Relationships/NestedPreparationTests.php +++ b/tests/e2e/Adapter/Scopes/Relationships/NestedPreparationTests.php @@ -2,6 +2,7 @@ namespace Tests\E2E\Adapter\Scopes\Relationships; +use Closure; use Utopia\Database\Adapter\Feature; use Utopia\Database\Attribute; use Utopia\Database\Collection; @@ -9,6 +10,7 @@ use Utopia\Database\Document; use Utopia\Database\Exception\Authorization as AuthorizationException; use Utopia\Database\Exception\Relationship as RelationshipException; +use Utopia\Database\Hook\Nested; use Utopia\Database\Id; use Utopia\Database\Permission; use Utopia\Database\Query; @@ -18,7 +20,14 @@ trait NestedPreparationTests { - public function testNestedPreparationGeneratesPlaceholderIdsOnCreate(): void + private const string NESTED_PREPARATION_MARKER = 'marker'; + + /** + * @var list The collection and ID of each related document the nested hook was given + */ + private array $nestedPreparationSeen = []; + + public function testNestedHookRewritesRelatedDocumentsOnCreate(): void { $database = $this->getDatabase(); if (! $database->getAdapter()->hasFeature(Feature\Relationships::class)) { @@ -27,35 +36,55 @@ public function testNestedPreparationGeneratesPlaceholderIdsOnCreate(): void return; } - [$parents, $children, $owners] = $this->createNestedPreparationCollections($database); + [$parents, $children, $owners, $profiles, $tags] = $this->createNestedPreparationHookCollections($database); + $hook = $this->nestedPreparationHook(); - $created = $database->withReplacePlaceholders(true, fn (): Document => $database->createDocument($parents, new Document([ - Document::ID => Id::PLACEHOLDER, - 'name' => 'parent', - 'children' => [ - new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'placeholder child']), - new Document(['name' => 'empty child']), - new Document([Document::ID => 'explicit', 'name' => 'explicit child']), - ], - 'owner' => new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'owner']), - ]))); + $database->addHook($hook); + try { + $database->createDocument($parents, new Document([ + Document::ID => 'created', + 'name' => 'created', + 'children' => [ + new Document([ + Document::ID => self::NESTED_PREPARATION_MARKER, + 'name' => 'marked child', + 'tags' => [new Document([Document::ID => self::NESTED_PREPARATION_MARKER, 'name' => 'marked tag'])], + ]), + new Document(['name' => 'empty child']), + new Document([Document::ID => 'explicit', 'name' => 'explicit child']), + ], + 'owner' => new Document([Document::ID => self::NESTED_PREPARATION_MARKER, 'name' => 'marked owner']), + 'profile' => new Document([Document::ID => self::NESTED_PREPARATION_MARKER, 'name' => 'marked profile']), + ])); + } finally { + $database->removeHook($hook); + } - $this->assertNotSame(Id::PLACEHOLDER, $created->getId()); - $stored = $database->getDocument($parents, $created->getId()); - $this->assertSame('parent', $stored->getAttribute('name')); + $stored = $database->getDocument($parents, 'created'); $childIds = $this->nestedPreparationIds($stored->getAttribute('children')); $this->assertCount(3, $childIds); $this->assertContains('explicit', $childIds); - $this->assertNotContains(Id::PLACEHOLDER, $childIds); - $owner = $stored->getAttribute('owner'); - $this->assertInstanceOf(Document::class, $owner); - $this->assertNotSame(Id::PLACEHOLDER, $owner->getId()); - $this->assertTrue($database->getDocument($children, Id::PLACEHOLDER)->isEmpty()); - $this->assertTrue($database->getDocument($owners, Id::PLACEHOLDER)->isEmpty()); - $this->assertFalse($database->isReplacingPlaceholders()); + $this->assertNotContains(self::NESTED_PREPARATION_MARKER, $childIds); + $child = $this->nestedPreparationNamed($database, $children, 'marked child'); + $this->assertContains($child->getId(), $childIds); + $tag = $this->nestedPreparationNamed($database, $tags, 'marked tag'); + $this->assertSame([$tag->getId()], $this->nestedPreparationIds($database->getDocument($children, $child->getId())->getAttribute('tags'))); + $owner = $this->nestedPreparationNamed($database, $owners, 'marked owner'); + $this->assertSame([$owner->getId()], $this->nestedPreparationIds($stored->getAttribute('owner'))); + $profile = $this->nestedPreparationNamed($database, $profiles, 'marked profile'); + $this->assertSame([$profile->getId()], $this->nestedPreparationIds($stored->getAttribute('profile'))); + $this->assertNothingStoredUnderTheNestedPreparationMarker($database, [$children, $owners, $profiles, $tags]); + $this->assertEqualsCanonicalizing([ + [$children, self::NESTED_PREPARATION_MARKER], + [$tags, self::NESTED_PREPARATION_MARKER], + [$children, ''], + [$children, 'explicit'], + [$owners, self::NESTED_PREPARATION_MARKER], + [$profiles, self::NESTED_PREPARATION_MARKER], + ], $this->nestedPreparationSeen); } - public function testNestedPreparationGeneratesPlaceholderIdsOnUpdateAndUpsert(): void + public function testNestedHookRewritesOnUpdateUpsertAndCreateDocuments(): void { $database = $this->getDatabase(); if (! $database->getAdapter()->hasFeature(Feature\Relationships::class)) { @@ -64,38 +93,72 @@ public function testNestedPreparationGeneratesPlaceholderIdsOnUpdateAndUpsert(): return; } - [$parents, $children] = $this->createNestedPreparationCollections($database); + [$parents, $children, $owners, $profiles, $tags] = $this->createNestedPreparationHookCollections($database); $database->createDocument($parents, new Document([Document::ID => 'updated', 'name' => 'updated'])); + $upserts = $database->getAdapter()->hasFeature(Feature\Upserts::class); + $hook = $this->nestedPreparationHook(); - $database->withReplacePlaceholders(true, function () use ($database, $parents): void { + $database->addHook($hook); + try { $database->updateDocument($parents, 'updated', new Document([ - 'children' => [new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'updated child'])], + 'children' => [$this->nestedPreparationMarked('updated child', ['tags' => [$this->nestedPreparationMarked('updated tag')]])], ])); - - if ($database->getAdapter()->hasFeature(Feature\Upserts::class)) { + $database->createDocuments($parents, [ + new Document([Document::ID => 'batched', 'name' => 'batched', 'owner' => $this->nestedPreparationMarked('batched owner'), 'profile' => $this->nestedPreparationMarked('batched profile')]), + ]); + if ($upserts) { $database->upsertDocument($parents, new Document([ Document::ID => 'upserted', 'name' => 'upserted', - 'children' => [new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'upserted child'])], + 'children' => [$this->nestedPreparationMarked('upserted child')], + 'profile' => $this->nestedPreparationMarked('upserted profile'), ])); + $database->upsertDocuments($parents, [new Document([ + Document::ID => 'upserted in bulk', + 'name' => 'upserted in bulk', + 'owner' => $this->nestedPreparationMarked('upserted owner'), + ])]); } - }); - - $updated = $this->nestedPreparationIds($database->getDocument($parents, 'updated')->getAttribute('children')); - $this->assertCount(1, $updated); - $this->assertNotSame(Id::PLACEHOLDER, $updated[0]); - $this->assertSame('updated child', $database->getDocument($children, $updated[0])->getAttribute('name')); - $this->assertTrue($database->getDocument($children, Id::PLACEHOLDER)->isEmpty()); - - if ($database->getAdapter()->hasFeature(Feature\Upserts::class)) { - $upserted = $this->nestedPreparationIds($database->getDocument($parents, 'upserted')->getAttribute('children')); - $this->assertCount(1, $upserted); - $this->assertNotSame(Id::PLACEHOLDER, $upserted[0]); - $this->assertNotSame($updated[0], $upserted[0]); + } finally { + $database->removeHook($hook); } + + $updatedChild = $this->nestedPreparationNamed($database, $children, 'updated child'); + $this->assertSame([$updatedChild->getId()], $this->nestedPreparationIds($database->getDocument($parents, 'updated')->getAttribute('children'))); + $updatedTag = $this->nestedPreparationNamed($database, $tags, 'updated tag'); + $this->assertSame([$updatedTag->getId()], $this->nestedPreparationIds($database->getDocument($children, $updatedChild->getId())->getAttribute('tags'))); + $batched = $database->getDocument($parents, 'batched'); + $batchedOwner = $this->nestedPreparationNamed($database, $owners, 'batched owner'); + $batchedProfile = $this->nestedPreparationNamed($database, $profiles, 'batched profile'); + $this->assertSame([$batchedOwner->getId()], $this->nestedPreparationIds($batched->getAttribute('owner'))); + $this->assertSame([$batchedProfile->getId()], $this->nestedPreparationIds($batched->getAttribute('profile'))); + $generated = [$updatedChild->getId(), $updatedTag->getId(), $batchedOwner->getId(), $batchedProfile->getId()]; + + if ($upserts) { + $upserted = $database->getDocument($parents, 'upserted'); + $upsertedChild = $this->nestedPreparationNamed($database, $children, 'upserted child'); + $upsertedProfile = $this->nestedPreparationNamed($database, $profiles, 'upserted profile'); + $upsertedOwner = $this->nestedPreparationNamed($database, $owners, 'upserted owner'); + $this->assertSame([$upsertedChild->getId()], $this->nestedPreparationIds($upserted->getAttribute('children'))); + $this->assertSame([$upsertedProfile->getId()], $this->nestedPreparationIds($upserted->getAttribute('profile'))); + $this->assertSame([$upsertedOwner->getId()], $this->nestedPreparationIds($database->getDocument($parents, 'upserted in bulk')->getAttribute('owner'))); + \array_push($generated, $upsertedChild->getId(), $upsertedProfile->getId(), $upsertedOwner->getId()); + } + + $this->assertNotContains(self::NESTED_PREPARATION_MARKER, $generated); + $this->assertCount(\count($generated), \array_unique($generated)); + $this->assertNothingStoredUnderTheNestedPreparationMarker($database, [$children, $owners, $profiles, $tags]); + $marker = self::NESTED_PREPARATION_MARKER; + $this->assertEqualsCanonicalizing([ + [$children, $marker], + [$tags, $marker], + [$owners, $marker], + [$profiles, $marker], + ...($upserts ? [[$children, $marker], [$profiles, $marker], [$owners, $marker]] : []), + ], $this->nestedPreparationSeen); } - public function testNestedPreparationKeepsAStringPlaceholderAsAReference(): void + public function testNestedHookRewritesFromTheChildSide(): void { $database = $this->getDatabase(); if (! $database->getAdapter()->hasFeature(Feature\Relationships::class)) { @@ -104,19 +167,77 @@ public function testNestedPreparationKeepsAStringPlaceholderAsAReference(): void return; } - [$parents, , $owners] = $this->createNestedPreparationCollections($database); - $database->createDocument($owners, new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'literal'])); + [$parents, $children, $owners, $profiles, $tags] = $this->createNestedPreparationHookCollections($database); + $marked = static fn (string $name, array $attributes = []): Document => new Document([Document::ID => self::NESTED_PREPARATION_MARKER, 'name' => $name, ...$attributes]); + $hook = $this->nestedPreparationHook(); - $database->withReplacePlaceholders(true, fn (): Document => $database->createDocument($parents, new Document([ - Document::ID => 'linked', - 'name' => 'linked', - 'owner' => Id::PLACEHOLDER, - ]))); + $database->addHook($hook); + try { + $database->createDocument($children, new Document([ + Document::ID => 'child side', + 'name' => 'child side', + 'parent' => $this->nestedPreparationMarked('parent of a child', ['owner' => $this->nestedPreparationMarked('owner of a parent')]), + ])); + $database->createDocument($owners, new Document([Document::ID => 'owner side', 'name' => 'owner side', 'parents' => [$this->nestedPreparationMarked('parent of an owner')]])); + $database->createDocument($profiles, new Document([Document::ID => 'profile side', 'name' => 'profile side', 'parent' => $this->nestedPreparationMarked('parent of a profile')])); + $database->createDocument($tags, new Document([Document::ID => 'tag side', 'name' => 'tag side', 'children' => [$this->nestedPreparationMarked('child of a tag')]])); + } finally { + $database->removeHook($hook); + } + + $parentOfAChild = $this->nestedPreparationNamed($database, $parents, 'parent of a child'); + $this->assertSame([$parentOfAChild->getId()], $this->nestedPreparationIds($database->getDocument($children, 'child side')->getAttribute('parent'))); + $ownerOfAParent = $this->nestedPreparationNamed($database, $owners, 'owner of a parent'); + $this->assertSame([$ownerOfAParent->getId()], $this->nestedPreparationIds($database->getDocument($parents, $parentOfAChild->getId())->getAttribute('owner'))); + $parentOfAnOwner = $this->nestedPreparationNamed($database, $parents, 'parent of an owner'); + $this->assertSame([$parentOfAnOwner->getId()], $this->nestedPreparationIds($database->getDocument($owners, 'owner side')->getAttribute('parents'))); + $parentOfAProfile = $this->nestedPreparationNamed($database, $parents, 'parent of a profile'); + $this->assertSame([$parentOfAProfile->getId()], $this->nestedPreparationIds($database->getDocument($profiles, 'profile side')->getAttribute('parent'))); + $childOfATag = $this->nestedPreparationNamed($database, $children, 'child of a tag'); + $this->assertSame([$childOfATag->getId()], $this->nestedPreparationIds($database->getDocument($tags, 'tag side')->getAttribute('children'))); + $this->assertNothingStoredUnderTheNestedPreparationMarker($database, [$parents, $children, $owners]); + $marker = self::NESTED_PREPARATION_MARKER; + $this->assertEqualsCanonicalizing([ + [$parents, $marker], + [$owners, $marker], + [$parents, $marker], + [$parents, $marker], + [$children, $marker], + ], $this->nestedPreparationSeen); + } + + public function testNestedHookKeepsAStringMarkerAsAReference(): void + { + $database = $this->getDatabase(); + if (! $database->getAdapter()->hasFeature(Feature\Relationships::class)) { + $this->expectNotToPerformAssertions(); + + return; + } + + [$parents, , $owners] = $this->createNestedPreparationHookCollections($database); + $database->createDocument($owners, new Document([Document::ID => self::NESTED_PREPARATION_MARKER, 'name' => 'literal'])); + $hook = $this->nestedPreparationHook(); + + $database->addHook($hook); + try { + $database->createDocument($parents, new Document([ + Document::ID => self::NESTED_PREPARATION_MARKER, + 'name' => 'linked', + 'owner' => self::NESTED_PREPARATION_MARKER, + ])); + } finally { + $database->removeHook($hook); + } - $owner = $database->getDocument($parents, 'linked')->getAttribute('owner'); + $linked = $database->getDocument($parents, self::NESTED_PREPARATION_MARKER); + $this->assertSame('linked', $linked->getAttribute('name'), 'The top-level ID is stored as given'); + $owner = $linked->getAttribute('owner'); $this->assertInstanceOf(Document::class, $owner); - $this->assertSame(Id::PLACEHOLDER, $owner->getId()); + $this->assertSame(self::NESTED_PREPARATION_MARKER, $owner->getId()); $this->assertSame('literal', $owner->getAttribute('name')); + $this->assertCount(1, $database->find($owners)); + $this->assertSame([], $this->nestedPreparationSeen); } public function testNestedPreparationChecksGrants(): void @@ -151,7 +272,7 @@ public function testNestedPreparationChecksGrants(): void $this->assertSame([], $this->nestedPreparationIds($database->getDocument($parents, 'update target')->getAttribute('children'))); } - public function testNestedPreparationRefusesAnUnheldPlaceholderInWritesAndPreChecks(): void + public function testNestedHookRunsInThePreChecksBeforeTheGrantCheck(): void { $database = $this->getDatabase(); if (! $database->getAdapter()->hasFeature(Feature\Relationships::class)) { @@ -160,25 +281,36 @@ public function testNestedPreparationRefusesAnUnheldPlaceholderInWritesAndPreChe return; } - [$parents, $children] = $this->createNestedPreparationCollections($database); - $parent = static fn (string $id, string $role): Document => new Document([ - Document::ID => $id, - 'name' => $id, - 'children' => [new Document([Document::ID => Id::PLACEHOLDER, 'name' => $id, Document::PERMISSIONS => [Permission::read(Role::parse($role))]])], - ]); - $prepared = static fn (callable $callback): mixed => $database->withReplacePlaceholders(true, fn (): mixed => $database->withCheckGrants(true, $callback)); + [$parents, $children] = $this->createNestedPreparationHookCollections($database); + $unheld = [Permission::read(Role::user('nestedPreparationUnheld'))]; + $database->getAuthorization()->skip(fn (): Document => $database->createDocument($children, new Document([ + Document::ID => self::NESTED_PREPARATION_MARKER, + 'name' => 'stored', + Document::PERMISSIONS => $unheld, + ]))); + $checked = static fn (callable $callback): mixed => $database->withCheckGrants(true, $callback); - $this->assertSame($this->nestedPreparationUnheldRefusal($database), $this->nestedPreparationRefusal(fn () => $prepared(fn () => $database->authorizeCreate($parents, $parent('checked', 'user:nestedPreparationUnheld'))))); - $this->assertSame($this->nestedPreparationUnheldRefusal($database), $this->nestedPreparationRefusal(fn () => $prepared(fn (): Document => $database->createDocument($parents, $parent('refused', 'user:nestedPreparationUnheld'))))); - $this->assertTrue($database->getDocument($parents, 'refused')->isEmpty()); - $this->assertSame([], $database->find($children)); + $this->assertNull($this->nestedPreparationRefusal(fn () => $checked(fn () => $database->authorizeCreate($parents, $this->nestedPreparationMarkedParent('unhooked', $unheld)))), 'Without the hook the stored document keeps its permissions'); + + $hook = $this->nestedPreparationHook(); + $database->addHook($hook); + try { + $this->assertSame($this->nestedPreparationUnheldRefusal($database), $this->nestedPreparationRefusal(fn () => $checked(fn () => $database->authorizeCreate($parents, $this->nestedPreparationMarkedParent('checked', $unheld))))); + $this->assertSame($this->nestedPreparationUnheldRefusal($database), $this->nestedPreparationRefusal(fn () => $checked(fn (): Document => $database->createDocument($parents, $this->nestedPreparationMarkedParent('refused', $unheld))))); + $this->assertNull($this->nestedPreparationRefusal(fn () => $checked(fn () => $database->authorizeCreate($parents, $this->nestedPreparationMarkedParent('created', [Permission::read(Role::any())]))))); + $checked(fn (): Document => $database->createDocument($parents, $this->nestedPreparationMarkedParent('created', [Permission::read(Role::any())]))); + } finally { + $database->removeHook($hook); + } - $this->assertNull($this->nestedPreparationRefusal(fn () => $prepared(fn () => $database->authorizeCreate($parents, $parent('created', 'any'))))); - $prepared(fn (): Document => $database->createDocument($parents, $parent('created', 'any'))); + $this->assertTrue($database->getDocument($parents, 'refused')->isEmpty()); $created = $this->nestedPreparationIds($database->getDocument($parents, 'created')->getAttribute('children')); $this->assertCount(1, $created); - $this->assertNotSame(Id::PLACEHOLDER, $created[0]); - $this->assertTrue($database->getDocument($children, Id::PLACEHOLDER)->isEmpty()); + $this->assertNotSame(self::NESTED_PREPARATION_MARKER, $created[0]); + $this->assertSame([Permission::read(Role::any())], $database->getDocument($children, $created[0])->getPermissions()); + $this->assertCount(2, $database->getAuthorization()->skip(fn (): array => $database->find($children))); + $this->assertSame($unheld, $database->getAuthorization()->skip(fn (): Document => $database->getDocument($children, self::NESTED_PREPARATION_MARKER))->getPermissions()); + $this->assertSame(\array_fill(0, 4, [$children, self::NESTED_PREPARATION_MARKER]), $this->nestedPreparationSeen); } public function testNestedPreparationRefusesRelatedDocumentsInABulkUpdate(): void @@ -192,24 +324,18 @@ public function testNestedPreparationRefusesRelatedDocumentsInABulkUpdate(): voi [$parents, $children, $owners] = $this->createNestedPreparationCollections($database); $database->createDocument($parents, new Document([Document::ID => 'bulk', 'name' => 'bulk'])); - $scopes = [ - 'placeholders replaced' => static fn (callable $callback): mixed => $database->withReplacePlaceholders(true, $callback), - 'grants checked' => static fn (callable $callback): mixed => $database->withCheckGrants(true, $callback), - ]; $permissions = [Permission::update(Role::user('nestedPreparationUnheld'))]; $values = [ - 'children' => [new Document([Document::ID => Id::PLACEHOLDER, 'name' => 'bulk child', Document::PERMISSIONS => $permissions])], + 'children' => [new Document([Document::ID => self::NESTED_PREPARATION_MARKER, 'name' => 'bulk child', Document::PERMISSIONS => $permissions])], 'owner' => ['name' => 'bulk owner', Document::PERMISSIONS => $permissions], ]; - foreach ($scopes as $toggle => $scope) { - foreach ($values as $key => $value) { - try { - $scope(fn (): mixed => $database->updateDocuments($parents, new Document([$key => $value]))); - $this->fail("A bulk update with {$key} given as a document is refused while {$toggle}"); - } catch (RelationshipException $error) { - $this->assertSame('Invalid relationship value. A bulk update takes related documents by ID while placeholders are replaced or grants are checked.', $error->getMessage()); - } + foreach ($values as $key => $value) { + try { + $database->withCheckGrants(true, fn (): int => $database->updateDocuments($parents, new Document([$key => $value]))); + $this->fail("A bulk update with {$key} given as a document is refused while grants are checked"); + } catch (RelationshipException $error) { + $this->assertSame('Invalid relationship value. A bulk update takes related documents by ID while grants are checked.', $error->getMessage()); } } @@ -220,11 +346,11 @@ public function testNestedPreparationRefusesRelatedDocumentsInABulkUpdate(): voi $this->assertNull($stored->getAttribute('owner')); $database->createDocument($owners, new Document([Document::ID => 'linked owner', 'name' => 'linked owner'])); - $scopes['placeholders replaced'](fn (): mixed => $scopes['grants checked'](fn (): mixed => $database->updateDocuments( + $database->withCheckGrants(true, fn (): int => $database->updateDocuments( $parents, new Document(['owner' => 'linked owner']), [Query::equal(Document::ID, ['bulk'])], - ))); + )); $this->assertSame(['linked owner'], $this->nestedPreparationIds($database->getDocument($parents, 'bulk')->getAttribute('owner'))); } @@ -269,6 +395,141 @@ private function createNestedPreparationCollections(Database $database): array return [$parents, $children, $owners]; } + /** + * Parents have children (one to many, back by parent), an owner (many to one, back by parents) and a profile + * (one to one, back by parent); children have tags (many to many, back by children). + * + * @return array{string, string, string, string, string} The parent, child, owner, profile and tag collection IDs + */ + private function createNestedPreparationHookCollections(Database $database): array + { + $parents = Id::unique(); + $children = Id::unique(); + $owners = Id::unique(); + $profiles = Id::unique(); + $tags = Id::unique(); + $permissions = [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ]; + + foreach ([$parents, $children, $owners, $profiles, $tags] as $collection) { + $database->createCollection(Collection::create( + id: $collection, + attributes: [Attribute::string(key: 'name', size: 64)], + permissions: $permissions, + documentSecurity: false, + )); + } + + $database->createRelationship($parents, Relationship::oneToMany( + relatedCollection: $children, + key: 'children', + twoWay: true, + twoWayKey: 'parent', + onDelete: RelationshipDeleteAction::SetNull, + )); + $database->createRelationship($parents, Relationship::manyToOne( + relatedCollection: $owners, + key: 'owner', + twoWay: true, + twoWayKey: 'parents', + onDelete: RelationshipDeleteAction::SetNull, + )); + $database->createRelationship($parents, Relationship::oneToOne( + relatedCollection: $profiles, + key: 'profile', + twoWay: true, + twoWayKey: 'parent', + onDelete: RelationshipDeleteAction::SetNull, + )); + $database->createRelationship($children, Relationship::manyToMany( + relatedCollection: $tags, + key: 'tags', + twoWay: true, + twoWayKey: 'children', + onDelete: RelationshipDeleteAction::SetNull, + )); + + return [$parents, $children, $owners, $profiles, $tags]; + } + + /** + * A hook that records each related document's collection and ID and replaces the marker ID with a generated one. + */ + private function nestedPreparationHook(): Nested + { + return new class (self::NESTED_PREPARATION_MARKER, $this->recordNestedPreparation(...)) implements Nested { + /** + * @param Closure(string, string): void $record + */ + public function __construct( + private readonly string $marker, + private readonly Closure $record, + ) { + } + + #[\Override] + public function prepare(Document $collection, Document $document): Document + { + ($this->record)($collection->getId(), $document->getId()); + + return $document->getId() === $this->marker ? $document->setAttribute(Document::ID, Id::unique()) : $document; + } + }; + } + + private function recordNestedPreparation(string $collection, string $id): void + { + $this->nestedPreparationSeen[] = [$collection, $id]; + } + + /** + * @param array $attributes + */ + private function nestedPreparationMarked(string $name, array $attributes = []): Document + { + return new Document([Document::ID => self::NESTED_PREPARATION_MARKER, 'name' => $name, ...$attributes]); + } + + /** + * @param list $permissions The permissions of the parent's child, stored under the marker + */ + private function nestedPreparationMarkedParent(string $id, array $permissions): Document + { + return new Document([ + Document::ID => $id, + 'name' => $id, + 'children' => [new Document([Document::ID => self::NESTED_PREPARATION_MARKER, 'name' => 'stored', Document::PERMISSIONS => $permissions])], + ]); + } + + /** + * The only document named $name in $collection. + */ + private function nestedPreparationNamed(Database $database, string $collection, string $name): Document + { + $matches = $database->getAuthorization()->skip(fn (): array => $database->find($collection, [Query::equal('name', [$name])])); + + $this->assertCount(1, $matches, "One document named {$name}"); + $this->assertNotSame(self::NESTED_PREPARATION_MARKER, $matches[0]->getId()); + $this->assertNotSame('', $matches[0]->getId()); + + return $matches[0]; + } + + /** + * @param list $collections + */ + private function assertNothingStoredUnderTheNestedPreparationMarker(Database $database, array $collections): void + { + foreach ($collections as $collection) { + $this->assertTrue($database->getAuthorization()->skip(fn (): Document => $database->getDocument($collection, self::NESTED_PREPARATION_MARKER))->isEmpty(), "Nothing is stored under the marker in {$collection}"); + } + } + /** * @param list $permissions The permissions of the parent's new child */ From 676bc210939017824d78022289b90f91057d5505 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 19:17:21 +1300 Subject: [PATCH 46/55] test(relationships): drop the unused marker closure and narrow the null resolver PHPStan at level max reports the leftover closure's untyped spread and a ?Document return that is only ever null. Co-Authored-By: Claude Opus 5.5 --- .../e2e/Adapter/Scopes/Relationships/NestedPreparationTests.php | 1 - tests/unit/Relationships/NestedPreCheckTest.php | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/Relationships/NestedPreparationTests.php b/tests/e2e/Adapter/Scopes/Relationships/NestedPreparationTests.php index 3d94f1c27..bdd004309 100644 --- a/tests/e2e/Adapter/Scopes/Relationships/NestedPreparationTests.php +++ b/tests/e2e/Adapter/Scopes/Relationships/NestedPreparationTests.php @@ -168,7 +168,6 @@ public function testNestedHookRewritesFromTheChildSide(): void } [$parents, $children, $owners, $profiles, $tags] = $this->createNestedPreparationHookCollections($database); - $marked = static fn (string $name, array $attributes = []): Document => new Document([Document::ID => self::NESTED_PREPARATION_MARKER, 'name' => $name, ...$attributes]); $hook = $this->nestedPreparationHook(); $database->addHook($hook); diff --git a/tests/unit/Relationships/NestedPreCheckTest.php b/tests/unit/Relationships/NestedPreCheckTest.php index cb013c7b0..51b412994 100644 --- a/tests/unit/Relationships/NestedPreCheckTest.php +++ b/tests/unit/Relationships/NestedPreCheckTest.php @@ -335,7 +335,7 @@ private function record(string $collection, string $id): void $this->seen[] = [$collection, $id]; } - private function resolveAsStored(string $collection, string $id): ?Document + private function resolveAsStored(string $collection, string $id): null { $this->resolved[] = [$collection, $id]; From 19e140299e4113d3dac38f032c5629ade2104abe Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 19:25:52 +1300 Subject: [PATCH 47/55] docs: describe 8.1.0 as the nested hook, grants and fractions only The placeholder API and the one-to-many join rewrite left this PR, so the 8.1.0 notes still promised Id::PLACEHOLDER, setReplacePlaceholders(), a changed join rule and a required Snapshot argument that no longer exist. They now describe what ships: Hook\Nested as the way an application gives nested related documents its own meaning (8.1 still stores 'unique()' as given), the grant check and its bulk refusal while grants are checked, the optional Snapshot arguments, and only the public arithmetic types. With no nested hook and both toggles off 8.1.0 behaves as 8.0.3, so nothing is listed as changed. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 59 +++++++------------ UPGRADE.md | 163 +++++++++++++++++++++++++++------------------------ 2 files changed, 108 insertions(+), 114 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ee311fbb..a5396e7a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,27 +2,31 @@ ## 8.1.0 -8.1.0 adds opt-in and additive APIs for work that callers had to do themselves. With none of the new toggles set, a -`Database` behaves as in 8.0.3, except for the joins listed under Changed. See -[Upgrading from 8.0 to 8.1](UPGRADE.md#upgrading-from-80-to-81). +8.1.0 adds opt-in and additive APIs for work that callers had to do themselves. With no `Hook\Nested` hook registered +and with `setCheckGrants()` and `setRoundFractions()` off, which is the default, a `Database` behaves exactly as in +8.0.3. See [Upgrading from 8.0 to 8.1](UPGRADE.md#upgrading-from-80-to-81). ### Added -- `Id::PLACEHOLDER` (`'unique()'`) and `Id::isPlaceholder(string $id): bool`. -- `Database::setReplacePlaceholders(bool $replace): static`, `isReplacingPlaceholders(): bool` and - `withReplacePlaceholders(bool $replace, callable $callback): mixed`, off by default. While on, a document whose `$id` - is `Id::PLACEHOLDER` gets a generated ID, as one with an empty `$id` does: the documents given to - `createDocument()`, `createDocuments()`, `upsertDocument()` and `upsertDocuments()`, and every related document - given as a document to those and to `updateDocument()`, at every depth the write relates. A related document given - as the string `'unique()'` is still a reference to the document stored under that ID. +- `Utopia\Database\Hook\Nested`, a hook with `prepare(Document $collection, Document $document): Document`. Register + it with `Database::addHook()` and remove it with `removeHook()`. It is given a copy of each related document that + `createDocument()`, `createDocuments()`, `updateDocument()`, `upsertDocument()`, `upsertDocuments()`, + `authorizeCreate()` or `authorizeUpdate()` is given as a document, at every depth the write relates. A parent is + prepared before its children, and hooks run in the order they were registered. The write checks grants on, and + relates, the document the hook returns. The hook is not given the top-level document, a related document given as + an ID, or anything from `updateDocuments()`. A `Mirror` runs it on itself, for the pre-checks, and on its source. + Its destination receives the IDs the source wrote. The library still never interprets `'unique()'`: an ID is stored + as given, as in 8.0. - `Database::setCheckGrants(bool $check): static`, `isCheckingGrants(): bool` and `withCheckGrants(bool $check, callable $callback): mixed`, off by default. While on and while authorization is enabled, a related document given as a document to `createDocument()`, `createDocuments()`, `updateDocument()`, `upsertDocument()`, `upsertDocuments()`, `authorizeCreate()` or `authorizeUpdate()` may add permissions only for - roles the caller holds. An unheld role throws `Exception\Authorization` (`Permissions must be one of: ()`) - and malformed permissions throw `Exception\Structure`, before anything is written. -- While either toggle is on, `updateDocuments()` refuses a related document given as a document with - `Exception\Relationship`. Related documents given by ID still link. + roles the caller holds. The check judges the document the `Hook\Nested` hooks return. An unheld role throws + `Exception\Authorization` (`Permissions must be one of: ()`) and malformed permissions throw + `Exception\Structure`, before anything is written. +- While grants are checked, `updateDocuments()` refuses a related document given as a document with + `Exception\Relationship` (`Invalid relationship value. A bulk update takes related documents by ID while grants are + checked.`). Related documents given by ID still link. - `Database::setRoundFractions(bool $round): static`, `isRoundingFractions(): bool` and `withRoundFractions(bool $round, callable $callback): mixed`, off by default. While on, a fractional change, amount or limit on a single-valued integer or big integer attribute is stored as the adapter's engine stored it on 7.4.1, @@ -40,23 +44,9 @@ - `Utopia\Database\Arithmetic\Rounding`, an enum with the cases `HalfEven`, `HalfAwayFromZero` and `TowardZero`, and the methods `apply(int|string $base, float $fraction): int|string`, `round(float $value): int|string` and `ratio(int|string $numerator, int|string $denominator): int|string`. -- `Utopia\Database\Arithmetic\Decimal`, a number read exactly from its text: - `Decimal::parse(int|float|string $value): ?Decimal`, `floor(): int|string`, `ceil(): int|string`, - `integer(): int|string|null`, `Decimal::fraction(int|float|string $value): ?array`, - `Decimal::isOverlong(int|float|string $value): bool`, `Decimal::isNegative(int|float|string $value): bool`, - `Decimal::isTiny(int|float|string $value): bool` and `Decimal::order(int|float|string $value): ?int`. -- `Utopia\Database\Arithmetic\Fractional`, which resolves fractional amounts and limits from a stored value: - `new Fractional(Arithmetic $arithmetic)`, - `operator(Operator $operator, int|float|string|null $stored): Operator`, - `counter(int|string $stored, int|float|string $value, bool $increase): int|string`, and the static methods - `isFractional(mixed $value): bool`, `isWhole(int|float|string $value): bool`, - `wholeValue(int|float|string $value): int|string|null`, `needsStored(Operator $operator): bool`, - `exceeds(int|string $stored, int|float|string $value, int|float|string $bound, bool $increase): bool`, - `maximum(int|float|string|null $bound): int|float|string|null`, - `minimum(int|float|string|null $bound): int|float|string|null` and `whole(Operator $operator): Operator`. - `Validator\Operator` and `Validator\Structure` (and so `Validator\PartialStructure`) take a trailing `bool $fractions = false`. When it is set, a numeric operator on an integer attribute may carry a finite fractional - amount or limit within the attribute's range, for a caller that resolves it before writing. + amount or limit within the attribute's range, for a caller that resolves it into a whole one before writing. - `Database::resolveCursor(string $collection, string $id, CursorDirection $direction, array $queries = []): Document` returns the cursor that a list with `$queries` pages from when it names a document by ID. The document is read with authorization skipped, and only the cursor fields are returned: `$id`, `$sequence`, `$collection` and @@ -67,15 +57,8 @@ otherwise they are what `find()` returns. Aggregate queries under a permission other than read throw `Exception\Query`. `find()` still refuses aggregate queries. - `ParsedQuery::isAggregation(): bool` tells whether the queries aggregate or group. - -### Changed - -- A join condition that compares the parent side of a one-to-many relationship, or the child side of a two-way - many-to-one relationship, with the `$id` of the related collection is now read through the two-way key the related - documents store. In 8.0 it threw `Exception\Query` (`Cannot join on virtual relationship attribute: `). Every - other join on a relationship side that holds no column is still refused. -- `State\Snapshot` is `@internal`. Its constructor takes three more required arguments, `replacePlaceholders`, - `checkGrants` and `roundFractions`. `Database::snapshot()` fills them in. +- The `State\Snapshot` constructor takes two more trailing arguments, `bool $checkGrants = false` and + `bool $roundFractions = false`. `Database::snapshot()` fills them in. ## 8.0.3 diff --git a/UPGRADE.md b/UPGRADE.md index 589807ccc..ed6b263f9 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -1,65 +1,99 @@ # Upgrading from 8.0 to 8.1 -8.1 is a minor release. Every change in it is additive or opt-in. With none of the new toggles set, a `Database` -behaves exactly as in 8.0.3, with two exceptions: joins on a to-many relationship attribute (see -[Joining on a to-many relationship](#joining-on-a-to-many-relationship)) and the `@internal` `State\Snapshot` -constructor. [CHANGELOG.md](CHANGELOG.md) lists every new signature. +8.1 is a minor release. Every change in it is additive or opt-in. With no `Hook\Nested` hook registered and with +`setCheckGrants()` and `setRoundFractions()` off, which is the default, a `Database` behaves exactly as in 8.0.3. +[CHANGELOG.md](CHANGELOG.md) lists every new signature. -- [Placeholder IDs](#placeholder-ids) +- [Preparing nested related documents](#preparing-nested-related-documents) - [Grant check on related documents](#grant-check-on-related-documents) - [Fractional operands on integer attributes](#fractional-operands-on-integer-attributes) - [Cursor from an ID](#cursor-from-an-id) - [One read for documents and aggregates](#one-read-for-documents-and-aggregates) -- [Joining on a to-many relationship](#joining-on-a-to-many-relationship) - [Spatial support behind a pool](#spatial-support-behind-a-pool) - [`State\Snapshot`](#statesnapshot) -The three new toggles follow the 8.0 toggle idiom (see [Configuration toggles](#configuration-toggles)). Each one is +The two new toggles follow the 8.0 toggle idiom (see [Configuration toggles](#configuration-toggles)). Each one is scoped to the calling coroutine and the coroutines it starts. `withSnapshot()` carries it into relationship writes and other work that runs in another coroutine. | Toggle | Set | Read | Scope | |---|---|---|---| -| Placeholder IDs | `setReplacePlaceholders(bool $replace): static` | `isReplacingPlaceholders(): bool` | `withReplacePlaceholders(bool $replace, callable $callback): mixed` | | Grant check | `setCheckGrants(bool $check): static` | `isCheckingGrants(): bool` | `withCheckGrants(bool $check, callable $callback): mixed` | | Fractions | `setRoundFractions(bool $round): static` | `isRoundingFractions(): bool` | `withRoundFractions(bool $round, callable $callback): mixed` | -## Placeholder IDs +## Preparing nested related documents -Opt-in. By default 8.1 stores a `$id` of `'unique()'` as given, as 8.0 deliberately does: only an empty `$id` asks for -a generated one. `Id::PLACEHOLDER` names the placeholder (`'unique()'`), and `Id::isPlaceholder(string $id): bool` -checks for it. +Additive. 8.1 still never interprets `'unique()'`. A `$id` of `'unique()'` is stored as given, at the top level and +nested, exactly as in 8.0: only an empty `$id` asks for a generated one. An application that gives some IDs of nested +related documents a meaning of its own, such as an ID that asks for a generated one, implements it as a +`Utopia\Database\Hook\Nested` hook: -With `setReplacePlaceholders(true)`, a document whose `$id` is `Id::PLACEHOLDER` gets a generated ID, just as a -document with an empty `$id` does: +```php +interface Nested extends Hook +{ + public function prepare(Document $collection, Document $document): Document; +} +``` -- the documents given to `createDocument()`, `createDocuments()`, `upsertDocument()` and `upsertDocuments()`. An - upsert with a placeholder ID always creates a new document. -- every related document given as a document (a `Document` or an associative array) to those methods and to - `updateDocument()`, as deep as the write relates them. This needs the relationship hook to be attached and enabled. - A related document with an empty `$id` also gets its ID before anything is written, so links and junction rows see - the final ID. +Register it with `$database->addHook($hook)`. Remove it with `removeHook($hook)`, or remove every one with +`removeHook(Nested::class)`. + +- **What it is given.** Each related document that `createDocument()`, `createDocuments()`, `updateDocument()`, + `upsertDocument()`, `upsertDocuments()`, `authorizeCreate()` or `authorizeUpdate()` is given as a document (a + `Document` or an associative array), at every depth the write relates. `$collection` is the collection the related + document belongs to. +- **When it runs.** Before the write checks the document's grants and relates it, a parent before its children. + Several hooks run in the order they were registered, each given the document the previous one returned. +- **What it returns.** It is given a copy, never the caller's document. The write checks and relates the document it + returns, so links and junction rows use the IDs it returns. In `createDocuments()`, `upsertDocument()` and + `upsertDocuments()`, the prepared related documents replace the relationship values of the caller's top-level + documents in place, as those writes prepare the documents they are given. +- **What it is not given.** The top-level document, a related document given as an ID string, and anything from + `updateDocuments()`. A string stays a reference to the document stored under that ID, as in 8.0. +- **Running more than once.** One logical write can run it more than once: `authorizeCreate()` or `authorizeUpdate()` + and then the write, or a retried transaction. A pre-check prepares a copy, and the write prepares the document again. + So the hook must return the same document when it is given its own output. +- **Conditions.** It runs while the relationship hook is attached and enabled, so not inside `skipRelationships()`, + whether or not events are silenced. Its exceptions abort the write before anything is written. +- **New documents.** A related document whose returned ID is not stored is new. Relating it, in the write and in + `authorizeCreate()` and `authorizeUpdate()`, needs create permission on its collection, and with the + [grant check](#grant-check-on-related-documents) on, all of its permissions are checked. +- **Malformed values.** While a hook is registered or grants are checked, a related document given as an associative + array that is not a valid document throws `Exception\Relationship` (`Invalid relationship value. `) before + anything is written. +- **Mirror.** `Mirror::addHook()` registers it on the mirror itself, for its pre-checks, and on the source, never on + the destination. The destination replicates what the source wrote, with the IDs the hook gave there. + `removeHook()` removes it from both. -A related document given as the string `'unique()'` is still a reference to the document stored under that ID: it -links that document, or it is refused as missing, as in 8.0. While the toggle is on, a document stored with the -literal ID `'unique()'` can no longer be created or changed through a nested document. A string reference still links -it. +```php +use Utopia\Database\Document; +use Utopia\Database\Hook\Nested; +use Utopia\Database\Id; -`authorizeCreate()` and `authorizeUpdate()` give the related documents they check the same IDs, on a copy. A -placeholder related document is judged as new, so it needs create permission on its collection. While the toggle is -on, `updateDocuments()` refuses a related document given as a document with `Exception\Relationship`, because one -generated ID cannot serve every document the update changes. Related documents given by ID still link. `Mirror` -forwards the toggle to its source and destination. +final readonly class GeneratedIds implements Nested +{ + public function __construct(private string $marker) + { + } -```php -$database->setReplacePlaceholders(true); + public function prepare(Document $collection, Document $document): Document + { + if ($document->getId() === $this->marker) { + $document->setAttribute(Document::ID, Id::unique()); + } + + return $document; + } +} + +$database->addHook(new GeneratedIds('new')); $database->createDocument('authors', new Document([ - '$id' => Id::PLACEHOLDER, + '$id' => 'ada', 'name' => 'Ada', 'books' => [ - ['$id' => Id::PLACEHOLDER, 'title' => 'Notes'], // created under a generated ID - 'unique()', // links the stored document 'unique()' + ['$id' => 'new', 'title' => 'Notes'], // created under a generated ID + 'new', // still a reference to the document stored as 'new' ], ])); ``` @@ -70,19 +104,22 @@ Opt-in. By default 8.1 never compares the permissions a write grants with the ro With `setCheckGrants(true)`, the check covers related documents written through a parent: every related document given as a document to `createDocument()`, `createDocuments()`, `updateDocument()`, `upsertDocument()`, -`upsertDocuments()`, `authorizeCreate()` or `authorizeUpdate()`, as deep as the write relates them. The document the -write is given is not checked. For each related document whose `$permissions` is not null: +`upsertDocuments()`, `authorizeCreate()` or `authorizeUpdate()`, as deep as the write relates them, each as the +[`Hook\Nested` hooks](#preparing-nested-related-documents) return it. The document the write is given is not checked. +For each related document whose `$permissions` is not null: - Malformed permissions throw `Exception\Structure` with the `Validator\Permissions` description. - The permissions it adds must name roles the caller holds (`Authorization::hasRole()`). Otherwise the write throws `Exception\Authorization` (`Permissions must be one of: ()`). The added permissions are - `Permission::aggregate()` of the sent permissions minus those the document already has. A new document has none. - Permissions sent back unchanged pass, and so does an empty list. + `Permission::aggregate()` of the sent permissions minus those the document already has. A document that is not + stored yet, with an empty `$id` or an ID no stored document has, has none. Permissions sent back unchanged pass, + and so does an empty list. - Permissions a related document inherits from its parent are never checked. - Both refusals happen before anything is written. -The check is skipped while authorization is disabled (`$authorization->skip()` or `setStatus(false)`). To write -without it while authorization stays enabled, use `withCheckGrants(false, $callback)`. +The check is skipped while authorization is disabled (`$authorization->skip()` or `setStatus(false)`), and it runs only +while the relationship hook is attached and enabled. To write without it while authorization stays enabled, use +`withCheckGrants(false, $callback)`. A related document's current permissions are read whether or not the caller can read the document: @@ -94,9 +131,12 @@ A related document's current permissions are read whether or not the caller can and from the stored document when it returns `null`. A document `$resolve` returns empty (staged for deletion) is new. -While the toggle is on, `updateDocuments()` refuses a related document given as a document with -`Exception\Relationship`, because it is not checked there. `Mirror` checks on its source only. The destination replays -what the source allowed with the check off, so a destination that is behind never refuses it. +While grants are checked, `updateDocuments()` refuses a related document given as a document with +`Exception\Relationship` (`Invalid relationship value. A bulk update takes related documents by ID while grants are +checked.`), because it is not checked there. Related documents given by ID still link. + +`Mirror` checks on its source only. The destination replays what the source allowed with the check off, so a +destination that is behind never refuses it. Because the current permissions are read regardless of the caller's read permission, the error can reveal one fact to a caller who cannot read a related document: whether a permission it sends is already on that document. If it is, this @@ -182,8 +222,7 @@ held. 8.1 does not reproduce this: limits hold as on Memory and Redis. `Validator\Operator` and `Validator\Structure` (and `Validator\PartialStructure`) take a trailing `bool $fractions = false` for code that validates operators itself. When it is set, a finite fractional amount or limit -within the attribute's range passes, and the caller resolves it before writing, for example with -`Arithmetic\Fractional::operator()`. +within the attribute's range passes, and the caller resolves it into a whole one before writing. ```php $database->setRoundFractions(true); @@ -229,34 +268,6 @@ PermissionType::Read): array`. It returns a list of `Document`: `ParsedQuery::isAggregation(): bool` (from `Query::groupByType($queries)`) tells which of the two a list is. -## Joining on a to-many relationship - -Changed for joins that 8.0 refused. Nothing else changes. The parent side of a one-to-many relationship, and the child -side of a two-way many-to-one relationship, hold no column: the related documents store this document's ID under the -two-way key. 8.0 refused a join on such an attribute (`Cannot join on virtual relationship attribute: `). 8.1 -reads it when all of the following hold: - -- the condition is `Query::on(, '$id')` or `Query::on(, '.$id')` in the - join's ON list; -- the joined collection is the relationship's related collection; -- the relationship has a two-way key. - -The condition then compares this document's `$id` with `.`, keeping its operator and alias. The -caller's queries are not changed. `find()`, `count()`, `sum()`, `aggregate()`, `getDocument()` and `resolveCursor()` -all do this. - -The parent side of a many-to-one, one-to-one and many-to-many relationships, a one-way many-to-one child side, a join -of another collection and a column of an earlier join are refused as in 8.0. The read rule for joined collections is -unchanged (see [Joins](#joins)). - -```php -// authors.books is the parent side of a one-to-many relationship with books.author -$database->find('authors', [ - Query::join('books', 'b', [Query::on('books', '$id')]), // reads as $id = b.author - Query::select(['name', 'b.title']), -]); -``` - ## Spatial support behind a pool No change. 8.0.3 already answers correctly, and 8.1 adds tests for it. Ask @@ -267,9 +278,9 @@ source. Creating a `point` attribute fails exactly when `hasFeature(Feature\Spat ## `State\Snapshot` -`State\Snapshot` is `@internal`, as `Database::snapshot()` and `withSnapshot()` already were. Its constructor takes -three more required arguments, `replacePlaceholders`, `checkGrants` and `roundFractions`, which `Database::snapshot()` -fills in. Code that builds a `Snapshot` itself has to pass them. +Additive. The `State\Snapshot` constructor takes two more trailing arguments, `bool $checkGrants = false` and +`bool $roundFractions = false`, which `Database::snapshot()` fills in. Code that builds a `Snapshot` itself still +works; a snapshot built without them runs its callback with both toggles off. # Upgrading from 7.x to 8.0 From cd3105473166f80ab9f16cc7af8fe1202a6f1e95 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 19:37:02 +1300 Subject: [PATCH 48/55] fix(mirror): register a nested hook only in the role the mirror missed Database::addHook() registers an object under every hook interface it implements, so the unconditional source->addHook() for Nested hooks re-registered the hook's other roles. A Lifecycle+Nested hook ended up on the source twice (every lifecycle event handled twice, prepare() run twice per related document) and never on the mirror, so the mirror's own authorizeCreate()/authorizeUpdate() pre-checks skipped it; a Write+Nested hook was appended twice to the adapter the mirror shares with the source; an Invalidator+Nested hook was prepared twice on the source. Each branch already registers the hook fully on one side, so the Nested role is now added only to the side that branch skipped: the mirror for lifecycle hooks (registered on the source), the source for everything else except invalidators (already on both). Hooks that are not Nested register exactly as before, and removeHook() already strips every list on both sides. Co-Authored-By: Claude Opus 5.5 --- src/Database/Mirror.php | 10 ++- tests/unit/MirrorReplicationTest.php | 98 ++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 3 deletions(-) diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php index 62afb5ec4..f54529a94 100644 --- a/src/Database/Mirror.php +++ b/src/Database/Mirror.php @@ -1860,12 +1860,16 @@ public function addHook(\Utopia\Query\Hook $hook): static $this->source->addHook($hook); } elseif ($hook instanceof Lifecycle) { $this->addLifecycleHook($hook); + + if ($hook instanceof Nested) { + $this->nestedHooks[] = $hook; + } } else { parent::addHook($hook); - } - if ($hook instanceof Nested) { - $this->source->addHook($hook); + if ($hook instanceof Nested) { + $this->source->nestedHooks[] = $hook; + } } if ($hook instanceof Relationships) { diff --git a/tests/unit/MirrorReplicationTest.php b/tests/unit/MirrorReplicationTest.php index cd62d884b..4fd012a3d 100644 --- a/tests/unit/MirrorReplicationTest.php +++ b/tests/unit/MirrorReplicationTest.php @@ -23,9 +23,14 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Event; +use Utopia\Database\Event\Document\Created; +use Utopia\Database\Event\Domain; use Utopia\Database\Exception as DatabaseException; +use Utopia\Database\Hook\Interceptor; +use Utopia\Database\Hook\Lifecycle; use Utopia\Database\Hook\Nested; use Utopia\Database\Hook\Relationships; +use Utopia\Database\Hook\WriteContext; use Utopia\Database\Id; use Utopia\Database\Index; use Utopia\Database\Mirror; @@ -493,6 +498,99 @@ public function prepare(Document $collection, Document $document): Document } } + public function testALifecycleHookThatIsAlsoNestedRunsOnceInEachRole(): void + { + $this->relate(); + $hook = new class () implements Lifecycle, Nested { + /** + * @var list + */ + public array $created = []; + + /** + * @var list + */ + public array $prepared = []; + + #[\Override] + public function handle(Domain $event): void + { + if ($event instanceof Created) { + $this->created[] = [$event->collection, $event->document->getId()]; + } + } + + #[\Override] + public function prepare(Document $collection, Document $document): Document + { + $this->prepared[] = $document->getId(); + + return $document; + } + }; + $this->mirror->addHook($hook); + $parent = static fn (string $id): Document => new Document([ + Document::ID => $id, + 'title' => $id, + 'children' => [new Document([Document::ID => $id.'-child', 'title' => $id.' child'])], + ]); + + $this->mirror->authorizeCreate(self::PARENTS, $parent('checked')); + $this->assertSame(['checked-child'], $hook->prepared); + + $this->mirror->createDocument(self::PARENTS, $parent('created')); + + $this->assertSame([], $this->errors); + $this->assertSame(['checked-child', 'created-child'], $hook->prepared); + $this->assertSame([[self::PARENTS, 'created']], $hook->created); + $this->assertSame('created-child', $this->childTitled($this->destination, 'created child')->getId()); + } + + public function testAWriteHookThatIsAlsoNestedRunsOnceInEachRole(): void + { + $this->relate(); + $hook = new class () extends Interceptor implements Nested { + /** + * @var list> + */ + public array $created = []; + + /** + * @var list + */ + public array $prepared = []; + + #[\Override] + public function afterDocumentCreate(string $collection, array $documents, WriteContext $context): void + { + $this->created[] = \array_values(\array_map(static fn (Document $document): string => $document->getId(), $documents)); + } + + #[\Override] + public function prepare(Document $collection, Document $document): Document + { + $this->prepared[] = $document->getId(); + + return $document; + } + }; + $this->mirror->addHook($hook); + + $this->mirror->createDocument(self::NOTES, new Document([Document::ID => 'note', 'title' => 'note'])); + $this->mirror->createDocument(self::PARENTS, new Document([ + Document::ID => 'parent', + 'title' => 'parent', + 'children' => [new Document([Document::ID => 'child', 'title' => 'child'])], + ])); + + $this->assertSame([], $this->errors); + $this->assertSame(['child'], $hook->prepared); + $this->assertSame([['note'], ['note']], \array_values(\array_filter( + $hook->created, + static fn (array $ids): bool => $ids === ['note'], + )), 'one call from the source adapter and one from the destination adapter'); + } + /** * @return iterable */ From c9c0966a38891f41d7133674a2fb06a5e4a11cee Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 19:49:23 +1300 Subject: [PATCH 49/55] docs: describe the nested hook without appwrite's placeholder The 'unique()' placeholder is an appwrite feature, not a library one, so the 8.1.0 notes should not frame Hook\Nested around it. The hook is a generic extension point for rewriting nested related documents before they are written, so the upgrade guide now shows it filling in a default attribute instead of generating IDs. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +-- UPGRADE.md | 23 ++++++++++------------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5396e7a8..67ed9d968 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,8 +15,7 @@ and with `setCheckGrants()` and `setRoundFractions()` off, which is the default, prepared before its children, and hooks run in the order they were registered. The write checks grants on, and relates, the document the hook returns. The hook is not given the top-level document, a related document given as an ID, or anything from `updateDocuments()`. A `Mirror` runs it on itself, for the pre-checks, and on its source. - Its destination receives the IDs the source wrote. The library still never interprets `'unique()'`: an ID is stored - as given, as in 8.0. + Its destination receives the IDs the source wrote. - `Database::setCheckGrants(bool $check): static`, `isCheckingGrants(): bool` and `withCheckGrants(bool $check, callable $callback): mixed`, off by default. While on and while authorization is enabled, a related document given as a document to `createDocument()`, `createDocuments()`, `updateDocument()`, diff --git a/UPGRADE.md b/UPGRADE.md index ed6b263f9..7293f977f 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -23,10 +23,8 @@ other work that runs in another coroutine. ## Preparing nested related documents -Additive. 8.1 still never interprets `'unique()'`. A `$id` of `'unique()'` is stored as given, at the top level and -nested, exactly as in 8.0: only an empty `$id` asks for a generated one. An application that gives some IDs of nested -related documents a meaning of its own, such as an ID that asks for a generated one, implements it as a -`Utopia\Database\Hook\Nested` hook: +Additive. An application that rewrites nested related documents before they are written, such as filling in an +attribute the caller left out, implements a `Utopia\Database\Hook\Nested` hook: ```php interface Nested extends Hook @@ -62,38 +60,37 @@ Register it with `$database->addHook($hook)`. Remove it with `removeHook($hook)` array that is not a valid document throws `Exception\Relationship` (`Invalid relationship value. `) before anything is written. - **Mirror.** `Mirror::addHook()` registers it on the mirror itself, for its pre-checks, and on the source, never on - the destination. The destination replicates what the source wrote, with the IDs the hook gave there. + the destination. The destination replicates what the source wrote, as the hook returned it there. `removeHook()` removes it from both. ```php use Utopia\Database\Document; use Utopia\Database\Hook\Nested; -use Utopia\Database\Id; -final readonly class GeneratedIds implements Nested +final readonly class DefaultStatus implements Nested { - public function __construct(private string $marker) + public function __construct(private string $status) { } public function prepare(Document $collection, Document $document): Document { - if ($document->getId() === $this->marker) { - $document->setAttribute(Document::ID, Id::unique()); + if ($collection->getId() === 'books' && $document->getAttribute('status') === null) { + $document->setAttribute('status', $this->status); } return $document; } } -$database->addHook(new GeneratedIds('new')); +$database->addHook(new DefaultStatus('draft')); $database->createDocument('authors', new Document([ '$id' => 'ada', 'name' => 'Ada', 'books' => [ - ['$id' => 'new', 'title' => 'Notes'], // created under a generated ID - 'new', // still a reference to the document stored as 'new' + ['$id' => 'notes', 'title' => 'Notes'], // stored with status 'draft' + 'sketches', // a reference to a stored book, not given to the hook ], ])); ``` From 484beb2c727f9752f2ddfa01b0539971ea2261ce Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 20:12:50 +1300 Subject: [PATCH 50/55] fix(relationships): judge a hook-assigned related id new in the grant check With grants checked, a related document whose ID a Hook\Nested hook assigned was judged by the permissions stored (or staged) under that ID, read with getDocument(forUpdate: true) inside createDocument() and updateDocument(). The ID is not stored yet, so the locking read only took a gap lock on the missing row, which serialises concurrent writes that assign fresh IDs and can deadlock them, for no benefit: such a document has no permissions to keep. When the hooks change a document's ID it is now new to the grant check, so every permission it sends is checked and nothing is read, locked or resolved for it. Documents whose ID the hooks keep, and every write with no hook registered, behave as before; the create-permission pre-checks already judged a hook-assigned ID that is not stored as new. The tests prove the absence of the read behaviourally: a hook that maps a marker onto an ID stored with a role the caller does not hold used to pass when the same permissions were sent back, because they were read as current, and is now refused. Co-Authored-By: Claude Opus 5.5 --- UPGRADE.md | 2 + src/Database/Hook/Relationships.php | 11 +-- .../unit/Relationships/NestedPreCheckTest.php | 68 +++++++++++++++++++ .../Relationships/NestedPreparationTest.php | 30 ++++++++ 4 files changed, 107 insertions(+), 4 deletions(-) diff --git a/UPGRADE.md b/UPGRADE.md index 7293f977f..c75e198f5 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -127,6 +127,8 @@ A related document's current permissions are read whether or not the caller can - `authorizeCreate()` and `authorizeUpdate()` take them from `$resolve($collection, $id)` when it returns a document, and from the stored document when it returns `null`. A document `$resolve` returns empty (staged for deletion) is new. +- A related document whose `$id` a `Hook\Nested` hook assigned is new to this check: none of its permissions are read, + locked or resolved, so all of them are checked. While grants are checked, `updateDocuments()` refuses a related document given as a document with `Exception\Relationship` (`Invalid relationship value. A bulk update takes related documents by ID while grants are diff --git a/src/Database/Hook/Relationships.php b/src/Database/Hook/Relationships.php index 2fe24abe0..2d15ccc68 100644 --- a/src/Database/Hook/Relationships.php +++ b/src/Database/Hook/Relationships.php @@ -3170,9 +3170,10 @@ private function authorizeLink(Document $collection, Document $related): void * Prepare the related documents $document holds as documents, as deep as the write relates them, replacing its * relationship values with prepared copies so the caller's related documents are not changed. Each copy is first * given to the {@see Nested} hooks, in order, parent before children, and the document they return is the one - * whose grants are checked and that is related. A related document that does not exist yet has no permissions - * to keep, so all of its permissions are checked; any other keeps those of its staged view, else of the stored - * one. A write nested in another relationship write is left as given: the outermost write prepared it. + * whose grants are checked and that is related. A related document that does not exist yet, or whose ID a hook + * assigned, has no permissions to keep, so all of its permissions are checked without reading any; any other + * keeps those of its staged view, else of the stored one. A write nested in another relationship write is left + * as given: the outermost write prepared it. * * @throws AuthorizationException When a related document adds a permission for a role the caller does not hold * @throws RelationshipException When a related document given as an array is not a valid document @@ -3250,12 +3251,14 @@ private function prepareNestedValues(Document $collection, Document $document, a */ private function prepareNestedDocument(Document $collection, Document $document, array $writeStack, Preparation $preparation): Document { + $given = $document->getId(); foreach ($preparation->hooks as $hook) { $document = $hook->prepare($collection, $document); } if ($preparation->check) { - $this->authorizeGrants($collection, $document, $document->getId() === '', $preparation); + $id = $document->getId(); + $this->authorizeGrants($collection, $document, $id === '' || $id !== $given, $preparation); } foreach ($this->prepareNestedValues($collection, $document, $writeStack, $preparation) as $key => $value) { diff --git a/tests/unit/Relationships/NestedPreCheckTest.php b/tests/unit/Relationships/NestedPreCheckTest.php index 51b412994..cd8a5be73 100644 --- a/tests/unit/Relationships/NestedPreCheckTest.php +++ b/tests/unit/Relationships/NestedPreCheckTest.php @@ -220,6 +220,54 @@ public function testTheGrantCheckRunsAfterTheRewrite(Closure $adapter, Relations $this->assertSame([], $this->relatedNames()); } + /** + * @param Closure(): Adapter $adapter + */ + #[DataProvider('types')] + public function testAnIdTheHookAssignsIsJudgedNewWithoutTheStoredOrStagedPermissions(Closure $adapter, RelationshipType $type): void + { + $foreign = [Permission::read(Role::user('u2'))]; + $this->seed($adapter(), $type, [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], $foreign); + $this->database->setCheckGrants(true)->addHook($this->assigning(self::STORED)); + $staged = static fn (string $collection, string $id): ?Document => $collection === self::RELATED && $id === self::STORED + ? new Document([Document::ID => self::STORED, Document::COLLECTION => self::RELATED, 'name' => 'Stored', Document::PERMISSIONS => $foreign]) + : null; + $kept = $this->parent($type, new Document([Document::ID => self::STORED, 'name' => 'Stored', Document::PERMISSIONS => $foreign])); + $assigned = $this->parent($type, new Document([Document::ID => self::MARKER, 'name' => 'Stored', Document::PERMISSIONS => $foreign])); + + $this->assertNull($this->refusal(fn () => $this->database->authorizeCreate('parents', $kept)), 'An ID the hook keeps is judged by its stored permissions'); + $this->assertNull($this->refusal(fn () => $this->database->authorizeCreate('parents', $kept, $staged)), 'An ID the hook keeps is judged by its staged permissions'); + $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $this->database->authorizeCreate('parents', $assigned)), 'The stored permissions of an ID the hook assigns are not read'); + $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $this->database->authorizeCreate('parents', $assigned, $staged)), 'The staged permissions of an ID the hook assigns are not resolved'); + $this->assertSame(self::UNHELD_REFUSAL, $this->refusal(fn () => $this->database->authorizeUpdate('parents', $this->storedParent(), new Document([self::RELATED => $assigned->getAttribute(self::RELATED)]), $staged))); + $this->assertSame([], $this->relatedNames()); + } + + /** + * @param Closure(): Adapter $adapter + */ + #[DataProvider('types')] + public function testAnIdTheHookAssignsNeedsCreateWhileGrantsAreChecked(Closure $adapter, RelationshipType $type): void + { + $held = [Permission::read(Role::user(self::USER))]; + $related = new Document([Document::ID => self::MARKER, 'name' => 'Created', Document::PERMISSIONS => $held]); + + $this->chain($adapter(), $type, [Permission::read(Role::any()), Permission::update(Role::any())]); + $this->database->setCheckGrants(true)->addHook($this->marker()); + $this->assertSame(self::NO_CREATE, $this->refusal(fn () => $this->database->authorizeCreate('parents', $this->parent($type, $related)))); + $this->assertSame(self::NO_CREATE, $this->refusal(fn () => $this->database->createDocument('parents', $this->parent($type, $related)))); + $this->assertSame([], $this->relatedNames()); + + $this->chain($adapter(), $type, [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())]); + $this->database->setCheckGrants(true)->addHook($this->marker()); + $this->assertNull($this->refusal(fn () => $this->database->authorizeCreate('parents', $this->parent($type, $related)))); + $created = self::ids($this->database->createDocument('parents', $this->parent($type, $related))->getAttribute(self::RELATED)); + $this->assertCount(1, $created); + $this->assertNotSame(self::MARKER, $created[0]); + $this->assertSame($held, $this->authorization->skip(fn (): Document => $this->database->getDocument(self::RELATED, $created[0]))->getPermissions()); + $this->assertSame(['Created'], $this->relatedNames()); + } + /** * @param list $collectionPermissions * @param list|null $storedPermissions The related document's permissions; null stores none @@ -330,6 +378,26 @@ public function prepare(Document $collection, Document $document): Document }; } + /** + * A hook that replaces a MARKER ID with $id. + */ + private function assigning(string $id): Nested + { + return new class (self::MARKER, $id) implements Nested { + public function __construct( + private readonly string $marker, + private readonly string $id, + ) { + } + + #[\Override] + public function prepare(Document $collection, Document $document): Document + { + return $document->getId() === $this->marker ? $document->setAttribute(Document::ID, $this->id) : $document; + } + }; + } + private function record(string $collection, string $id): void { $this->seen[] = [$collection, $id]; diff --git a/tests/unit/Relationships/NestedPreparationTest.php b/tests/unit/Relationships/NestedPreparationTest.php index 3b5fb0730..90902c0c5 100644 --- a/tests/unit/Relationships/NestedPreparationTest.php +++ b/tests/unit/Relationships/NestedPreparationTest.php @@ -446,6 +446,36 @@ public function testTheGrantCheckJudgesTheRewrittenDocument(Closure $adapter, Re $this->assertSame($foreign, $this->authorization->skip(fn (): Document => $database->getDocument(self::SECOND, self::MARKER))->getPermissions()); } + /** + * @param Closure(): Adapter $adapter + */ + #[DataProvider('types')] + public function testAnIdTheHookAssignsIsJudgedNewWithoutReadingTheStoredPermissions(Closure $adapter, RelationshipType $type): void + { + $database = $this->twoWayDatabase($adapter(), $type); + $foreign = [Permission::read(Role::user('u2'))]; + $database->createDocument(self::SECOND, new Document([Document::ID => 'stored', 'name' => 'stored', Document::PERMISSIONS => $foreign])); + $database->createDocument(self::FIRST, new Document([Document::ID => 'updated', 'name' => 'updated'])); + $database->setCheckGrants(true)->addHook($this->hook(static fn (string $id): string => $id === self::MARKER ? 'stored' : $id)); + $child = static fn (string $id): Document => new Document([Document::ID => $id, 'name' => 'stored', Document::PERMISSIONS => $foreign]); + + $this->assertSame( + self::UNHELD_REFUSAL, + $this->refusal(fn () => $database->createDocument(self::FIRST, new Document([Document::ID => 'created', 'name' => 'created', self::NEXT => self::value($type, $child(self::MARKER))]))), + 'The stored permissions of an ID the hook assigns are not read, so sending them back grants them anew', + ); + $this->assertSame( + self::UNHELD_REFUSAL, + $this->refusal(fn () => $database->updateDocument(self::FIRST, 'updated', new Document([self::NEXT => self::value($type, $child(self::MARKER))]))), + ); + $this->assertSame(['updated'], self::names($database, self::FIRST)); + $this->assertSame([], self::ids($database->getDocument(self::FIRST, 'updated')->getAttribute(self::NEXT))); + + $database->createDocument(self::FIRST, new Document([Document::ID => 'kept', 'name' => 'kept', self::NEXT => self::value($type, $child('stored'))])); + $this->assertSame(['stored'], self::ids($database->getDocument(self::FIRST, 'kept')->getAttribute(self::NEXT)), 'An ID the hook keeps is judged by its stored permissions'); + $this->assertSame($foreign, $this->authorization->skip(fn (): Document => $database->getDocument(self::SECOND, 'stored'))->getPermissions()); + } + /** * @param Closure(): Adapter $adapter */ From 8585e2e5a17b341571168b87ffb4bb07f534be2c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 20:19:25 +1300 Subject: [PATCH 51/55] feat(joins): join on the parent side of a one-to-many relationship The parent side of a one-to-many relationship holds no column, so 8.0.3 refused a join condition such as on('orders', '$id') and appwrite had to rewrite it before calling the library. The library now resolves it the way appwrite's Joins::resolveColumns() did: the condition compares the main $id with the relationship's two-way key in the joined collection, keeping its operator. getDocument(), find(), aggregate(), read(), count(), sum() and resolveCursor() resolve it on copies of the caller's joins before validation, so authorization, tenancy and cursor paging apply to the resolved join exactly as to any other. Only the parent side is resolved: appwrite's check matched the child side too, which holds a column and already joins, so following it would break a join that works on 8.0.3. Every other relationship type is still refused. JoinInternalColumnsTest's refusal of on('books', '$id') becomes the joined rows, the one behaviour the change is meant to alter. Co-Authored-By: Claude Opus 5.5 --- src/Database/Trait/Cursors.php | 4 +- src/Database/Trait/Documents.php | 69 +++ tests/e2e/Adapter/Base.php | 2 + .../Adapter/Scopes/JoinRelationshipTests.php | 456 ++++++++++++++++++ tests/unit/Joins/JoinInternalColumnsTest.php | 13 +- tests/unit/Joins/RelationshipJoinTest.php | 418 ++++++++++++++++ 6 files changed, 957 insertions(+), 5 deletions(-) create mode 100644 tests/e2e/Adapter/Scopes/JoinRelationshipTests.php create mode 100644 tests/unit/Joins/RelationshipJoinTest.php diff --git a/src/Database/Trait/Cursors.php b/src/Database/Trait/Cursors.php index 855232c12..b16a26ce4 100644 --- a/src/Database/Trait/Cursors.php +++ b/src/Database/Trait/Cursors.php @@ -49,10 +49,10 @@ public function resolveCursor(string $collection, string $id, CursorDirection $d $definition = $this->silent(fn (): Collection => $this->getCollection($collection)); $this->checkQueryTypes($queries); - $queries = \array_values(\array_filter( + $queries = self::resolveOneToManyJoins($definition, \array_values(\array_filter( $queries, static fn (Query $query): bool => $query->getMethod() !== Method::CursorAfter && $query->getMethod() !== Method::CursorBefore, - )); + ))); $joinedById = $this->resolveJoinedCollections($queries); if ($this->validation()->get()) { diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index fb5b44c6c..8aebd8144 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -495,6 +495,7 @@ private function readDocument(Collection $collection, string $id, array $queries $attributes = $collection->attributes(); $this->checkQueryTypes($queries); + $queries = self::resolveOneToManyJoins($collection, $queries); $joinedCollections = null; @@ -3994,6 +3995,7 @@ private function fetch(Collection $collection, array $queries, PermissionType $f $queryCacheQueries = $queries; $this->checkQueryTypes($queries); + $queries = self::resolveOneToManyJoins($collection, $queries); $joinedCollectionsById = null; @@ -4364,6 +4366,7 @@ public function count(string $collection, array $queries = [], ?int $max = null) $collection = $this->silent(fn () => $this->getCollection($collection)); $this->checkQueryTypes($queries); + $queries = self::resolveOneToManyJoins($collection, $queries); $joinedCollections = null; @@ -4420,6 +4423,7 @@ public function sum(string $collection, string $attribute, array $queries = [], $collection = $this->silent(fn () => $this->getCollection($collection)); $this->checkQueryTypes($queries); + $queries = self::resolveOneToManyJoins($collection, $queries); $joinedCollections = null; @@ -5081,6 +5085,71 @@ private function resolveJoinedCollections(array $queries): array return $collections; } + /** + * The parent side of a one-to-many relationship holds no column: each child stores the parent's ID under the + * relationship's two-way key. A join condition whose left column is such an attribute compares the main `$id` + * with that key instead, keeping its operator. The joins returned are copies; the caller's are left as given. + * + * @param array $queries + * @return array + */ + private static function resolveOneToManyJoins(Collection $collection, array $queries): array + { + $twoWayKeys = null; + + foreach ($queries as $index => $query) { + if (! $query->getMethod()->isJoin()) { + continue; + } + + $twoWayKeys ??= self::oneToManyTwoWayKeys($collection); + if ($twoWayKeys === []) { + return $queries; + } + + $resolved = null; + foreach ($query->getJoinOnQueries() as $position => $condition) { + $values = $condition->getValues(); + $left = $values[0] ?? null; + if ($condition->getMethod() !== Method::On || ! \is_string($left) || ! isset($twoWayKeys[$left])) { + continue; + } + + $values[0] = Document::ID; + $values[2] = $twoWayKeys[$left]; + $resolved ??= clone $query; + $resolved->getJoinOnQueries()[$position]->setValues($values); + } + + if ($resolved !== null) { + $queries[$index] = $resolved; + } + } + + return $queries; + } + + /** + * @return array The two-way key of each one-to-many relationship the collection is the parent of + */ + private static function oneToManyTwoWayKeys(Collection $collection): array + { + $twoWayKeys = []; + foreach (self::relationshipAttributes($collection) as $attribute) { + $relationship = $attribute->relationship; + if ( + $relationship?->type === RelationshipType::OneToMany + && $attribute->side === RelationshipSide::Parent + && $relationship->twoWayKey !== null + && $relationship->twoWayKey !== '' + ) { + $twoWayKeys[$attribute->key] = $relationship->twoWayKey; + } + } + + return $twoWayKeys; + } + /** * @param array $queries * @param list $relationships diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index e8820d99b..fe70bc2b2 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -17,6 +17,7 @@ use Tests\E2E\Adapter\Scopes\IndexTests; use Tests\E2E\Adapter\Scopes\JoinComboTests; use Tests\E2E\Adapter\Scopes\JoinReadRuleTests; +use Tests\E2E\Adapter\Scopes\JoinRelationshipTests; use Tests\E2E\Adapter\Scopes\JoinTests; use Tests\E2E\Adapter\Scopes\MetadataCacheTests; use Tests\E2E\Adapter\Scopes\ObjectAttributeTests; @@ -61,6 +62,7 @@ abstract class Base extends TestCase use IndexTests; use JoinComboTests; use JoinReadRuleTests; + use JoinRelationshipTests; use JoinTests; use MetadataCacheTests; use ObjectAttributeTests; diff --git a/tests/e2e/Adapter/Scopes/JoinRelationshipTests.php b/tests/e2e/Adapter/Scopes/JoinRelationshipTests.php new file mode 100644 index 000000000..be7ccc011 --- /dev/null +++ b/tests/e2e/Adapter/Scopes/JoinRelationshipTests.php @@ -0,0 +1,456 @@ +getDatabase(); + if (! $this->supportsJoinRelationships($database)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $this->seedJoinRelationships($database); + + try { + $this->assertSame( + ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4', 'c3/-'], + $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, [Query::leftJoin(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id')])], 'purchase'), + 'a left join keeps the customer without orders', + ); + $this->assertSame( + ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], + $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, [Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', 'purchase.$id')])], 'purchase'), + ); + $this->assertSame( + ['c1/o3', 'c1/o4', 'c2/o1', 'c2/o2', 'c3/o1', 'c3/o2', 'c3/o3', 'c3/o4'], + $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, [Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id', '!=')])], 'purchase'), + 'the operator of the condition is kept', + ); + $this->assertSame( + ['c1/o1', 'c2/o4'], + $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, [Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [ + Query::on('orders', '$id'), + Query::on('region', 'region'), + Query::equal('purchase.status', ['paid']), + ])], 'purchase'), + 'the other conditions of the ON list compare what they named', + ); + + $rows = $database->find(self::JOIN_RELATIONSHIP_CUSTOMERS, [ + Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id')]), + Query::select(['name', 'purchase.amount']), + Query::orderAsc('purchase.amount'), + ]); + $this->assertSame([10, 20, 30, 40], \array_map(static fn (Document $row): ?int => self::joinRelationshipInteger($row->getAttribute('purchase.amount')), $rows)); + $this->assertSame(['Ada', 'Ada', 'Bob', 'Bob'], \array_map(static fn (Document $row): mixed => $row->getAttribute('name'), $rows)); + } finally { + $this->deleteJoinRelationshipCollections($database); + } + } + + public function testJoinRelationshipKeepsEveryOtherRelationshipAsBefore(): void + { + $database = $this->getDatabase(); + if (! $this->supportsJoinRelationships($database)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $this->seedJoinRelationships($database); + + try { + $this->assertSame( + ['o1/c1', 'o2/c1', 'o3/c2', 'o4/c2'], + $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_ORDERS, [Query::join(self::JOIN_RELATIONSHIP_CUSTOMERS, 'buyer', [Query::on('customer', '$id')])], 'buyer'), + 'the child side of a one-to-many holds the parent ID in its own column', + ); + $this->assertSame( + ['c1/r1', 'c2/r2', 'c3/r1'], + $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, [Query::join(self::JOIN_RELATIONSHIP_REGIONS, 'area', [Query::on('region', '$id')])], 'area'), + 'the parent side of a many-to-one holds the related ID in its own column', + ); + + try { + $database->find(self::JOIN_RELATIONSHIP_REGIONS, [Query::join(self::JOIN_RELATIONSHIP_CUSTOMERS, 'resident', [Query::on('customers', '$id')])]); + $this->fail('the child side of a many-to-one has no column to join on'); + } catch (QueryException $exception) { + $this->assertStringContainsString('Cannot join on virtual relationship attribute: customers', $exception->getMessage()); + } + } finally { + $this->deleteJoinRelationshipCollections($database); + } + } + + public function testJoinRelationshipIsMatchedByEveryRead(): void + { + $database = $this->getDatabase(); + if (! $this->supportsJoinRelationships($database)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $this->seedJoinRelationships($database); + + try { + $join = Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id')]); + $leftJoin = Query::leftJoin(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id')]); + + $this->assertSame(4, $database->count(self::JOIN_RELATIONSHIP_CUSTOMERS, [$join])); + $this->assertSame(5, $database->count(self::JOIN_RELATIONSHIP_CUSTOMERS, [$leftJoin])); + $this->assertSame(100, self::joinRelationshipInteger($database->sum(self::JOIN_RELATIONSHIP_CUSTOMERS, 'purchase.amount', [$join]))); + $this->assertSame( + [['Ada', 30], ['Bob', 70]], + \array_map( + static fn (array $row): array => [$row['name'], self::joinRelationshipInteger($row['total'])], + $database->aggregate(self::JOIN_RELATIONSHIP_CUSTOMERS, [$join, Query::sum('purchase.amount', 'total'), Query::groupBy(['name']), Query::orderAsc('name')]), + ), + ); + $this->assertSame( + [4], + \array_map( + static fn (Document $row): ?int => self::joinRelationshipInteger($row->getAttribute('rows')), + $database->read(self::JOIN_RELATIONSHIP_CUSTOMERS, [$join, Query::count('*', 'rows')]), + ), + ); + $this->assertSame( + 20, + self::joinRelationshipInteger($database->getDocument(self::JOIN_RELATIONSHIP_CUSTOMERS, 'c1', [ + Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id'), Query::equal('purchase.status', ['open'])]), + Query::select(['name', 'purchase.amount']), + ])->getAttribute('purchase.amount')), + ); + $this->assertSame(['orders', '=', '$id'], $join->getJoinOnQueries()[0]->getValues(), 'the caller\'s join is left as given'); + } finally { + $this->deleteJoinRelationshipCollections($database); + } + } + + public function testJoinRelationshipReadsOnlyTheJoinedRowsTheCallerCanRead(): void + { + $database = $this->getDatabase(); + if (! $this->supportsJoinRelationships($database)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $this->seedJoinRelationships($database); + + try { + $authorization = $database->getAuthorization(); + $database->updateCollection(self::JOIN_RELATIONSHIP_ORDERS, new CollectionUpdate(permissions: [Permission::create(Role::any())], documentSecurity: true)); + $authorization->skip(fn (): Document => $database->updateDocument(self::JOIN_RELATIONSHIP_ORDERS, 'o2', new Document(['$permissions' => [Permission::read(Role::user('bob'))]]))); + $queries = [Query::leftJoin(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id')])]; + + $this->withJoinRelationshipRoles($database, [Role::any()->toString()], function () use ($database, $queries): void { + $this->assertSame(['c1/o1', 'c2/o3', 'c2/o4', 'c3/-'], $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, $queries, 'purchase')); + $this->assertSame(80, self::joinRelationshipInteger($database->sum(self::JOIN_RELATIONSHIP_CUSTOMERS, 'purchase.amount', $queries))); + }); + $this->withJoinRelationshipRoles($database, [Role::any()->toString(), Role::user('bob')->toString()], function () use ($database, $queries): void { + $this->assertSame(['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4', 'c3/-'], $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, $queries, 'purchase')); + }); + + $database->updateCollection(self::JOIN_RELATIONSHIP_ORDERS, new CollectionUpdate(permissions: [Permission::create(Role::any())], documentSecurity: false)); + $this->withJoinRelationshipRoles($database, [Role::any()->toString()], function () use ($database, $queries): void { + try { + $database->find(self::JOIN_RELATIONSHIP_CUSTOMERS, $queries); + $this->fail('a joined collection readable neither at collection nor at document level is refused'); + } catch (AuthorizationException $exception) { + $this->assertStringContainsString(self::JOIN_RELATIONSHIP_ORDERS, $exception->getMessage()); + } + }); + } finally { + $this->deleteJoinRelationshipCollections($database); + } + } + + public function testJoinRelationshipCursorPagesEveryJoinedRowOnce(): void + { + $database = $this->getDatabase(); + if (! $this->supportsJoinRelationships($database)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $this->seedJoinRelationships($database); + + try { + foreach ([ + 'left join' => [Query::leftJoin(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id')]), ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4', 'c3/-']], + 'inner join' => [Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id')]), ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4']], + ] as $case => [$join, $expected]) { + $queries = [$join, Query::orderAsc('purchase.amount')]; + $all = \array_values($database->find(self::JOIN_RELATIONSHIP_CUSTOMERS, [...$queries, Query::limit(100)])); + $keys = \array_map(static fn (Document $row): string => self::joinRelationshipPair($row, 'purchase'), $all); + $sorted = $keys; + \sort($sorted); + $this->assertSame($expected, $sorted, "{$case}: the unpaged read returns each joined row once"); + + $forward = []; + $cursor = null; + for ($page = 0; $page <= \count($all); $page++) { + $rows = $database->find(self::JOIN_RELATIONSHIP_CUSTOMERS, [...$queries, Query::limit(2), ...($cursor === null ? [] : [Query::cursorAfter($cursor)])]); + \array_push($forward, ...\array_map(static fn (Document $row): string => self::joinRelationshipPair($row, 'purchase'), $rows)); + if (\count($rows) < 2) { + break; + } + $cursor = $rows[1]; + } + $this->assertSame($keys, $forward, "{$case}: paging forward in pages of two"); + + $backward = []; + $cursor = $all[\count($all) - 1]; + for ($page = 0; $page <= \count($all); $page++) { + $rows = $database->find(self::JOIN_RELATIONSHIP_CUSTOMERS, [...$queries, Query::limit(2), Query::cursorBefore($cursor)]); + $backward = [...\array_map(static fn (Document $row): string => self::joinRelationshipPair($row, 'purchase'), $rows), ...$backward]; + if (\count($rows) < 2) { + break; + } + $cursor = $rows[0]; + } + $this->assertSame(\array_slice($keys, 0, -1), $backward, "{$case}: paging backward in pages of two from the last row"); + } + + $queries = [Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id')]), Query::orderAsc('purchase.amount')]; + $this->assertSame(20, self::joinRelationshipInteger($database->resolveCursor(self::JOIN_RELATIONSHIP_CUSTOMERS, 'c1', CursorDirection::After, $queries)->getAttribute('purchase.amount'))); + $this->assertSame(30, self::joinRelationshipInteger($database->resolveCursor(self::JOIN_RELATIONSHIP_CUSTOMERS, 'c2', CursorDirection::Before, $queries)->getAttribute('purchase.amount'))); + + $forward = []; + $cursor = null; + for ($page = 0; $page <= 4; $page++) { + $rows = $database->find(self::JOIN_RELATIONSHIP_CUSTOMERS, [ + ...$queries, + Query::limit(2), + ...($cursor === null ? [] : [Query::cursorAfter($database->resolveCursor(self::JOIN_RELATIONSHIP_CUSTOMERS, $cursor, CursorDirection::After, $queries))]), + ]); + if ($rows === []) { + break; + } + \array_push($forward, ...\array_map(static fn (Document $row): string => self::joinRelationshipPair($row, 'purchase'), $rows)); + $cursor = $rows[\count($rows) - 1]->getId(); + } + $this->assertSame(['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], $forward, 'paging by the ID of each page\'s last row returns every row once'); + } finally { + $this->deleteJoinRelationshipCollections($database); + } + } + + public function testJoinRelationshipReadsOnlyTheCurrentTenantsRows(): void + { + $database = $this->getDatabase(); + if (! $this->supportsJoinRelationships($database) || ! $database->hasSharedTables()) { + $this->expectNotToPerformAssertions(); + + return; + } + + $tenant = $database->getTenant(); + $integerTenants = $database->getIdAttributeType() === ColumnType::Integer; + $first = $integerTenants ? 801 : 'tenant_801'; + $second = $integerTenants ? 802 : 'tenant_802'; + + try { + $database->setTenant(null); + $this->deleteJoinRelationshipCollections($database); + $this->createJoinRelationshipSchema($database); + + $database->setTenant($first); + $this->createJoinRelationshipDocuments($database, [ + [self::JOIN_RELATIONSHIP_CUSTOMERS, 'c1', ['name' => 'Ada']], + [self::JOIN_RELATIONSHIP_ORDERS, 'o1', ['amount' => 10, 'status' => 'paid', 'customer' => 'c1']], + [self::JOIN_RELATIONSHIP_ORDERS, 'o2', ['amount' => 20, 'status' => 'open', 'customer' => 'c1']], + ]); + + $database->setTenant($second); + $this->createJoinRelationshipDocuments($database, [ + [self::JOIN_RELATIONSHIP_CUSTOMERS, 'c1', ['name' => 'Ada']], + [self::JOIN_RELATIONSHIP_ORDERS, 'o9', ['amount' => 90, 'status' => 'paid', 'customer' => 'c1']], + ]); + + $queries = [Query::leftJoin(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id')])]; + + $database->setTenant($first); + $this->assertSame(['c1/o1', 'c1/o2'], $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, $queries, 'purchase')); + $this->assertSame(30, self::joinRelationshipInteger($database->sum(self::JOIN_RELATIONSHIP_CUSTOMERS, 'purchase.amount', $queries))); + + $database->setTenant($second); + $this->assertSame(['c1/o9'], $this->joinRelationshipPairs($database, self::JOIN_RELATIONSHIP_CUSTOMERS, $queries, 'purchase')); + $this->assertSame(1, $database->count(self::JOIN_RELATIONSHIP_CUSTOMERS, $queries)); + } finally { + $database->setTenant(null); + $this->deleteJoinRelationshipCollections($database); + $database->setTenant($tenant); + } + } + + private function supportsJoinRelationships(Database $database): bool + { + $adapter = $database->getAdapter(); + + return $adapter->supports(Capability::Joins) + && $adapter->supports(Capability::Aggregations) + && $adapter->hasFeature(Feature\Relationships::class); + } + + /** + * Customers c1 (region r1, orders o1 and o2), c2 (region r2, orders o3 and o4) and c3 (region r1, no orders). + */ + private function seedJoinRelationships(Database $database): void + { + $this->deleteJoinRelationshipCollections($database); + $this->createJoinRelationshipSchema($database); + $this->createJoinRelationshipDocuments($database, [ + [self::JOIN_RELATIONSHIP_REGIONS, 'r1', ['name' => 'north']], + [self::JOIN_RELATIONSHIP_REGIONS, 'r2', ['name' => 'south']], + [self::JOIN_RELATIONSHIP_CUSTOMERS, 'c1', ['name' => 'Ada', 'region' => 'r1']], + [self::JOIN_RELATIONSHIP_CUSTOMERS, 'c2', ['name' => 'Bob', 'region' => 'r2']], + [self::JOIN_RELATIONSHIP_CUSTOMERS, 'c3', ['name' => 'Cy', 'region' => 'r1']], + [self::JOIN_RELATIONSHIP_ORDERS, 'o1', ['amount' => 10, 'status' => 'paid', 'region' => 'r1', 'customer' => 'c1']], + [self::JOIN_RELATIONSHIP_ORDERS, 'o2', ['amount' => 20, 'status' => 'open', 'region' => 'r1', 'customer' => 'c1']], + [self::JOIN_RELATIONSHIP_ORDERS, 'o3', ['amount' => 30, 'status' => 'paid', 'region' => 'r1', 'customer' => 'c2']], + [self::JOIN_RELATIONSHIP_ORDERS, 'o4', ['amount' => 40, 'status' => 'paid', 'region' => 'r2', 'customer' => 'c2']], + ]); + } + + private function createJoinRelationshipSchema(Database $database): void + { + $permissions = [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())]; + $database->createCollection(Collection::create( + id: self::JOIN_RELATIONSHIP_CUSTOMERS, + attributes: [Attribute::string(key: 'name', size: 16)], + permissions: $permissions, + )); + $database->createCollection(Collection::create( + id: self::JOIN_RELATIONSHIP_ORDERS, + attributes: [ + Attribute::integer(key: 'amount'), + Attribute::string(key: 'status', size: 16), + Attribute::string(key: 'region', size: 16), + ], + permissions: $permissions, + )); + $database->createCollection(Collection::create( + id: self::JOIN_RELATIONSHIP_REGIONS, + attributes: [Attribute::string(key: 'name', size: 16)], + permissions: $permissions, + )); + + $database->createRelationship(self::JOIN_RELATIONSHIP_CUSTOMERS, Relationship::oneToMany(relatedCollection: self::JOIN_RELATIONSHIP_ORDERS, key: 'orders', twoWayKey: 'customer')); + $database->createRelationship(self::JOIN_RELATIONSHIP_CUSTOMERS, Relationship::manyToOne(relatedCollection: self::JOIN_RELATIONSHIP_REGIONS, key: 'region', twoWay: true, twoWayKey: 'customers')); + } + + /** + * @param list}> $documents + */ + private function createJoinRelationshipDocuments(Database $database, array $documents): void + { + foreach ($documents as [$collection, $id, $attributes]) { + $database->createDocument($collection, new Document([ + '$id' => $id, + '$permissions' => [Permission::read(Role::any())], + ...$attributes, + ])); + } + } + + private function deleteJoinRelationshipCollections(Database $database): void + { + foreach (self::JOIN_RELATIONSHIP_COLLECTIONS as $collection) { + if ($database->collectionExists($collection)) { + $database->deleteCollection($collection); + } + } + } + + /** + * Each row as `
/`, sorted, with `-` for a row the outer join did not match. + * + * @param list $queries + * @return list + */ + private function joinRelationshipPairs(Database $database, string $collection, array $queries, string $alias): array + { + $pairs = \array_map( + static fn (Document $row): string => self::joinRelationshipPair($row, $alias), + $database->find($collection, [...$queries, Query::limit(100)]), + ); + \sort($pairs); + + return $pairs; + } + + private static function joinRelationshipPair(Document $row, string $alias): string + { + $joined = $row->getAttribute($alias.'.$id'); + + return $row->getId().'/'.(\is_string($joined) ? $joined : '-'); + } + + /** + * @param list $roles + * @param callable(): void $callback + */ + private function withJoinRelationshipRoles(Database $database, array $roles, callable $callback): void + { + $authorization = $database->getAuthorization(); + $previous = $authorization->getRoles(); + $authorization->cleanRoles(); + foreach ($roles as $role) { + $authorization->addRole($role); + } + + try { + $callback(); + } finally { + $authorization->cleanRoles(); + foreach ($previous as $role) { + $authorization->addRole($role); + } + } + } + + private static function joinRelationshipInteger(mixed $value): ?int + { + return \is_numeric($value) ? (int) $value : null; + } +} diff --git a/tests/unit/Joins/JoinInternalColumnsTest.php b/tests/unit/Joins/JoinInternalColumnsTest.php index 7cf423df5..f69bee478 100644 --- a/tests/unit/Joins/JoinInternalColumnsTest.php +++ b/tests/unit/Joins/JoinInternalColumnsTest.php @@ -237,10 +237,17 @@ public function testJoinConditionOverARelationshipNamesItsColumn(): void $this->assertSame([['rows' => 2]], $this->database->aggregate('persons', [Query::join('books', 'book', [Query::on('$id', 'owner')]), Query::count('*', 'rows')]), 'the child side of a one-to-many relationship holds a column'); $this->assertSame(2, $this->database->count('books', [Query::join('persons', 'person', [Query::on('owner', '$id')])])); - $this->assertInvalidQuery( - 'Invalid query: Cannot join on virtual relationship attribute: books', - fn (): mixed => $this->database->find('persons', [Query::join('books', 'book', [Query::on('books', '$id')])]), + $owned = $this->database->find('persons', [ + Query::join('books', 'book', [Query::on('books', '$id')]), + Query::select(['name', 'book.title']), + Query::orderAsc('book.title'), + ]); + $this->assertSame( + [['Ada', 'One'], ['Ada', 'Two']], + \array_map(static fn (Document $person): array => [$person->getAttribute('name'), $person->getAttribute('book.title')], $owned), + 'the parent side of a one-to-many relationship joins on the two-way key its children store', ); + $this->assertInvalidQuery( 'Invalid query: Cannot join on virtual relationship attribute: person', fn (): mixed => $this->database->find('persons', [Query::join('libraries', 'lib', [Query::on('$id', 'person')])]), diff --git a/tests/unit/Joins/RelationshipJoinTest.php b/tests/unit/Joins/RelationshipJoinTest.php new file mode 100644 index 000000000..ebb89b819 --- /dev/null +++ b/tests/unit/Joins/RelationshipJoinTest.php @@ -0,0 +1,418 @@ +database = new Database(new SQLite(new PDO('sqlite::memory:')), new Cache(new NoCache())); + $this->database + ->setDatabase('relationship_join') + ->setNamespace('relationship_join_'.\uniqid()) + ->setAuthorization(new Authorization()); + $this->database->addHook(new Relationships()); + $this->database->addHook(new Permissions()); + $this->database->create(); + + $this->createCollection('customers', [Attribute::string(key: 'name', size: 16)]); + $this->createCollection('orders', [ + Attribute::integer(key: 'amount'), + Attribute::string(key: 'status', size: 16), + Attribute::string(key: 'region', size: 16), + ]); + $this->createCollection('regions', [Attribute::string(key: 'name', size: 16)]); + + $this->database->createRelationship('customers', Relationship::oneToMany(relatedCollection: 'orders', key: 'orders', twoWayKey: 'customer')); + $this->database->createRelationship('customers', Relationship::manyToOne(relatedCollection: 'regions', key: 'region', twoWay: true, twoWayKey: 'customers')); + + $this->createDocument('regions', 'r1', ['name' => 'north']); + $this->createDocument('regions', 'r2', ['name' => 'south']); + $this->createDocument('customers', 'c1', ['name' => 'Ada', 'region' => 'r1']); + $this->createDocument('customers', 'c2', ['name' => 'Bob', 'region' => 'r2']); + $this->createDocument('customers', 'c3', ['name' => 'Cy', 'region' => 'r1']); + + foreach ([ + 'o1' => [10, 'paid', 'r1', 'c1'], + 'o2' => [20, 'open', 'r1', 'c1'], + 'o3' => [30, 'paid', 'r1', 'c2'], + 'o4' => [40, 'paid', 'r2', 'c2'], + ] as $id => [$amount, $status, $region, $customer]) { + $this->createDocument('orders', $id, ['amount' => $amount, 'status' => $status, 'region' => $region, 'customer' => $customer]); + } + } + + public function testLeftJoinOnAOneToManyRelationshipPairsEachParentWithItsChildren(): void + { + $this->assertSame( + ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4', 'c3/-'], + $this->pairs('customers', [Query::leftJoin('orders', 'ord', [Query::on('orders', '$id')])], 'ord'), + ); + } + + public function testJoinOnAOneToManyRelationshipMatchesTheParentIdAgainstTheTwoWayKey(): void + { + $this->assertSame( + ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], + $this->pairs('customers', [Query::join('orders', 'ord', [Query::on('orders', '$id')])], 'ord'), + ); + } + + public function testRightColumnOfTheConditionBecomesTheTwoWayKey(): void + { + $expected = ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4']; + + $this->assertSame($expected, $this->pairs('customers', [Query::join('orders', 'ord', [Query::on('orders', 'ord.$id')])], 'ord')); + $this->assertSame($expected, $this->pairs('customers', [Query::join('orders', 'ord', [Query::on('orders', 'status')])], 'ord'), 'the right column is replaced, as appwrite replaced it'); + } + + public function testRewrittenJoinKeepsItsAlias(): void + { + $rows = $this->database->find('customers', [ + Query::join('orders', 'purchase', [Query::on('orders', '$id')]), + Query::select(['name', 'purchase.amount']), + Query::orderAsc('purchase.amount'), + ]); + + $this->assertSame([10, 20, 30, 40], \array_map(static fn (Document $row): mixed => $row->getAttribute('purchase.amount'), $rows)); + $this->assertSame(['Ada', 'Ada', 'Bob', 'Bob'], \array_map(static fn (Document $row): mixed => $row->getAttribute('name'), $rows)); + } + + public function testOnlyTheOneToManyConditionOfTheOnListIsRewritten(): void + { + $this->assertSame( + ['c1/o1', 'c2/o4'], + $this->pairs('customers', [Query::join('orders', 'ord', [ + Query::on('orders', '$id'), + Query::on('region', 'region'), + Query::equal('ord.status', ['paid']), + ])], 'ord'), + 'the many-to-one column and the ON filter keep comparing what they named', + ); + } + + public function testOperatorOfTheRewrittenConditionIsKept(): void + { + $this->assertSame( + ['c1/o3', 'c1/o4', 'c2/o1', 'c2/o2', 'c3/o1', 'c3/o2', 'c3/o3', 'c3/o4'], + $this->pairs('customers', [Query::join('orders', 'ord', [Query::on('orders', '$id', '!=')])], 'ord'), + ); + } + + public function testConditionOnAnEarlierJoinsColumnIsNotTheMainRelationship(): void + { + $this->expectException(QueryException::class); + $this->expectExceptionMessage(self::VIRTUAL.'peer.orders'); + + $this->database->find('customers', [ + Query::join('customers', 'peer', [Query::on('$id', '$id')]), + Query::join('orders', 'sub', [Query::on('peer.orders', '$id')]), + ]); + } + + public function testOneToManyWithoutATwoWayKeyIsNotRewritten(): void + { + $this->createCollection('ledgers', [Attribute::relationship('orders', Relationship::oneToMany(relatedCollection: 'orders'), RelationshipSide::Parent)]); + + $this->expectException(QueryException::class); + $this->expectExceptionMessage(self::VIRTUAL.'orders'); + + $this->database->find('ledgers', [Query::join('orders', 'ord', [Query::on('orders', '$id')])]); + } + + public function testOneToManyDeclaredInTheCollectionDefinitionIsRewritten(): void + { + $this->createCollection('areas', [ + Attribute::string(key: 'name', size: 16), + Attribute::relationship('orders', Relationship::oneToMany(relatedCollection: 'orders', twoWayKey: 'region'), RelationshipSide::Parent), + ]); + $this->createDocument('areas', 'r1', ['name' => 'north']); + $this->createDocument('areas', 'r2', ['name' => 'south']); + + $this->assertSame( + ['r1/o1', 'r1/o2', 'r1/o3', 'r2/o4'], + $this->pairs('areas', [Query::join('orders', 'ord', [Query::on('orders', '$id')])], 'ord'), + ); + } + + public function testJoinOnTheChildSideOfAOneToManyKeepsItsColumn(): void + { + $this->assertSame( + ['o1/c1', 'o2/c1', 'o3/c2', 'o4/c2'], + $this->pairs('orders', [Query::join('customers', 'buyer', [Query::on('customer', '$id')])], 'buyer'), + 'the child side of a one-to-many holds the parent ID in its own column', + ); + } + + public function testJoinOnTheParentSideOfAManyToOneKeepsItsColumn(): void + { + $this->assertSame( + ['c1/r1', 'c2/r2', 'c3/r1'], + $this->pairs('customers', [Query::join('regions', 'area', [Query::on('region', '$id')])], 'area'), + ); + } + + /** + * @return iterable + */ + public static function refusedRelationshipJoins(): iterable + { + yield 'the child side of a two-way many-to-one' => [ + static fn (Database $database): mixed => $database->find('regions', [Query::join('customers', 'resident', [Query::on('customers', '$id')])]), + 'customers', + ]; + yield 'the parent side of a many-to-many' => [ + static function (Database $database): mixed { + $database->createCollection(Collection::create(id: 'tags', permissions: [Permission::read(Role::any())])); + $database->createRelationship('customers', Relationship::manyToMany(relatedCollection: 'tags', key: 'tags', twoWay: true, twoWayKey: 'customers')); + + return $database->find('customers', [Query::join('tags', 'tag', [Query::on('tags', '$id')])]); + }, + 'tags', + ]; + yield 'the child side of a one-way one-to-one' => [ + static function (Database $database): mixed { + $database->createCollection(Collection::create(id: 'profiles', permissions: [Permission::read(Role::any())])); + $database->createRelationship('customers', Relationship::oneToOne(relatedCollection: 'profiles', key: 'profile', twoWayKey: 'owner')); + + return $database->find('profiles', [Query::join('customers', 'customer', [Query::on('owner', '$id')])]); + }, + 'owner', + ]; + } + + /** + * @param Closure(Database): mixed $read + */ + #[DataProvider('refusedRelationshipJoins')] + public function testJoinOnAnotherRelationshipWithoutAColumnIsStillRefused(Closure $read, string $attribute): void + { + $this->expectException(QueryException::class); + $this->expectExceptionMessage(self::VIRTUAL.$attribute); + + $read($this->database); + } + + /** + * @return iterable + */ + public static function reads(): iterable + { + $join = Query::join('orders', 'ord', [Query::on('orders', '$id')]); + + yield 'count' => [static fn (Database $database): int => $database->count('customers', [$join]), 4]; + yield 'sum' => [static fn (Database $database): int => (int) $database->sum('customers', 'ord.amount', [$join]), 100]; + yield 'aggregate' => [ + static fn (Database $database): array => \array_map( + static fn (array $row): array => [$row['name'], self::integer($row['total'])], + $database->aggregate('customers', [$join, Query::sum('ord.amount', 'total'), Query::groupBy(['name']), Query::orderAsc('name')]), + ), + [['Ada', 30], ['Bob', 70]], + ]; + yield 'read' => [ + static fn (Database $database): array => \array_map( + static fn (Document $row): ?int => self::integer($row->getAttribute('rows')), + $database->read('customers', [$join, Query::count('*', 'rows')]), + ), + [4], + ]; + yield 'getDocument' => [ + static fn (Database $database): mixed => $database->getDocument('customers', 'c1', [ + Query::join('orders', 'ord', [Query::on('orders', '$id'), Query::equal('ord.status', ['open'])]), + Query::select(['name', 'ord.amount']), + ])->getAttribute('ord.amount'), + 20, + ]; + } + + /** + * @param Closure(Database): mixed $read + */ + #[DataProvider('reads')] + public function testEveryReadOfAOneToManyJoinMatchesTheTwoWayKey(Closure $read, mixed $expected): void + { + $this->assertSame($expected, $read($this->database)); + } + + public function testOneToManyJoinIsRewrittenWithoutValidation(): void + { + $pairs = $this->database->skipValidation(fn (): array => $this->pairs('customers', [Query::join('orders', 'ord', [Query::on('orders', '$id')])], 'ord')); + + $this->assertSame(['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], $pairs); + } + + public function testCallersJoinIsLeftAsGiven(): void + { + $condition = Query::on('orders', '$id'); + $join = Query::join('orders', 'ord', [$condition]); + + $this->assertSame(4, $this->database->count('customers', [$join])); + $this->assertSame(['orders', '=', '$id'], $condition->getValues()); + $this->assertSame($condition, $join->getJoinOnQueries()[0]); + $this->assertSame(['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], $this->pairs('customers', [$join], 'ord')); + } + + public function testJoinedRowsAreTheOnesTheCallerCanRead(): void + { + $authorization = $this->database->getAuthorization(); + $this->database->updateCollection('orders', new CollectionUpdate(permissions: [Permission::create(Role::any())], documentSecurity: true)); + $authorization->skip(fn (): Document => $this->database->updateDocument('orders', 'o2', new Document(['$permissions' => [Permission::read(Role::user('bob'))]]))); + $join = Query::leftJoin('orders', 'ord', [Query::on('orders', '$id')]); + + $this->assertSame(['c1/o1', 'c2/o3', 'c2/o4', 'c3/-'], $this->pairs('customers', [$join], 'ord')); + + $authorization->addRole(Role::user('bob')->toString()); + $this->assertSame(['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4', 'c3/-'], $this->pairs('customers', [$join], 'ord')); + } + + public function testCursorPagingOverAOneToManyJoinReturnsEveryJoinedRowOnce(): void + { + $queries = [Query::leftJoin('orders', 'ord', [Query::on('orders', '$id')]), Query::orderAsc('ord.amount')]; + $all = \array_values($this->database->find('customers', [...$queries, Query::limit(100)])); + $keys = \array_map(static fn (Document $row): string => self::pair($row, 'ord'), $all); + $this->assertSame(['c3/-', 'c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], $keys); + + foreach ($all as $index => $row) { + $this->assertSame(\array_slice($keys, $index + 1), $this->keys([...$queries, Query::cursorAfter($row)]), "after {$keys[$index]}"); + $this->assertSame(\array_slice($keys, 0, $index), $this->keys([...$queries, Query::cursorBefore($row)]), "before {$keys[$index]}"); + } + + $forward = []; + $cursor = null; + for ($page = 0; $page <= \count($all); $page++) { + $rows = $this->database->find('customers', [...$queries, Query::limit(2), ...($cursor === null ? [] : [Query::cursorAfter($cursor)])]); + \array_push($forward, ...\array_map(static fn (Document $row): string => self::pair($row, 'ord'), $rows)); + if (\count($rows) < 2) { + break; + } + $cursor = $rows[1]; + } + $this->assertSame($keys, $forward, 'paging forward in pages of two'); + } + + public function testCursorResolvedFromAnIdPagesAOneToManyJoin(): void + { + $queries = [Query::join('orders', 'ord', [Query::on('orders', '$id')]), Query::orderAsc('ord.amount')]; + + $this->assertSame(20, $this->database->resolveCursor('customers', 'c1', CursorDirection::After, $queries)->getAttribute('ord.amount')); + $this->assertSame(30, $this->database->resolveCursor('customers', 'c2', CursorDirection::Before, $queries)->getAttribute('ord.amount')); + + $forward = []; + $cursor = null; + for ($page = 0; $page <= 4; $page++) { + $rows = $this->database->find('customers', [ + ...$queries, + Query::limit(2), + ...($cursor === null ? [] : [Query::cursorAfter($this->database->resolveCursor('customers', $cursor, CursorDirection::After, $queries))]), + ]); + if ($rows === []) { + break; + } + \array_push($forward, ...\array_map(static fn (Document $row): string => self::pair($row, 'ord'), $rows)); + $cursor = $rows[\count($rows) - 1]->getId(); + } + $this->assertSame(['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], $forward, 'paging by the ID of each page\'s last row returns every row once'); + + $before = $this->database->find('customers', [...$queries, Query::cursorBefore($this->database->resolveCursor('customers', 'c2', CursorDirection::Before, $queries))]); + $this->assertSame(['c1/o1', 'c1/o2'], \array_map(static fn (Document $row): string => self::pair($row, 'ord'), $before)); + } + + /** + * Each row as `
/`, sorted, with `-` for a row the outer join did not match. + * + * @param list $queries + * @return list + */ + private function pairs(string $collection, array $queries, string $alias): array + { + $pairs = \array_map( + static fn (Document $row): string => self::pair($row, $alias), + $this->database->find($collection, [...$queries, Query::limit(100)]), + ); + \sort($pairs); + + return $pairs; + } + + /** + * @param list $queries + * @return list + */ + private function keys(array $queries): array + { + return \array_values(\array_map( + static fn (Document $row): string => self::pair($row, 'ord'), + $this->database->find('customers', [...$queries, Query::limit(100)]), + )); + } + + private static function pair(Document $row, string $alias): string + { + $joined = $row->getAttribute($alias.'.$id'); + + return $row->getId().'/'.(\is_string($joined) ? $joined : '-'); + } + + /** + * @param list $attributes + */ + private function createCollection(string $id, array $attributes): void + { + $this->database->createCollection(Collection::create( + id: $id, + attributes: $attributes, + permissions: [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], + )); + } + + /** + * @param array $attributes + */ + private function createDocument(string $collection, string $id, array $attributes): void + { + $this->database->createDocument($collection, new Document([ + '$id' => $id, + '$permissions' => [Permission::read(Role::any())], + ...$attributes, + ])); + } + + private static function integer(mixed $value): ?int + { + return \is_numeric($value) ? (int) $value : null; + } +} From 11c2b2522dacb2242e2f0440bbf93fc8fea5ee42 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 20:19:26 +1300 Subject: [PATCH 52/55] docs: describe joins on a one-to-many relationship in 8.1.0 8.1.0 no longer behaves exactly as 8.0.3 in one respect: it runs joins on the parent side of a one-to-many relationship that 8.0.3 refused, so the changelog and upgrade guide say so and show how the condition is resolved. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 ++++++++- UPGRADE.md | 28 +++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67ed9d968..f44d89f66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ 8.1.0 adds opt-in and additive APIs for work that callers had to do themselves. With no `Hook\Nested` hook registered and with `setCheckGrants()` and `setRoundFractions()` off, which is the default, a `Database` behaves exactly as in -8.0.3. See [Upgrading from 8.0 to 8.1](UPGRADE.md#upgrading-from-80-to-81). +8.0.3, except that it runs joins on a one-to-many relationship attribute that 8.0.3 refused. See +[Upgrading from 8.0 to 8.1](UPGRADE.md#upgrading-from-80-to-81). ### Added @@ -56,6 +57,12 @@ and with `setCheckGrants()` and `setRoundFractions()` off, which is the default, otherwise they are what `find()` returns. Aggregate queries under a permission other than read throw `Exception\Query`. `find()` still refuses aggregate queries. - `ParsedQuery::isAggregation(): bool` tells whether the queries aggregate or group. +- A join condition whose left column is the parent side of a one-to-many relationship of the main collection, such + as `Query::on('orders', '$id')`, compares the main `$id` with the relationship's two-way key in the joined + collection. The condition keeps its operator. `getDocument()`, `find()`, `findOne()`, `read()`, `aggregate()`, + `count()`, `sum()` and `resolveCursor()` run it; 8.0.3 refused it with `Cannot join on virtual relationship + attribute`. The joined collection is read under the same permission and tenant rules as any join. The child side of + a one-to-many and every other relationship type join as before. - The `State\Snapshot` constructor takes two more trailing arguments, `bool $checkGrants = false` and `bool $roundFractions = false`. `Database::snapshot()` fills them in. diff --git a/UPGRADE.md b/UPGRADE.md index 7293f977f..794621119 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -1,7 +1,8 @@ # Upgrading from 8.0 to 8.1 8.1 is a minor release. Every change in it is additive or opt-in. With no `Hook\Nested` hook registered and with -`setCheckGrants()` and `setRoundFractions()` off, which is the default, a `Database` behaves exactly as in 8.0.3. +`setCheckGrants()` and `setRoundFractions()` off, which is the default, a `Database` behaves exactly as in 8.0.3, +except that it runs [joins on a one-to-many relationship](#joins-on-a-one-to-many-relationship) that 8.0.3 refused. [CHANGELOG.md](CHANGELOG.md) lists every new signature. - [Preparing nested related documents](#preparing-nested-related-documents) @@ -9,6 +10,7 @@ - [Fractional operands on integer attributes](#fractional-operands-on-integer-attributes) - [Cursor from an ID](#cursor-from-an-id) - [One read for documents and aggregates](#one-read-for-documents-and-aggregates) +- [Joins on a one-to-many relationship](#joins-on-a-one-to-many-relationship) - [Spatial support behind a pool](#spatial-support-behind-a-pool) - [`State\Snapshot`](#statesnapshot) @@ -265,6 +267,30 @@ PermissionType::Read): array`. It returns a list of `Document`: `ParsedQuery::isAggregation(): bool` (from `Query::groupByType($queries)`) tells which of the two a list is. +## Joins on a one-to-many relationship + +Additive. The parent side of a one-to-many relationship holds no column: each child stores the parent's ID under the +relationship's two-way key. 8.0.3 refused a join condition whose left column is such an attribute +(`Cannot join on virtual relationship attribute`). 8.1 compares the main `$id` with the two-way key in the joined +collection instead. + +```php +// customers: Relationship::oneToMany(relatedCollection: 'orders', key: 'orders', twoWayKey: 'customer') +$rows = $database->find('customers', [ + Query::leftJoin('orders', 'ord', [Query::on('orders', '$id')]), // runs as Query::on('$id', 'customer') + Query::select(['name', 'ord.amount']), +]); +``` + +- The condition keeps its operator, and its right column becomes the two-way key. Other conditions of the join are + unchanged, and the caller's `Query` objects are not modified. +- `getDocument()`, `find()`, `findOne()`, `read()`, `aggregate()`, `count()`, `sum()` and `resolveCursor()` run it, so + `cursorAfter()` and `cursorBefore()` page over the joined rows. +- Only an unqualified left column that is the parent side of a one-to-many with a two-way key is resolved. The child + side holds a column and joins as before. Other relationship types, and a left column under a join's alias, are + refused as before. +- The joined collection is read under the same permission and tenant rules as any join. + ## Spatial support behind a pool No change. 8.0.3 already answers correctly, and 8.1 adds tests for it. Ask From 49b55f7bbbedc59a3885a15140b50b74c48b9049 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 20:35:14 +1300 Subject: [PATCH 53/55] fix(joins): check cursor() pages of a one-to-many join against the rewritten join pages() checks a page's last row before yielding it, using the caller's raw queries. find() rewrites on('orders', '$id') to on('$id', 'customer'), but nextPageCheck() parsed the unrewritten join, so it took the condition for a match on the joined $id, judged it at most one row and added no joined-$id tie-break. find() does add it, so a read whose rows lack the joined $id yielded its first page and only then threw OrderException on the second, breaking cursor()'s promise that an unpageable read fails before any row. nextPageCheck() now resolves one-to-many joins on the collection it already loads, exactly as find() does. Queries without a parent-side one-to-many join come back unchanged, so their check is as in 8.0.3. Co-Authored-By: Claude Opus 5.5 --- src/Database/Trait/Documents.php | 7 ++-- .../Adapter/Scopes/JoinRelationshipTests.php | 38 +++++++++++++++++++ tests/unit/Joins/RelationshipJoinTest.php | 37 ++++++++++++++++++ 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 8aebd8144..9e36a53bc 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -4861,14 +4861,15 @@ private function addTieBreaks(array $orderAttributes, array $orderTypes, array $ private function nextPageCheck(string $collection, array $queries): Closure { $parsed = Query::groupByType($queries); - $joins = $parsed->joins; - $distinct = $parsed->distinct; - if (($joins === [] && ! $distinct) || $parsed->aggregations !== [] || $parsed->groupBy !== []) { + if (($parsed->joins === [] && ! $parsed->distinct) || $parsed->aggregations !== [] || $parsed->groupBy !== []) { return static function (Document $cursor): void { }; } $collection = $this->silent(fn () => $this->getCollection($collection)); + $parsed = Query::groupByType(self::resolveOneToManyJoins($collection, $queries)); + $joins = $parsed->joins; + $distinct = $parsed->distinct; $joinedCollections = $this->joinedCollectionsByAlias($joins, $this->resolveJoinedCollections($joins)); $selects = $parsed->selections; $filters = $parsed->filters; diff --git a/tests/e2e/Adapter/Scopes/JoinRelationshipTests.php b/tests/e2e/Adapter/Scopes/JoinRelationshipTests.php index be7ccc011..c634eac38 100644 --- a/tests/e2e/Adapter/Scopes/JoinRelationshipTests.php +++ b/tests/e2e/Adapter/Scopes/JoinRelationshipTests.php @@ -10,6 +10,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception\Authorization as AuthorizationException; +use Utopia\Database\Exception\Order as OrderException; use Utopia\Database\Exception\Query as QueryException; use Utopia\Database\Permission; use Utopia\Database\Query; @@ -275,6 +276,43 @@ public function testJoinRelationshipCursorPagesEveryJoinedRowOnce(): void } } + public function testJoinRelationshipIteratesEveryJoinedRowOnceOrFailsBeforeYieldingARow(): void + { + $database = $this->getDatabase(); + if (! $this->supportsJoinRelationships($database)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $this->seedJoinRelationships($database); + + try { + $join = Query::join(self::JOIN_RELATIONSHIP_ORDERS, 'purchase', [Query::on('orders', '$id')]); + + $rows = $database->cursor(self::JOIN_RELATIONSHIP_CUSTOMERS, [$join, Query::select(['name', 'purchase.amount']), Query::orderAsc('purchase.amount')], 2); + /** @var int $yielded */ + $yielded = 0; + try { + foreach ($rows as $row) { + $yielded++; + } + $this->fail('a one-to-many join pairs a customer with several orders, so its next page orders by the joined $id the rows lack'); + } catch (OrderException $exception) { + $this->assertSame('purchase.$id', $exception->getAttribute()); + } + $this->assertSame(0, $yielded, 'the read is refused before the caller acts on any of its rows'); + + $rows = $database->cursor(self::JOIN_RELATIONSHIP_CUSTOMERS, [$join, Query::select(['name', 'purchase.$id', 'purchase.amount']), Query::orderAsc('purchase.amount')], 2); + $this->assertSame( + ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], + \array_map(static fn (Document $row): string => self::joinRelationshipPair($row, 'purchase'), \iterator_to_array($rows, false)), + ); + } finally { + $this->deleteJoinRelationshipCollections($database); + } + } + public function testJoinRelationshipReadsOnlyTheCurrentTenantsRows(): void { $database = $this->getDatabase(); diff --git a/tests/unit/Joins/RelationshipJoinTest.php b/tests/unit/Joins/RelationshipJoinTest.php index ebb89b819..c408d0f97 100644 --- a/tests/unit/Joins/RelationshipJoinTest.php +++ b/tests/unit/Joins/RelationshipJoinTest.php @@ -14,6 +14,7 @@ use Utopia\Database\CollectionUpdate; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Exception\Order as OrderException; use Utopia\Database\Exception\Query as QueryException; use Utopia\Database\Hook\Permissions; use Utopia\Database\Hook\Relationships; @@ -351,6 +352,42 @@ public function testCursorResolvedFromAnIdPagesAOneToManyJoin(): void $this->assertSame(['c1/o1', 'c1/o2'], \array_map(static fn (Document $row): string => self::pair($row, 'ord'), $before)); } + public function testPagingAOneToManyJoinItCannotPageFailsBeforeYieldingARow(): void + { + $rows = $this->database->cursor('customers', [ + Query::join('orders', 'purchase', [Query::on('orders', '$id')]), + Query::select(['name', 'purchase.amount']), + Query::orderAsc('purchase.amount'), + ], 2); + /** @var int $yielded */ + $yielded = 0; + + try { + foreach ($rows as $row) { + $yielded++; + } + $this->fail('A one-to-many join pairs a customer with several orders, so its next page orders by the joined $id the rows lack'); + } catch (OrderException $exception) { + $this->assertSame('purchase.$id', $exception->getAttribute()); + } + + $this->assertSame(0, $yielded, 'The read must be refused before the caller acts on any of its rows'); + } + + public function testPagingAOneToManyJoinPagesEveryJoinedRowOnce(): void + { + $rows = $this->database->cursor('customers', [ + Query::join('orders', 'purchase', [Query::on('orders', '$id')]), + Query::select(['name', 'purchase.$id', 'purchase.amount']), + Query::orderAsc('purchase.amount'), + ], 2); + + $this->assertSame( + ['c1/o1', 'c1/o2', 'c2/o3', 'c2/o4'], + \array_map(static fn (Document $row): string => self::pair($row, 'purchase'), \iterator_to_array($rows, false)), + ); + } + /** * Each row as `
/`, sorted, with `-` for a row the outer join did not match. * From 6d552fb486d6ee184e5ab429ed3c1503bdad4d12 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 20:35:14 +1300 Subject: [PATCH 54/55] docs: say other relationship joins behave as in 8.0.3, not that they are refused The upgrade note claimed every relationship type other than a parent-side one-to-many is refused, but the child side of a one-to-many, the parent side of a many-to-one and others hold a column and join; only the virtual sides are refused, exactly as in 8.0.3. The changelog line likewise implied they all join. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- UPGRADE.md | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f44d89f66..e79d0e122 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,7 +62,7 @@ and with `setCheckGrants()` and `setRoundFractions()` off, which is the default, collection. The condition keeps its operator. `getDocument()`, `find()`, `findOne()`, `read()`, `aggregate()`, `count()`, `sum()` and `resolveCursor()` run it; 8.0.3 refused it with `Cannot join on virtual relationship attribute`. The joined collection is read under the same permission and tenant rules as any join. The child side of - a one-to-many and every other relationship type join as before. + a one-to-many and every other relationship type join, or are refused, as before. - The `State\Snapshot` constructor takes two more trailing arguments, `bool $checkGrants = false` and `bool $roundFractions = false`. `Database::snapshot()` fills them in. diff --git a/UPGRADE.md b/UPGRADE.md index 8e261fef0..4dccbdfdc 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -288,9 +288,8 @@ $rows = $database->find('customers', [ unchanged, and the caller's `Query` objects are not modified. - `getDocument()`, `find()`, `findOne()`, `read()`, `aggregate()`, `count()`, `sum()` and `resolveCursor()` run it, so `cursorAfter()` and `cursorBefore()` page over the joined rows. -- Only an unqualified left column that is the parent side of a one-to-many with a two-way key is resolved. The child - side holds a column and joins as before. Other relationship types, and a left column under a join's alias, are - refused as before. +- Only an unqualified left column that is the parent side of a one-to-many with a two-way key is resolved. Every + other relationship attribute joins, or is refused, exactly as in 8.0.3; so is a left column under a join's alias. - The joined collection is read under the same permission and tenant rules as any join. ## Spatial support behind a pool From 2c0212c9163128379d0905fbfb83ae5580f63783 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 13 Oct 2026 20:50:00 +1300 Subject: [PATCH 55/55] docs: list the one-to-many join under changed in 8.1.0 The parent-side one-to-many join now runs where 8.0.3 refused it, so it is a behaviour change rather than an addition. The 8.1.0 changelog and upgrade guide filed it as additive, which contradicted the PR body's Changed section and hid the one default-on delta from readers scanning for changes. The changelog also omitted that a Hook\Nested-assigned $id is judged new by the grant check, which the upgrade guide and PR body already state. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 18 +++++++++++------- UPGRADE.md | 9 +++++---- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e79d0e122..a4b382fd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,8 @@ 8.1.0 adds opt-in and additive APIs for work that callers had to do themselves. With no `Hook\Nested` hook registered and with `setCheckGrants()` and `setRoundFractions()` off, which is the default, a `Database` behaves exactly as in -8.0.3, except that it runs joins on a one-to-many relationship attribute that 8.0.3 refused. See -[Upgrading from 8.0 to 8.1](UPGRADE.md#upgrading-from-80-to-81). +8.0.3, except that it runs a join on the parent side of a one-to-many relationship attribute, which 8.0.3 refused +(listed under Changed). See [Upgrading from 8.0 to 8.1](UPGRADE.md#upgrading-from-80-to-81). ### Added @@ -21,9 +21,10 @@ and with `setCheckGrants()` and `setRoundFractions()` off, which is the default, `withCheckGrants(bool $check, callable $callback): mixed`, off by default. While on and while authorization is enabled, a related document given as a document to `createDocument()`, `createDocuments()`, `updateDocument()`, `upsertDocument()`, `upsertDocuments()`, `authorizeCreate()` or `authorizeUpdate()` may add permissions only for - roles the caller holds. The check judges the document the `Hook\Nested` hooks return. An unheld role throws - `Exception\Authorization` (`Permissions must be one of: ()`) and malformed permissions throw - `Exception\Structure`, before anything is written. + roles the caller holds. The check judges the document the `Hook\Nested` hooks return. A related document whose + `$id` a `Hook\Nested` hook assigned is judged new: none of its permissions are read or locked, and it needs create + permission. An unheld role throws `Exception\Authorization` (`Permissions must be one of: ()`) and malformed + permissions throw `Exception\Structure`, before anything is written. - While grants are checked, `updateDocuments()` refuses a related document given as a document with `Exception\Relationship` (`Invalid relationship value. A bulk update takes related documents by ID while grants are checked.`). Related documents given by ID still link. @@ -57,14 +58,17 @@ and with `setCheckGrants()` and `setRoundFractions()` off, which is the default, otherwise they are what `find()` returns. Aggregate queries under a permission other than read throw `Exception\Query`. `find()` still refuses aggregate queries. - `ParsedQuery::isAggregation(): bool` tells whether the queries aggregate or group. +- The `State\Snapshot` constructor takes two more trailing arguments, `bool $checkGrants = false` and + `bool $roundFractions = false`. `Database::snapshot()` fills them in. + +### Changed + - A join condition whose left column is the parent side of a one-to-many relationship of the main collection, such as `Query::on('orders', '$id')`, compares the main `$id` with the relationship's two-way key in the joined collection. The condition keeps its operator. `getDocument()`, `find()`, `findOne()`, `read()`, `aggregate()`, `count()`, `sum()` and `resolveCursor()` run it; 8.0.3 refused it with `Cannot join on virtual relationship attribute`. The joined collection is read under the same permission and tenant rules as any join. The child side of a one-to-many and every other relationship type join, or are refused, as before. -- The `State\Snapshot` constructor takes two more trailing arguments, `bool $checkGrants = false` and - `bool $roundFractions = false`. `Database::snapshot()` fills them in. ## 8.0.3 diff --git a/UPGRADE.md b/UPGRADE.md index 4dccbdfdc..3124798e9 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -1,8 +1,9 @@ # Upgrading from 8.0 to 8.1 -8.1 is a minor release. Every change in it is additive or opt-in. With no `Hook\Nested` hook registered and with -`setCheckGrants()` and `setRoundFractions()` off, which is the default, a `Database` behaves exactly as in 8.0.3, -except that it runs [joins on a one-to-many relationship](#joins-on-a-one-to-many-relationship) that 8.0.3 refused. +8.1 is a minor release. Every change in it is additive or opt-in, with one exception: it runs +[joins on a one-to-many relationship](#joins-on-a-one-to-many-relationship) that 8.0.3 refused. With no `Hook\Nested` +hook registered and with `setCheckGrants()` and `setRoundFractions()` off, which is the default, a `Database` behaves +exactly as in 8.0.3 apart from that join. [CHANGELOG.md](CHANGELOG.md) lists every new signature. - [Preparing nested related documents](#preparing-nested-related-documents) @@ -271,7 +272,7 @@ PermissionType::Read): array`. It returns a list of `Document`: ## Joins on a one-to-many relationship -Additive. The parent side of a one-to-many relationship holds no column: each child stores the parent's ID under the +Changed. The parent side of a one-to-many relationship holds no column: each child stores the parent's ID under the relationship's two-way key. 8.0.3 refused a join condition whose left column is such an attribute (`Cannot join on virtual relationship attribute`). 8.1 compares the main `$id` with the two-way key in the joined collection instead.