Skip to content

feat(aggregate): type, name and decode rows as a read returns them #1024

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
abnegate merged 1 commit into from
Oct 15, 2026
Merged

feat(aggregate): type, name and decode rows as a read returns them #1024

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
412 changes: 412 additions & 0 deletions src/Database/Aggregation.php
View file
Open in desktop

Large diffs are not rendered by default.

22 changes: 17 additions & 5 deletions src/Database/Trait/Documents.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
use Utopia\Database\Adapter;
use Utopia\Database\Adapter\Feature;
use Utopia\Database\Adapter\ReadWritePool;
use Utopia\Database\Aggregation;
use Utopia\Database\Attribute;
use Utopia\Database\Cache\Epoch;
use Utopia\Database\Cache\Owners;
Expand Down Expand Up @@ -4209,16 +4210,19 @@ public function find(string $collection, array $queries = [], PermissionType $fo
* @param array<Query>|null $adapterQueries Receives the queries as the adapter's find() was handed them, or
* null when they can match no document and the adapter is not read
* @param-out array<Query>|null $adapterQueries
* @param array<string, Document> $joinedByAlias Receives the collection each join reads, keyed by join alias
* @param-out array<string, Document> $joinedByAlias
* @return array<Document>
*
* @throws DatabaseException
* @throws QueryException
* @throws TimeoutException
* @throws Exception
*/
private function fetch(Collection $collection, array $queries, PermissionType $forPermission, bool $aggregate, ?array &$adapterQueries = null): array
private function fetch(Collection $collection, array $queries, PermissionType $forPermission, bool $aggregate, ?array &$adapterQueries = null, array &$joinedByAlias = []): array
{
$adapterQueries = null;
$joinedByAlias = [];

$queryCacheQueries = $queries;

Expand Down Expand Up @@ -5084,6 +5088,10 @@ private function pages(
* Run aggregation queries (count, sum, avg, min, max, groupBy) and return one row per group. An aggregate without
* an alias comes back under `<method>_<attribute>`, or `<method>` for count('*').
*
* Counts and bitwise aggregates are integers, averages, deviations and variances floats, and a sum an integer over
* an integer attribute and a float otherwise; min() and max() are typed as their attribute. Each group is named as
* a read names its attribute, a joined one qualified by its join alias, and typed and decoded as a read returns it.
*
* @param array<Query> $queries At least one aggregate or groupBy query
* @return list<array<string, mixed>>
*
Expand All @@ -5097,13 +5105,18 @@ public function aggregate(string $collection, array $queries): array
$queries = self::aliasAggregates($this->rewriteQueries(Event::DocumentAggregate, $definition, $queries));

$floating = self::floatingAggregates($queries);
$adapterQueries = null;
$joinedByAlias = [];
$results = $this->fetch($definition, $queries, PermissionType::Read, true, $adapterQueries, $joinedByAlias);
$aggregation = Aggregation::from($this, $definition, $joinedByAlias, $queries);

$rows = [];
foreach ($this->fetch($definition, $queries, PermissionType::Read, true) as $row) {
foreach ($results as $row) {
$row = $row->getArrayCopy();
foreach ($floating as $alias) {
self::assertFinite($row[$alias] ?? null);
}
$rows[] = $row;
$rows[] = $aggregation->decode($row);
}

$listeners = $this->listens(Event::DocumentAggregate);
Expand Down Expand Up @@ -5178,8 +5191,7 @@ private static function aliasAggregates(array $queries): array
}

$attribute = $query->getAttribute();
$alias = $attribute === '*' || $attribute === '' ? $method->value : $method->value.'_'.$attribute;
$alias = \substr((string) \preg_replace('/[^A-Za-z0-9_]/', '_', $alias), 0, Aggregate::MAX_ALIAS_LENGTH);
$alias = Aggregation::alias($query);

if (isset($taken[$alias])) {
throw new QueryException("The default aggregate alias '{$alias}' is used by more than one aggregate; give each of them an explicit alias");
Expand Down
87 changes: 85 additions & 2 deletions tests/e2e/Adapter/Scopes/AggregationTests.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2411,9 +2411,9 @@ public function testGroupedSelectReturnsEachGroupOnceWithItsAggregate(): void
'full outer join' => [Query::fullOuterJoin($collection, 'product', [Query::on('product_uid', '$id')]), [3, 3, 5]],
] as $type => [$product, $totals]) {
$groups = $database->aggregate($orders, [$product, Query::count('*', 'total'), Query::groupBy(['product.category']), Query::select(['product.category']), Query::orderAsc('product.category')]);
$this->assertSame(['books', 'clothing', 'electronics'], \array_map(fn (array $group): mixed => $group['category'], $groups), $type);
$this->assertSame(['books', 'clothing', 'electronics'], \array_map(fn (array $group): mixed => $group['product.category'], $groups), $type);
$this->assertSame($totals, \array_map(fn (array $group): int => $this->intAttribute($group, 'total'), $groups), $type);
$this->assertSame(['category', 'total'], $this->sortedAttributeNames($groups[0]), $type);
$this->assertSame(['product.category', 'total'], $this->sortedAttributeNames($groups[0]), $type);
}

$database->deleteCollection($orders);
Expand Down Expand Up @@ -2827,6 +2827,89 @@ private function pageOneByOne(Database $database, string $collection, Query $sea
return $documents;
}

public function testAggregateRowsAreTypedNamedAndDecodedAsAReadReturnsThem(): void
{
$database = static::getDatabase();
if (! $database->getAdapter()->supports(Capability::Aggregations)) {
$this->expectNotToPerformAssertions();

return;
}

$collection = 'agg_typed_rows'.$this->getAggSuffix();
$database->createCollection(Collection::create(
id: $collection,
attributes: [
Attribute::string(key: 'label', size: 16, required: true),
Attribute::integer(key: 'quantity', required: true),
Attribute::double(key: 'price', required: true),
Attribute::boolean(key: 'active', required: true),
Attribute::datetime(key: 'shippedAt', required: true),
],
permissions: [Permission::create(Role::any()), Permission::read(Role::any())],
));
foreach ([
['first', 'a', 1, 1.5, true, '2023-06-01T00:00:00.000+00:00'],
['second', 'a', 2, 2.5, false, '2023-06-02T00:00:00.000+00:00'],
['third', 'b', 4, 3.5, true, '2023-06-01T00:00:00.000+00:00'],
] as [$id, $label, $quantity, $price, $active, $shippedAt]) {
$database->createDocument($collection, new Document([
'$id' => $id,
'label' => $label,
'quantity' => $quantity,
'price' => $price,
'active' => $active,
'shippedAt' => $shippedAt,
'$permissions' => [Permission::read(Role::any())],
]));
}

$totals = $database->aggregate($collection, [
Query::count('*', 'rows'),
Query::countDistinct('label', 'labels'),
Query::sum('quantity', 'units'),
Query::sum('price', 'revenue'),
Query::avg('quantity', 'mean'),
Query::min('quantity', 'fewest'),
Query::max('price', 'dearest'),
Query::min('shippedAt', 'firstShipped'),
]);
$this->assertCount(1, $totals);
$this->assertIsFloat($totals[0]['mean']);
$this->assertEqualsWithDelta(7 / 3, $totals[0]['mean'], 0.0001);
$this->assertSame([
'rows' => 3,
'labels' => 2,
'units' => 7,
'revenue' => 7.5,
'mean' => $totals[0]['mean'],
'fewest' => 1,
'dearest' => 3.5,
'firstShipped' => '2023-06-01T00:00:00.000+00:00',
], $totals[0]);

$this->assertSame(
[
['rows' => 1, 'active' => false, 'quantity' => 2, '$id' => 'second'],
['rows' => 1, 'active' => true, 'quantity' => 1, '$id' => 'first'],
['rows' => 1, 'active' => true, 'quantity' => 4, '$id' => 'third'],
],
$database->aggregate($collection, [
Query::count('*', 'rows'),
Query::groupBy(['active', 'quantity', '$id']),
Query::orderAsc('active'),
Query::orderAsc('quantity'),
]),
);

$this->assertSame(
[['count' => 2, 'sum_quantity' => 3, 'label' => 'a'], ['count' => 1, 'sum_quantity' => 4, 'label' => 'b']],
$database->aggregate($collection, [Query::count('*'), Query::sum('quantity'), Query::groupBy(['label']), Query::orderAsc('label')]),
);

$database->deleteCollection($collection);
}

public function testCountMatchesTheRowsOfGroupedDistinctAndAggregateReads(): void
{
$database = static::getDatabase();
Expand Down
6 changes: 3 additions & 3 deletions tests/e2e/Adapter/Scopes/DocumentTests.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10981,9 +10981,9 @@ public function testJoinedGroupSharingAMainGroupNameKeepsItsQualifiedName(): voi
$case.': a having on the qualified group',
);
$this->assertEquals(
[['orders' => 2, 'name' => 'apple'], ['orders' => 1, 'name' => 'banana']],
[['orders' => 2, 'it.name' => 'apple'], ['orders' => 1, 'it.name' => 'banana']],
$database->aggregate($orders, [$join, Query::count('*', 'orders'), Query::groupBy(['it.name']), Query::orderAsc('it.name')]),
$case.': a joined group alone keeps its bare name',
$case.': a joined group alone keeps its qualified name',
);
}
} finally {
Expand Down Expand Up @@ -11052,7 +11052,7 @@ public function testJoinWildcardSelectAndBareJoinedOrder(): void
$this->assertSame(['o2', 'o1', 'o3'], $ids($database->find($orders, [Query::fullOuterJoin($items, 'it', [Query::on('item', 'code')]), Query::orderDesc('price'), Query::orderAsc('$id')])));
$this->assertSame(['o2', 'o1', 'o3'], $ids($database->find($orders, [$item, Query::orderDesc('name'), Query::orderAsc('$id')])), 'a name the main collection declares reads the main table');
$this->assertEquals(
[['orders' => 1, 'code' => 'b'], ['orders' => 2, 'code' => 'a']],
[['orders' => 1, 'it.code' => 'b'], ['orders' => 2, 'it.code' => 'a']],
$database->aggregate($orders, [$item, Query::count('*', 'orders'), Query::groupBy(['code']), Query::orderDesc('code')]),
);

Expand Down
11 changes: 5 additions & 6 deletions tests/e2e/Adapter/Scopes/JoinTests.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
use Utopia\Database\Permission;
use Utopia\Database\Query;
use Utopia\Database\Role;
use Utopia\Database\Storage;
use Utopia\Query\Method;

trait JoinTests
Expand Down Expand Up @@ -6293,7 +6292,7 @@ private function fullOuterJoinGroups(array $rows): array
{
return \array_map(
fn (array $row): array => [
$row['category'] ?? null,
$row['b.category'] ?? null,
$this->intAttribute($row, 'rows'),
$this->intAttribute($row, 'total'),
$this->intAttribute($row, 'mainTotal'),
Expand Down Expand Up @@ -6483,7 +6482,7 @@ public function testJoinBareAggregateAttributeResolvesToTheDeclaringJoin(): void
]);
$totals = [];
foreach ($results as $result) {
$status = $result['status'];
$status = $result['j1.status'];
$this->assertIsString($status);
$totals[$status] = $this->intAttribute($result, 'total');
}
Expand Down Expand Up @@ -7849,7 +7848,7 @@ public function testJoinedInternalAttributesGroupTheJoinedRows(): void
foreach ([Document::ID, Document::SEQUENCE, Document::CREATED_AT, Document::UPDATED_AT, Document::PERMISSIONS] as $attribute) {
$total = 0;
foreach ($database->aggregate($customers, [$join, Query::count('*', 'rows'), Query::groupBy(['purchase.'.$attribute])]) as $group) {
$this->assertArrayHasKey(Storage::column($attribute), $group, $type.' grouped by purchase.'.$attribute);
$this->assertArrayHasKey('purchase.'.$attribute, $group, $type.' grouped by purchase.'.$attribute);
$total += $this->intAttribute($group, 'rows');
}
$this->assertSame(3, $total, $type.' grouped by purchase.'.$attribute);
Expand All @@ -7862,7 +7861,7 @@ public function testJoinedInternalAttributesGroupTheJoinedRows(): void
Query::groupBy(['purchase.$id']),
Query::orderAsc('purchase.$id'),
]);
$this->assertSame(['open', 'other', 'paid'], \array_map(static fn (array $group): mixed => $group[Storage::UID], $groups));
$this->assertSame(['open', 'other', 'paid'], \array_map(static fn (array $group): mixed => $group['purchase.$id'], $groups));
$this->assertSame([50, 7, 100], \array_map(fn (array $group): int => $this->intAttribute($group, 'total'), $groups));

$this->cleanupAggCollections($database, $collections);
Expand Down Expand Up @@ -8024,7 +8023,7 @@ public function testTenantIsReadOnlyWhereTheTablesHoldIt(): void
$grouped = $reads[$attribute]['groupBy']();
$this->assertCount(1, $grouped, $attribute);
$this->assertSame($rows, $this->intAttribute($grouped[0], 'total'), $attribute);
$value = $grouped[0][Storage::TENANT];
$value = $grouped[0][$attribute];
$this->assertIsScalar($value, $attribute);
$this->assertSame($tenant, (string) $value, $attribute);
}
Expand Down
Loading
Loading