fix(aggregates): refuse a floating aggregate that overflows a double

src/Database/Adapter/SQL.php

@@ -1940,15 +1940,15 @@ public function count(Document $collection, array $queries = [], ?int $max = nul
         }
 
         if ($otherQueries === []) {
-            return $this->countOf($this->fetchAggregate($collectionDoc, $name, $roles, 'COUNT(1)', '1', $max, Event::DocumentCount));
+            return $this->countOf($this->fetchAggregate($collectionDoc, $name, $roles, Query::count('1', 'sum'), '1', $max, Event::DocumentCount));
         }
 
         $otherQueries = $this->mainRowQueries($otherQueries, $collectionDoc);
         $builder = $this->newBuilder($name, $alias);
 
         $filters = $this->compileRowFilters($builder, $otherQueries);
         if ($filters !== null) {
-            return $this->countOf($this->fetchAggregate($collectionDoc, $name, $roles, 'COUNT(1)', '1', $max, Event::DocumentCount, $filters));
+            return $this->countOf($this->fetchAggregate($collectionDoc, $name, $roles, Query::count('1', 'sum'), '1', $max, Event::DocumentCount, $filters));
         }
 
         $this->applyFilters($builder, $otherQueries, $name, $alias);
@@ -2017,15 +2017,15 @@ public function sum(Document $collection, string $attribute, array $queries = []
         $column = $this->quote($attribute);
 
         if ($otherQueries === []) {
-            return $this->sumOf($this->fetchAggregate($collectionDoc, $name, $roles, "SUM({$column})", $column, $max, Event::DocumentSum));
+            return $this->sumOf($this->fetchAggregate($collectionDoc, $name, $roles, Query::sum($attribute, 'sum'), $column, $max, Event::DocumentSum));
         }
 
         $otherQueries = $this->mainRowQueries($otherQueries, $collectionDoc);
         $builder = $this->newBuilder($name, $alias);
 
         $filters = $this->compileRowFilters($builder, $otherQueries);
         if ($filters !== null) {
-            return $this->sumOf($this->fetchAggregate($collectionDoc, $name, $roles, "SUM({$column})", $column, $max, Event::DocumentSum, $filters));
+            return $this->sumOf($this->fetchAggregate($collectionDoc, $name, $roles, Query::sum($attribute, 'sum'), $column, $max, Event::DocumentSum, $filters));
         }
 
         $this->applyFilters($builder, $otherQueries, $name, $alias);
@@ -2184,7 +2184,7 @@ private function qualifySumSelect(string $attribute, array $joinTablePrefixes, D
      * @param  array<string>  $roles
      * @return array<string, mixed>
      */
-    private function fetchAggregate(Document $collection, string $name, array $roles, string $aggregate, string $column, ?int $max, Event $event, ?Condition $filters = null): array
+    private function fetchAggregate(Document $collection, string $name, array $roles, BaseQuery $aggregate, string $column, ?int $max, Event $event, ?Condition $filters = null): array
     {
         $alias = Query::DEFAULT_ALIAS;
         $conditions = [];
@@ -2212,11 +2212,11 @@ private function fetchAggregate(Document $collection, string $name, array $roles
             $rows .= ' WHERE '.\implode(' AND ', $conditions);
         }
 
-        $sum = $this->quote('sum');
+        $selection = $this->dialectBuilder()->compileAggregate($aggregate);
         if ($max === null) {
-            $sql = "SELECT {$aggregate} AS {$sum} FROM {$rows}";
+            $sql = "SELECT {$selection} FROM {$rows}";
         } else {
-            $sql = "SELECT {$aggregate} AS {$sum} FROM (SELECT {$column} FROM {$rows} LIMIT ?) AS {$this->quote('table_count')}";
+            $sql = "SELECT {$selection} FROM (SELECT {$column} FROM {$rows} LIMIT ?) AS {$this->quote('table_count')}";
             $bindings[] = $max;
         }
 
@@ -2252,17 +2252,25 @@ private function countOf(array $row): int
     }
 
     /**
+     * The sum in a row, with SQLite's `INF`, `-INF` and `NAN` read as the doubles they name for Database::sum() to
+     * refuse.
+     *
      * @param  array<string, mixed>  $row
      */
     private function sumOf(array $row): int|float
     {
         $sum = $row['sum'] ?? 0;
 
         if (\is_numeric($sum)) {
-            return \str_contains((string) $sum, '.') ? (float) $sum : (int) $sum;
+            return $sum + 0;
         }
 
-        return 0;
+        return match (\is_string($sum) ? \strtoupper($sum) : null) {
+            'INF' => \INF,
+            '-INF' => -\INF,
+            'NAN' => \NAN,
+            default => 0,
+        };
     }
 
     /**

src/Database/Builder/MariaDB.php

@@ -5,14 +5,15 @@
 use Utopia\Query\Builder\MariaDB as Base;
 
 /**
- * The MariaDB builder, which also compiles filters on their own and prepares search terms and matches containsAll()
- * as 7.x did.
+ * The MariaDB builder, which also compiles filters on their own and prepares search terms, matches containsAll()
+ * as 7.x did and raises a floating aggregate that overflowed.
  */
 class MariaDB extends Base implements Filtering, Scoping
 {
     use CompilesFilters;
     use MatchesContainsAllPatterns;
     use PreparesSearchTerms;
+    use RaisesFloatOverflow;
     use RefusesJoinedWrites;
     use ScopesCollections;
 }

src/Database/Builder/MySQL.php

@@ -5,14 +5,15 @@
 use Utopia\Query\Builder\MySQL as Base;
 
 /**
- * The MySQL builder, which also compiles filters on their own and prepares search terms and matches containsAll()
- * as 7.x did.
+ * The MySQL builder, which also compiles filters on their own and prepares search terms, matches containsAll()
+ * as 7.x did and raises a floating aggregate that overflowed.
  */
 class MySQL extends Base implements Filtering, Scoping
 {
     use CompilesFilters;
     use MatchesContainsAllPatterns;
     use PreparesSearchTerms;
+    use RaisesFloatOverflow;
     use RefusesJoinedWrites;
     use ScopesCollections;
 }

src/Database/Builder/RaisesFloatOverflow.php

@@ -0,0 +1,42 @@
+<?php
+
+namespace Utopia\Database\Builder;
+
+use Utopia\Query\Method;
+use Utopia\Query\Query;
+
+/**
+ * MySQL and MariaDB send a DOUBLE aggregate that overflowed to infinity as `0` over the text protocol, and turn an
+ * infinite or undefined variance into DBL_MAX or NULL under STDDEV. Adding zero runs the engine's float range check,
+ * which raises error 1690 for a value that is not finite, so each floating aggregate is compiled through one. An
+ * unaliased aggregate keeps the name the engine gave it.
+ */
+trait RaisesFloatOverflow
+{
+    private const array CHECKED_AGGREGATES = [Method::Sum, Method::Avg, Method::Variance, Method::VarPop, Method::VarSamp];
+
+    #[\Override]
+    public function compileAggregate(Query $query): string
+    {
+        $method = $query->getMethod();
+        $variance = match ($method) {
+            Method::Stddev => Method::Variance,
+            Method::StddevPop => Method::VarPop,
+            Method::StddevSamp => Method::VarSamp,
+            default => null,
+        };
+
+        if ($variance === null && ! \in_array($method, self::CHECKED_AGGREGATES, true)) {
+            return parent::compileAggregate($query);
+        }
+
+        $expression = parent::compileAggregate(new Query($method, $query->getAttribute()));
+        $checked = $variance === null
+            ? $expression.' + 0'
+            : 'SQRT('.parent::compileAggregate(new Query($variance, $query->getAttribute())).' + 0)';
+
+        $alias = $query->getAlias();
+
+        return $checked.' AS '.($alias === '' ? $this->quoteLiteral($expression) : $this->quote($alias));
+    }
+}

src/Database/Trait/Documents.php

@@ -73,6 +73,17 @@
 
 trait Documents
 {
+    private const array FLOATING_AGGREGATES = [
+        Method::Sum,
+        Method::Avg,
+        Method::Stddev,
+        Method::StddevPop,
+        Method::StddevSamp,
+        Method::Variance,
+        Method::VarPop,
+        Method::VarSamp,
+    ];
+
     private const string DOCUMENT_CACHE_ACTIVE_PREFIX = 'active:';
 
     private const string DOCUMENT_CACHE_BLOCKED_PREFIX = 'blocked:';
@@ -4611,6 +4622,7 @@ public function count(string $collection, array $queries = [], ?int $max = null)
      * @return float|int The sum of the attribute values
      *
      * @throws DatabaseException
+     * @throws LimitException When the sum overflows the range of a double
      * @throws QueryException When $max is not greater than 0
      */
     public function sum(string $collection, string $attribute, array $queries = [], ?int $max = null): float|int
@@ -4656,6 +4668,7 @@ public function sum(string $collection, string $attribute, array $queries = [],
 
         $getSum = fn () => $this->adapter->sum($collection, $attribute, $queries, $max);
         $sum = $skipAuth ? $this->authorization->skip($getSum) : $getSum();
+        self::assertFinite($sum);
 
         $listeners = $this->listens(Event::DocumentSum);
         if ($listeners !== []) {
@@ -4933,16 +4946,22 @@ private function pages(
      * @return list<array<string, mixed>>
      *
      * @throws DatabaseException
+     * @throws LimitException When a sum, average, deviation or variance overflows the range of a double
      * @throws QueryException When the queries hold no aggregate or groupBy
      */
     public function aggregate(string $collection, array $queries): array
     {
         $definition = $this->silent(fn () => $this->getCollection($collection));
         $queries = self::aliasAggregates($this->rewriteQueries(Event::DocumentAggregate, $definition, $queries));
 
+        $floating = self::floatingAggregates($queries);
         $rows = [];
         foreach ($this->fetch($definition, $queries, PermissionType::Read, true) as $row) {
-            $rows[] = $row->getArrayCopy();
+            $row = $row->getArrayCopy();
+            foreach ($floating as $alias) {
+                self::assertFinite($row[$alias] ?? null);
+            }
+            $rows[] = $row;
         }
 
         $listeners = $this->listens(Event::DocumentAggregate);
@@ -4953,6 +4972,41 @@ public function aggregate(string $collection, array $queries): array
         return $rows;
     }
 
+    /**
+     * @param  array<Query>  $queries  Each aggregate aliased by aliasAggregates()
+     * @return list<string>
+     */
+    private static function floatingAggregates(array $queries): array
+    {
+        $aliases = [];
+        foreach ($queries as $query) {
+            if (\in_array($query->getMethod(), self::FLOATING_AGGREGATES, true)) {
+                $aliases[] = $query->getAlias();
+            }
+        }
+
+        return $aliases;
+    }
+
+    /**
+     * SQLite, MongoDB and the adapters that sum in PHP answer a double that overflowed with infinity, which no caller
+     * can tell from a value, so it is refused as MariaDB, MySQL and PostgreSQL refuse it.
+     *
+     * @throws LimitException
+     */
+    private static function assertFinite(mixed $value): void
+    {
+        $finite = match (true) {
+            \is_float($value) => \is_finite($value),
+            \is_string($value) => \preg_match('/^[+-]?(inf|infinity|nan)$/i', $value) !== 1,
+            default => true,
+        };
+
+        if (! $finite) {
+            throw new LimitException('Value out of range');
+        }
+    }
+
     /**
      * @param  array<Query>  $queries
      * @return array<Query>

tests/e2e/Adapter/Scopes/AggregationTests.php

@@ -15,6 +15,7 @@
 use Utopia\Database\Collection;
 use Utopia\Database\Database;
 use Utopia\Database\Document;
+use Utopia\Database\Exception\Limit as LimitException;
 use Utopia\Database\Exception\NotFound as NotFoundException;
 use Utopia\Database\Exception\Query as QueryException;
 use Utopia\Database\Index;
@@ -1559,6 +1560,67 @@ public function testStddevAndVarianceArePopulationOnEveryAdapter(): void
         $database->deleteCollection('stat_contract');
     }
 
+    /**
+     * MariaDB and MySQL sent a double aggregate that overflowed as `0`, and MariaDB's STDDEV answered an overflowing
+     * variance with NULL. An aggregate past the range of a double is refused on every engine instead of answered with
+     * a wrong number.
+     */
+    public function testFloatAggregateOverflowIsRefused(): void
+    {
+        $database = static::getDatabase();
+        if (! $database->getAdapter()->supports(Capability::Aggregations)) {
+            $this->expectNotToPerformAssertions();
+
+            return;
+        }
+
+        $collection = 'agg_overflow'.$this->getAggSuffix();
+        $database->createCollection(Collection::create(
+            id: $collection,
+            attributes: [
+                Attribute::string(key: 'series', size: 20, required: true),
+                Attribute::double(key: 'reading', required: true),
+            ],
+            permissions: [Permission::create(Role::any()), Permission::read(Role::any())],
+        ));
+        foreach ([['huge', 1e308], ['huge', 1e308], ['spread', 1e308], ['spread', -1e308], ['spread', 0.0], ['single', 1e308]] as [$series, $reading]) {
+            $database->createDocument($collection, new Document([
+                'series' => $series,
+                'reading' => $reading,
+                '$permissions' => [Permission::read(Role::any())],
+            ]));
+        }
+
+        $overflows = [
+            [Query::equal('series', ['huge']), Query::sum('reading', 'result')],
+            [Query::equal('series', ['huge']), Query::avg('reading', 'result')],
+            [Query::sum('reading', 'result'), Query::groupBy(['series'])],
+        ];
+        if (! $database->getAdapter() instanceof SQLite) {
+            $overflows[] = [Query::equal('series', ['spread']), Query::stddevPop('reading', 'result')];
+            $overflows[] = [Query::equal('series', ['spread']), Query::varSamp('reading', 'result')];
+        }
+
+        foreach ($overflows as $queries) {
+            try {
+                $database->aggregate($collection, $queries);
+                $this->fail('An aggregate that overflows a double is refused');
+            } catch (LimitException) {
+                $this->addToAssertionCount(1);
+            }
+        }
+
+        $single = $database->aggregate($collection, [
+            Query::equal('series', ['single']),
+            Query::sum('reading', 'total'),
+            Query::avg('reading', 'mean'),
+        ]);
+        $this->assertEqualsWithDelta(1e308, $this->numericAttribute($single[0], 'total'), 1e293);
+        $this->assertEqualsWithDelta(1e308, $this->numericAttribute($single[0], 'mean'), 1e293);
+
+        $database->deleteCollection($collection);
+    }
+
     public function testStddevPopOfPrice(): void
     {
         $database = static::getDatabase();

tests/e2e/Adapter/Scopes/DocumentTests.php

@@ -2830,6 +2830,44 @@ public function testSum(): void
         $this->getDatabase()->getAuthorization()->addRole('user:x');
     }
 
+    /**
+     * MariaDB and MySQL sent a double sum that overflowed as `0`, and SQLite, MongoDB and the adapters that sum in
+     * PHP answered infinity. A sum past the range of a double is refused on every adapter instead.
+     */
+    public function testSumOverflowIsRefused(): void
+    {
+        $database = $this->getDatabase();
+        $collection = 'sum_overflow_'.\substr(\uniqid(), -6);
+        $database->createCollection(Collection::create(
+            id: $collection,
+            attributes: [
+                Attribute::string(key: 'series', size: 20, required: true),
+                Attribute::double(key: 'reading', required: true),
+            ],
+            permissions: [Permission::create(Role::any()), Permission::read(Role::any())],
+        ));
+        foreach ([['huge', 1e308], ['huge', 1e308], ['single', 1e308]] as [$series, $reading]) {
+            $database->createDocument($collection, new Document([
+                'series' => $series,
+                'reading' => $reading,
+                '$permissions' => [Permission::read(Role::any())],
+            ]));
+        }
+
+        foreach ([[[], null], [[Query::equal('series', ['huge'])], null], [[Query::equal('series', ['huge'])], 5], [[Query::greaterThan('reading', 0)], null]] as [$queries, $max]) {
+            try {
+                $database->sum($collection, 'reading', $queries, $max);
+                $this->fail('A sum that overflows a double is refused');
+            } catch (LimitException) {
+                $this->addToAssertionCount(1);
+            }
+        }
+
+        $this->assertEqualsWithDelta(1e308, $database->sum($collection, 'reading', [Query::equal('series', ['single'])]), 1e293);
+
+        $database->deleteCollection($collection);
+    }
+
     public function testUpdateDocument(): void
     {
         $document = $this->initDocumentsFixture();