fix(aggregate): answer bitwise aggregates signed and refuse integer sums past int64

src/Database/Adapter/Memory.php

@@ -1867,21 +1867,32 @@ public function sum(Document $collection, string $attribute, array $queries = []
             $rows = \array_slice($rows, 0, $max);
         }
 
-        $sum = 0;
+        $integers = 0;
+        $floats = 0.0;
         $isFloat = false;
         $column = $this->filter($attribute);
         foreach ($rows as $row) {
             $value = $row[$column] ?? null;
             if ($value === null || ! \is_numeric($value)) {
                 continue;
             }
-            if (\is_float($value)) {
+            if (\is_int($value) || (\is_string($value) && BigInt::isIntegerString($value))) {
+                $integers = BigInt::add($integers, $value);
+            } else {
                 $isFloat = true;
+                $floats += (float) $value;
             }
-            $sum += $value;
         }
 
-        return $isFloat ? (float) $sum : (int) $sum;
+        if ($isFloat) {
+            return $floats + (float) $integers;
+        }
+
+        if (\is_string($integers)) {
+            throw new LimitException('Value out of range');
+        }
+
+        return $integers;
     }
 
     #[\Override]

src/Database/Adapter/Mongo.php

@@ -2748,6 +2748,9 @@ public function sum(Document $collection, string $attribute, array $queries = []
             }
             /** @var mixed $sumTotal */
             $sumTotal = $sumFirstResult->total;
+            if (\is_float($sumTotal) && $this->sumsIntegers($collection, $attribute)) {
+                throw new LimitException('Value out of range');
+            }
             if (\is_int($sumTotal) || \is_float($sumTotal)) {
                 return $sumTotal;
             }
@@ -2758,6 +2761,21 @@ public function sum(Document $collection, string $attribute, array $queries = []
         }
     }
 
+    /**
+     * MongoDB sums integers into a long, and into a double only when the sum does not fit one.
+     */
+    private function sumsIntegers(Document $collection, string $attribute): bool
+    {
+        $definitions = $collection->getAttribute('attributes', []);
+        foreach (\is_array($definitions) ? $definitions : [] as $definition) {
+            if ($definition instanceof Document && $definition->getId() === $attribute) {
+                return \in_array($definition->getAttribute('type'), [Attribute::storedType(ColumnType::Integer), Attribute::storedType(ColumnType::BigInteger)], true);
+            }
+        }
+
+        return false;
+    }
+
     /**
      * @param  array<Document>  $documents
      * @return array<Document>

src/Database/Adapter/Redis.php

@@ -1536,22 +1536,31 @@ public function sum(Document $collection, string $attribute, array $queries = []
                 $documents = \array_slice($documents, 0, $max);
             }
 
-            $sum = 0;
+            $integers = 0;
+            $floats = 0.0;
             $isFloat = false;
             foreach ($documents as $document) {
                 $value = $this->resolveDocumentAttribute($document, $attribute);
-                if ($value === null) {
+                if ($value === null || ! \is_numeric($value)) {
                     continue;
                 }
-                if (\is_float($value)) {
+                if (\is_int($value) || (\is_string($value) && BigInt::isIntegerString($value))) {
+                    $integers = BigInt::add($integers, $value);
+                } else {
                     $isFloat = true;
+                    $floats += (float) $value;
                 }
-                if (\is_numeric($value)) {
-                    $sum += $value;
-                }
             }
 
-            return $isFloat ? (float) $sum : (int) $sum;
+            if ($isFloat) {
+                return $floats + (float) $integers;
+            }
+
+            if (\is_string($integers)) {
+                throw new LimitException('Value out of range');
+            }
+
+            return $integers;
         });
     }
 

src/Database/Adapter/SQL.php

@@ -36,6 +36,7 @@
 use Utopia\Database\Exception as DatabaseException;
 use Utopia\Database\Exception\Contention as ContentionException;
 use Utopia\Database\Exception\Duplicate as DuplicateException;
+use Utopia\Database\Exception\Limit as LimitException;
 use Utopia\Database\Exception\NotFound as NotFoundException;
 use Utopia\Database\Exception\Query as QueryException;
 use Utopia\Database\Exception\Timeout as TimeoutException;
@@ -2335,14 +2336,22 @@ private function countOf(array $row): int
 
     /**
      * The sum in a row, with SQLite's `INF`, `-INF` and `NAN` read as the doubles they name for Database::sum() to
-     * refuse.
+     * refuse. MariaDB, MySQL and PostgreSQL sum integers exactly past the range of a 64-bit integer, which is refused
+     * rather than read as an inexact double.
      *
      * @param  array<string, mixed>  $row
+     *
+     * @throws LimitException When an integer sum does not fit a 64-bit integer
      */
     private function sumOf(array $row): int|float
     {
         $sum = $row['sum'] ?? 0;
 
+        $whole = \is_string($sum) ? BigInt::whole($sum) : null;
+        if ($whole !== null && ! BigInt::fitsPhpInt($whole)) {
+            throw new LimitException('Value out of range');
+        }
+
         if (\is_numeric($sum)) {
             return $sum + 0;
         }

src/Database/Adapter/SQLite.php

@@ -1715,6 +1715,10 @@ protected function processException(PDOException $e): Exception
             return new DuplicateException('Attribute already exists', $e->getCode(), $e);
         }
 
+        if ($e->getCode() === 'HY000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1 && stripos($e->getMessage(), 'integer overflow') !== false) {
+            return new LimitException('Value out of range', $e->getCode(), $e);
+        }
+
         // Duplicate - SQLite uses various error codes for constraint violations:
         // - Error code 19 is SQLITE_CONSTRAINT (includes UNIQUE violations)
         // - Error code 1 is also used for some duplicate cases

src/Database/Aggregation.php

@@ -3,7 +3,9 @@
 namespace Utopia\Database;
 
 use Utopia\Database\Exception as DatabaseException;
+use Utopia\Database\Exception\Limit as LimitException;
 use Utopia\Database\Exception\Schema;
+use Utopia\Database\Validator\BigInt;
 use Utopia\Query\Method;
 use Utopia\Query\Schema\ColumnType;
 
@@ -21,6 +23,8 @@
         Method::BitXor,
     ];
 
+    private const float INTEGER_LIMIT = 9223372036854775808.0;
+
     private const array STATISTICS = [
         Method::Avg,
         Method::Stddev,
@@ -356,6 +360,7 @@ private static function scalar(ColumnType $type, mixed $value): mixed
         return match (true) {
             $type === ColumnType::Boolean => \is_bool($value) ? $value : \filter_var($value, FILTER_VALIDATE_BOOLEAN),
             $type === ColumnType::Id => \is_scalar($value) ? (string) $value : $value,
+            $type === ColumnType::BigInteger && \is_string($value) && BigInt::isIntegerString($value) && ! BigInt::fitsPhpInt($value) => $value,
             \in_array($type, self::INTEGERS, true) => self::integer($value),
             \in_array($type, self::FLOATS, true) => self::float($value),
             \in_array($type, self::DATETIMES, true) => \is_string($value) ? DateTime::formatTz($value) : $value,
@@ -364,23 +369,37 @@ private static function scalar(ColumnType $type, mixed $value): mixed
     }
 
     /**
-     * MariaDB sums integers into DECIMAL, so a whole value can arrive as "7.0000"; one past the platform's integers
-     * stays a float rather than wrap.
+     * An integer aggregate, or a group or bound of an integer attribute, as the integer it is. MariaDB sums integers
+     * into DECIMAL, so a whole value can arrive as "7.0000"; it and MySQL and PostgreSQL sum integers exactly past
+     * the range of a 64-bit integer, and a value that does not fit one is refused rather than answered inexactly.
+     *
+     * @throws LimitException When the value is a whole number past the range of a 64-bit integer
      */
     private static function integer(mixed $value): int|float
     {
         if (\is_int($value)) {
             return $value;
         }
 
-        $integer = \filter_var($value, FILTER_VALIDATE_INT);
-        if ($integer !== false) {
-            return $integer;
+        $whole = \is_string($value) ? BigInt::whole($value) : null;
+        if ($whole !== null) {
+            if (! BigInt::fitsPhpInt($whole)) {
+                throw new LimitException('Value out of range');
+            }
+
+            return (int) $whole;
         }
 
         $float = self::float($value);
+        if (\floor($float) !== $float) {
+            return $float;
+        }
+
+        if ($float < -self::INTEGER_LIMIT || $float >= self::INTEGER_LIMIT) {
+            throw new LimitException('Value out of range');
+        }
 
-        return \floor($float) === $float && \abs($float) < PHP_INT_MAX ? (int) $float : $float;
+        return (int) $float;
     }
 
     private static function float(mixed $value): float

src/Database/Builder/CompilesAggregates.php

@@ -0,0 +1,53 @@
+<?php
+
+namespace Utopia\Database\Builder;
+
+use Utopia\Query\Method;
+use Utopia\Query\Query;
+
+/**
+ * Compiles the aggregates MySQL and MariaDB answer differently from the other engines. An unaliased aggregate keeps
+ * the name the engine gave it.
+ *
+ * Both send a DOUBLE aggregate that overflowed to infinity as `0` over the text protocol, and turn an infinite or
+ * undefined variance into DBL_MAX or NULL under STDDEV. Adding zero runs the engine's float range check, which raises
+ * error 1690 for a value that is not finite, so each floating aggregate is compiled through one.
+ *
+ * Both answer BIT_AND, BIT_OR and BIT_XOR as BIGINT UNSIGNED, so a result with the sign bit set reads as an integer
+ * past the range of a signed one. Each is cast to SIGNED, which reads the same 64 bits as the two's complement
+ * integer PostgreSQL answers.
+ */
+trait CompilesAggregates
+{
+    private const array CHECKED_AGGREGATES = [Method::Sum, Method::Avg, Method::Variance, Method::VarPop, Method::VarSamp];
+
+    private const array BITWISE_AGGREGATES = [Method::BitAnd, Method::BitOr, Method::BitXor];
+
+    #[\Override]
+    public function compileAggregate(Query $query): string
+    {
+        $method = $query->getMethod();
+        $variance = match ($method) {
+            Method::Stddev => Method::Variance,
+            Method::StddevPop => Method::VarPop,
+            Method::StddevSamp => Method::VarSamp,
+            default => null,
+        };
+
+        $bitwise = \in_array($method, self::BITWISE_AGGREGATES, true);
+        if ($variance === null && ! $bitwise && ! \in_array($method, self::CHECKED_AGGREGATES, true)) {
+            return parent::compileAggregate($query);
+        }
+
+        $expression = parent::compileAggregate(new Query($method, $query->getAttribute()));
+        $compiled = match (true) {
+            $variance !== null => 'SQRT('.parent::compileAggregate(new Query($variance, $query->getAttribute())).' + 0)',
+            $bitwise => 'CAST('.$expression.' AS SIGNED)',
+            default => $expression.' + 0',
+        };
+
+        $alias = $query->getAlias();
+
+        return $compiled.' AS '.($alias === '' ? $this->quoteLiteral($expression) : $this->quote($alias));
+    }
+}

src/Database/Builder/MariaDB.php

@@ -6,14 +6,14 @@
 
 /**
  * The MariaDB builder, which also compiles filters on their own and prepares search terms, matches containsAll()
- * as 7.x did and raises a floating aggregate that overflowed.
+ * as 7.x did, raises a floating aggregate that overflowed and signs a bitwise one.
  */
 class MariaDB extends Base implements Filtering, Scoping
 {
     use CompilesFilters;
     use MatchesContainsAllPatterns;
     use PreparesSearchTerms;
-    use RaisesFloatOverflow;
+    use CompilesAggregates;
     use RefusesJoinedWrites;
     use ScopesCollections;
 }

src/Database/Builder/MySQL.php

@@ -6,14 +6,14 @@
 
 /**
  * The MySQL builder, which also compiles filters on their own and prepares search terms, matches containsAll()
- * as 7.x did and raises a floating aggregate that overflowed.
+ * as 7.x did, raises a floating aggregate that overflowed and signs a bitwise one.
  */
 class MySQL extends Base implements Filtering, Scoping
 {
     use CompilesFilters;
     use MatchesContainsAllPatterns;
     use PreparesSearchTerms;
-    use RaisesFloatOverflow;
+    use CompilesAggregates;
     use RefusesJoinedWrites;
     use ScopesCollections;
 }

src/Database/Trait/Documents.php

@@ -4760,7 +4760,8 @@ private function readCollection(Collection $collection, array $joins, array $joi
      * @return float|int The sum of the attribute values
      *
      * @throws DatabaseException
-     * @throws LimitException When the sum overflows the range of a double
+     * @throws LimitException When the sum overflows the range of a double, or an integer sum the range of a 64-bit
+     *                         integer
      * @throws QueryException When $max is not greater than 0, the adapter cannot read the queries, or the attribute is
      *                        not a column of the grouped or distinct rows
      */
@@ -5088,15 +5089,17 @@ private function pages(
      * Run aggregation queries (count, sum, avg, min, max, groupBy) and return one row per group. An aggregate without
      * an alias comes back under `<method>_<attribute>`, or `<method>` for count('*').
      *
-     * Counts and bitwise aggregates are integers, averages, deviations and variances floats, and a sum an integer over
-     * an integer attribute and a float otherwise; min() and max() are typed as their attribute. Each group is named as
+     * Counts and bitwise aggregates are integers, a bitwise one the signed 64-bit result on every engine, averages,
+     * deviations and variances floats, and a sum an integer over an integer attribute and a float otherwise; min() and
+     * max() are typed as their attribute. Each group is named as
      * a read names its attribute, a joined one qualified by its join alias, and typed and decoded as a read returns it.
      *
      * @param  array<Query>  $queries  At least one aggregate or groupBy query
      * @return list<array<string, mixed>>
      *
      * @throws DatabaseException
-     * @throws LimitException When a sum, average, deviation or variance overflows the range of a double
+     * @throws LimitException When a sum, average, deviation or variance overflows the range of a double, or an integer
+     *                         sum the range of a 64-bit integer
      * @throws QueryException When the queries hold no aggregate or groupBy
      */
     public function aggregate(string $collection, array $queries): array