From 69f114226bac4b930c3d3da538ab56ad7546b4ee Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 16 Oct 2026 14:39:28 +1300 Subject: [PATCH] fix(aggregate): answer bitwise aggregates signed and refuse integer sums past int64 MySQL and MariaDB answer BIT_AND, BIT_OR and BIT_XOR as BIGINT UNSIGNED, so a result with the sign bit set (any BIT_OR or BIT_XOR touching a negative value) read as an integer past PHP_INT_MAX and Aggregation::integer() turned it into a lossy float: rows 10 and -5 answered bitOr 1.8446744073709552e19 where PostgreSQL answered -5. Their builders now compile each as CAST(... AS SIGNED), which reads the same 64 bits as the two's complement integer PostgreSQL answers, so every engine returns the same int. The cast is done in SQL rather than in PHP so the value never leaves the engine unsigned. SQLite has no bitwise aggregates and keeps refusing them. MariaDB, MySQL and PostgreSQL sum integers exactly into DECIMAL or NUMERIC, so a sum past the range of a 64-bit integer came back as an inexact float (two INT64_MAX rows: 1.8446744073709552e19). SQLite raised a raw PDOException ("integer overflow"), Memory and Redis wrapped or warned on the int cast, and MongoDB returned the double it falls back to. As with a double that overflows (#1018, #1019), an integer sum that does not fit is refused with Exception\Limit instead of answered with a wrong number: - Aggregation::integer() reads the engine's exact digits through BigInt::whole() and BigInt::fitsPhpInt(), so the range check is exact, not a float comparison, and covers a grouped sum where one group overflows. - SQL::sumOf() does the same for sum(). - SQLite maps "integer overflow" to Exception\Limit. SQLite also refuses a sum whose running total overflows even when the final one fits. - Memory and Redis sum integers with BigInt::add() and refuse a total past int64. - MongoDB answers an integer sum that does not fit a long as a double, so a double sum of an integer attribute is refused. A min() or max() of an unsigned big integer past PHP_INT_MAX keeps its digits as a string, as a read returns it, rather than tripping the new range check or turning into a float. RaisesFloatOverflow becomes CompilesAggregates now that it compiles the bitwise aggregates too. Co-Authored-By: Claude Opus 5.5 --- src/Database/Adapter/Memory.php | 19 +- src/Database/Adapter/Mongo.php | 18 ++ src/Database/Adapter/Redis.php | 23 ++- src/Database/Adapter/SQL.php | 11 +- src/Database/Adapter/SQLite.php | 4 + src/Database/Aggregation.php | 31 +++- src/Database/Builder/CompilesAggregates.php | 53 ++++++ src/Database/Builder/MariaDB.php | 4 +- src/Database/Builder/MySQL.php | 4 +- src/Database/Builder/RaisesFloatOverflow.php | 42 ----- src/Database/Trait/Documents.php | 11 +- src/Database/Validator/BigInt.php | 17 ++ tests/e2e/Adapter/Scopes/AggregationTests.php | 170 ++++++++++++++++++ tests/e2e/Adapter/Scopes/DocumentTests.php | 45 +++++ tests/unit/Adapter/SumResultTest.php | 24 ++- tests/unit/AggregationTest.php | 45 +++++ tests/unit/Builder/FloatOverflowTest.php | 9 +- tests/unit/EmptySetAggregateContractTest.php | 13 +- tests/unit/Joins/AggregateMinorsTest.php | 13 +- tests/unit/Validator/BigIntTest.php | 23 +++ 20 files changed, 496 insertions(+), 83 deletions(-) create mode 100644 src/Database/Builder/CompilesAggregates.php delete mode 100644 src/Database/Builder/RaisesFloatOverflow.php diff --git a/src/Database/Adapter/Memory.php b/src/Database/Adapter/Memory.php index 792cfe8bb..d44e929f4 100644 --- a/src/Database/Adapter/Memory.php +++ b/src/Database/Adapter/Memory.php @@ -1867,7 +1867,8 @@ public function sum(Document $collection, string $attribute, array $queries = [] $rows = \array_slice($rows, 0, $max); } - $sum = 0; + $integers = 0; + $floats = 0.0; $isFloat = false; $column = $this->filter($attribute); foreach ($rows as $row) { @@ -1875,13 +1876,23 @@ public function sum(Document $collection, string $attribute, array $queries = [] if ($value === null || ! \is_numeric($value)) { continue; } - if (\is_float($value)) { + if (\is_int($value) || (\is_string($value) && BigInt::isIntegerString($value))) { + $integers = BigInt::add($integers, $value); + } else { $isFloat = true; + $floats += (float) $value; } - $sum += $value; } - return $isFloat ? (float) $sum : (int) $sum; + if ($isFloat) { + return $floats + (float) $integers; + } + + if (\is_string($integers)) { + throw new LimitException('Value out of range'); + } + + return $integers; } #[\Override] diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 687b9dd10..afd8aeaf7 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -2748,6 +2748,9 @@ public function sum(Document $collection, string $attribute, array $queries = [] } /** @var mixed $sumTotal */ $sumTotal = $sumFirstResult->total; + if (\is_float($sumTotal) && $this->sumsIntegers($collection, $attribute)) { + throw new LimitException('Value out of range'); + } if (\is_int($sumTotal) || \is_float($sumTotal)) { return $sumTotal; } @@ -2758,6 +2761,21 @@ public function sum(Document $collection, string $attribute, array $queries = [] } } + /** + * MongoDB sums integers into a long, and into a double only when the sum does not fit one. + */ + private function sumsIntegers(Document $collection, string $attribute): bool + { + $definitions = $collection->getAttribute('attributes', []); + foreach (\is_array($definitions) ? $definitions : [] as $definition) { + if ($definition instanceof Document && $definition->getId() === $attribute) { + return \in_array($definition->getAttribute('type'), [Attribute::storedType(ColumnType::Integer), Attribute::storedType(ColumnType::BigInteger)], true); + } + } + + return false; + } + /** * @param array $documents * @return array diff --git a/src/Database/Adapter/Redis.php b/src/Database/Adapter/Redis.php index f3d87e607..efcc16259 100644 --- a/src/Database/Adapter/Redis.php +++ b/src/Database/Adapter/Redis.php @@ -1536,22 +1536,31 @@ public function sum(Document $collection, string $attribute, array $queries = [] $documents = \array_slice($documents, 0, $max); } - $sum = 0; + $integers = 0; + $floats = 0.0; $isFloat = false; foreach ($documents as $document) { $value = $this->resolveDocumentAttribute($document, $attribute); - if ($value === null) { + if ($value === null || ! \is_numeric($value)) { continue; } - if (\is_float($value)) { + if (\is_int($value) || (\is_string($value) && BigInt::isIntegerString($value))) { + $integers = BigInt::add($integers, $value); + } else { $isFloat = true; + $floats += (float) $value; } - if (\is_numeric($value)) { - $sum += $value; - } } - return $isFloat ? (float) $sum : (int) $sum; + if ($isFloat) { + return $floats + (float) $integers; + } + + if (\is_string($integers)) { + throw new LimitException('Value out of range'); + } + + return $integers; }); } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 704515dd0..36ef14a21 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -36,6 +36,7 @@ use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Contention as ContentionException; use Utopia\Database\Exception\Duplicate as DuplicateException; +use Utopia\Database\Exception\Limit as LimitException; use Utopia\Database\Exception\NotFound as NotFoundException; use Utopia\Database\Exception\Query as QueryException; use Utopia\Database\Exception\Timeout as TimeoutException; @@ -2335,14 +2336,22 @@ private function countOf(array $row): int /** * The sum in a row, with SQLite's `INF`, `-INF` and `NAN` read as the doubles they name for Database::sum() to - * refuse. + * refuse. MariaDB, MySQL and PostgreSQL sum integers exactly past the range of a 64-bit integer, which is refused + * rather than read as an inexact double. * * @param array $row + * + * @throws LimitException When an integer sum does not fit a 64-bit integer */ private function sumOf(array $row): int|float { $sum = $row['sum'] ?? 0; + $whole = \is_string($sum) ? BigInt::whole($sum) : null; + if ($whole !== null && ! BigInt::fitsPhpInt($whole)) { + throw new LimitException('Value out of range'); + } + if (\is_numeric($sum)) { return $sum + 0; } diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index e9873a0a4..238dbe27e 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -1715,6 +1715,10 @@ protected function processException(PDOException $e): Exception return new DuplicateException('Attribute already exists', $e->getCode(), $e); } + if ($e->getCode() === 'HY000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1 && stripos($e->getMessage(), 'integer overflow') !== false) { + return new LimitException('Value out of range', $e->getCode(), $e); + } + // Duplicate - SQLite uses various error codes for constraint violations: // - Error code 19 is SQLITE_CONSTRAINT (includes UNIQUE violations) // - Error code 1 is also used for some duplicate cases diff --git a/src/Database/Aggregation.php b/src/Database/Aggregation.php index 28b9269f8..33890a076 100644 --- a/src/Database/Aggregation.php +++ b/src/Database/Aggregation.php @@ -3,7 +3,9 @@ namespace Utopia\Database; use Utopia\Database\Exception as DatabaseException; +use Utopia\Database\Exception\Limit as LimitException; use Utopia\Database\Exception\Schema; +use Utopia\Database\Validator\BigInt; use Utopia\Query\Method; use Utopia\Query\Schema\ColumnType; @@ -21,6 +23,8 @@ Method::BitXor, ]; + private const float INTEGER_LIMIT = 9223372036854775808.0; + private const array STATISTICS = [ Method::Avg, Method::Stddev, @@ -356,6 +360,7 @@ private static function scalar(ColumnType $type, mixed $value): mixed return match (true) { $type === ColumnType::Boolean => \is_bool($value) ? $value : \filter_var($value, FILTER_VALIDATE_BOOLEAN), $type === ColumnType::Id => \is_scalar($value) ? (string) $value : $value, + $type === ColumnType::BigInteger && \is_string($value) && BigInt::isIntegerString($value) && ! BigInt::fitsPhpInt($value) => $value, \in_array($type, self::INTEGERS, true) => self::integer($value), \in_array($type, self::FLOATS, true) => self::float($value), \in_array($type, self::DATETIMES, true) => \is_string($value) ? DateTime::formatTz($value) : $value, @@ -364,8 +369,11 @@ private static function scalar(ColumnType $type, mixed $value): mixed } /** - * MariaDB sums integers into DECIMAL, so a whole value can arrive as "7.0000"; one past the platform's integers - * stays a float rather than wrap. + * An integer aggregate, or a group or bound of an integer attribute, as the integer it is. MariaDB sums integers + * into DECIMAL, so a whole value can arrive as "7.0000"; it and MySQL and PostgreSQL sum integers exactly past + * the range of a 64-bit integer, and a value that does not fit one is refused rather than answered inexactly. + * + * @throws LimitException When the value is a whole number past the range of a 64-bit integer */ private static function integer(mixed $value): int|float { @@ -373,14 +381,25 @@ private static function integer(mixed $value): int|float return $value; } - $integer = \filter_var($value, FILTER_VALIDATE_INT); - if ($integer !== false) { - return $integer; + $whole = \is_string($value) ? BigInt::whole($value) : null; + if ($whole !== null) { + if (! BigInt::fitsPhpInt($whole)) { + throw new LimitException('Value out of range'); + } + + return (int) $whole; } $float = self::float($value); + if (\floor($float) !== $float) { + return $float; + } + + if ($float < -self::INTEGER_LIMIT || $float >= self::INTEGER_LIMIT) { + throw new LimitException('Value out of range'); + } - return \floor($float) === $float && \abs($float) < PHP_INT_MAX ? (int) $float : $float; + return (int) $float; } private static function float(mixed $value): float diff --git a/src/Database/Builder/CompilesAggregates.php b/src/Database/Builder/CompilesAggregates.php new file mode 100644 index 000000000..cb4cb49db --- /dev/null +++ b/src/Database/Builder/CompilesAggregates.php @@ -0,0 +1,53 @@ +getMethod(); + $variance = match ($method) { + Method::Stddev => Method::Variance, + Method::StddevPop => Method::VarPop, + Method::StddevSamp => Method::VarSamp, + default => null, + }; + + $bitwise = \in_array($method, self::BITWISE_AGGREGATES, true); + if ($variance === null && ! $bitwise && ! \in_array($method, self::CHECKED_AGGREGATES, true)) { + return parent::compileAggregate($query); + } + + $expression = parent::compileAggregate(new Query($method, $query->getAttribute())); + $compiled = match (true) { + $variance !== null => 'SQRT('.parent::compileAggregate(new Query($variance, $query->getAttribute())).' + 0)', + $bitwise => 'CAST('.$expression.' AS SIGNED)', + default => $expression.' + 0', + }; + + $alias = $query->getAlias(); + + return $compiled.' AS '.($alias === '' ? $this->quoteLiteral($expression) : $this->quote($alias)); + } +} diff --git a/src/Database/Builder/MariaDB.php b/src/Database/Builder/MariaDB.php index 745b7092f..6475ddcde 100644 --- a/src/Database/Builder/MariaDB.php +++ b/src/Database/Builder/MariaDB.php @@ -6,14 +6,14 @@ /** * The MariaDB builder, which also compiles filters on their own and prepares search terms, matches containsAll() - * as 7.x did and raises a floating aggregate that overflowed. + * as 7.x did, raises a floating aggregate that overflowed and signs a bitwise one. */ class MariaDB extends Base implements Filtering, Scoping { use CompilesFilters; use MatchesContainsAllPatterns; use PreparesSearchTerms; - use RaisesFloatOverflow; + use CompilesAggregates; use RefusesJoinedWrites; use ScopesCollections; } diff --git a/src/Database/Builder/MySQL.php b/src/Database/Builder/MySQL.php index 4ac3e57cc..0ff80e728 100644 --- a/src/Database/Builder/MySQL.php +++ b/src/Database/Builder/MySQL.php @@ -6,14 +6,14 @@ /** * The MySQL builder, which also compiles filters on their own and prepares search terms, matches containsAll() - * as 7.x did and raises a floating aggregate that overflowed. + * as 7.x did, raises a floating aggregate that overflowed and signs a bitwise one. */ class MySQL extends Base implements Filtering, Scoping { use CompilesFilters; use MatchesContainsAllPatterns; use PreparesSearchTerms; - use RaisesFloatOverflow; + use CompilesAggregates; use RefusesJoinedWrites; use ScopesCollections; } diff --git a/src/Database/Builder/RaisesFloatOverflow.php b/src/Database/Builder/RaisesFloatOverflow.php deleted file mode 100644 index bdd20c8ce..000000000 --- a/src/Database/Builder/RaisesFloatOverflow.php +++ /dev/null @@ -1,42 +0,0 @@ -getMethod(); - $variance = match ($method) { - Method::Stddev => Method::Variance, - Method::StddevPop => Method::VarPop, - Method::StddevSamp => Method::VarSamp, - default => null, - }; - - if ($variance === null && ! \in_array($method, self::CHECKED_AGGREGATES, true)) { - return parent::compileAggregate($query); - } - - $expression = parent::compileAggregate(new Query($method, $query->getAttribute())); - $checked = $variance === null - ? $expression.' + 0' - : 'SQRT('.parent::compileAggregate(new Query($variance, $query->getAttribute())).' + 0)'; - - $alias = $query->getAlias(); - - return $checked.' AS '.($alias === '' ? $this->quoteLiteral($expression) : $this->quote($alias)); - } -} diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 4cd80ed09..b95120b54 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -4760,7 +4760,8 @@ private function readCollection(Collection $collection, array $joins, array $joi * @return float|int The sum of the attribute values * * @throws DatabaseException - * @throws LimitException When the sum overflows the range of a double + * @throws LimitException When the sum overflows the range of a double, or an integer sum the range of a 64-bit + * integer * @throws QueryException When $max is not greater than 0, the adapter cannot read the queries, or the attribute is * not a column of the grouped or distinct rows */ @@ -5088,15 +5089,17 @@ private function pages( * Run aggregation queries (count, sum, avg, min, max, groupBy) and return one row per group. An aggregate without * an alias comes back under `_`, or `` for count('*'). * - * Counts and bitwise aggregates are integers, averages, deviations and variances floats, and a sum an integer over - * an integer attribute and a float otherwise; min() and max() are typed as their attribute. Each group is named as + * Counts and bitwise aggregates are integers, a bitwise one the signed 64-bit result on every engine, averages, + * deviations and variances floats, and a sum an integer over an integer attribute and a float otherwise; min() and + * max() are typed as their attribute. Each group is named as * a read names its attribute, a joined one qualified by its join alias, and typed and decoded as a read returns it. * * @param array $queries At least one aggregate or groupBy query * @return list> * * @throws DatabaseException - * @throws LimitException When a sum, average, deviation or variance overflows the range of a double + * @throws LimitException When a sum, average, deviation or variance overflows the range of a double, or an integer + * sum the range of a 64-bit integer * @throws QueryException When the queries hold no aggregate or groupBy */ public function aggregate(string $collection, array $queries): array diff --git a/src/Database/Validator/BigInt.php b/src/Database/Validator/BigInt.php index d7f31535f..3f95dfd36 100644 --- a/src/Database/Validator/BigInt.php +++ b/src/Database/Validator/BigInt.php @@ -67,6 +67,23 @@ public static function isIntegerString(string $value, bool $signed = true): bool return \preg_match($signed ? '/^-?\d+$/' : '/^\d+$/', $value) === 1; } + /** + * The integer a whole number written in decimal names, as "7" or as MariaDB writes a DECIMAL sum, "7.0000"; null + * for any other value. + */ + public static function whole(string $value): ?string + { + $dot = \strpos($value, '.'); + if ($dot !== false) { + if (\trim(\substr($value, $dot + 1), '0') !== '') { + return null; + } + $value = \substr($value, 0, $dot); + } + + return self::isIntegerString($value) ? $value : null; + } + public static function fitsPhpInt(string $value, bool $signed = true): bool { if (!self::isIntegerString($value, $signed)) { diff --git a/tests/e2e/Adapter/Scopes/AggregationTests.php b/tests/e2e/Adapter/Scopes/AggregationTests.php index 404a26be0..2a1f53708 100644 --- a/tests/e2e/Adapter/Scopes/AggregationTests.php +++ b/tests/e2e/Adapter/Scopes/AggregationTests.php @@ -1736,6 +1736,176 @@ public function testBitXorOfIntegerColumn(): void $database->deleteCollection('bit_xor'); } + /** + * MariaDB and MySQL answer BIT_AND, BIT_OR and BIT_XOR as BIGINT UNSIGNED, so a result with the sign bit set came + * back as a float past PHP_INT_MAX where PostgreSQL answered the signed integer. Every engine answers the signed + * 64-bit result; SQLite has no bitwise aggregates and refuses them. + */ + public function testBitwiseAggregatesAreSigned64BitIntegers(): void + { + $database = static::getDatabase(); + + $collection = 'agg_bitwise_sign'.$this->getAggSuffix(); + $database->createCollection(Collection::create( + id: $collection, + attributes: [ + Attribute::string(key: 'series', size: 20, required: true), + Attribute::bigInteger(key: 'amount'), + ], + permissions: [Permission::create(Role::any()), Permission::read(Role::any())], + )); + $rows = [ + ['positive', 10], ['positive', 3], + ['negative', -5], ['negative', -12], + ['mixed', 10], ['mixed', -5], + ['extremes', PHP_INT_MAX], ['extremes', PHP_INT_MIN], + ['nulls', null], ['nulls', null], + ]; + foreach ($rows as [$series, $amount]) { + $database->createDocument($collection, new Document([ + 'series' => $series, + 'amount' => $amount, + '$permissions' => [Permission::read(Role::any())], + ])); + } + + $bitwise = [Query::bitAnd('amount', 'all_bits'), Query::bitOr('amount', 'any_bits'), Query::bitXor('amount', 'odd_bits')]; + + if ($this->refusesAggregation($database, $collection, $bitwise)) { + return; + } + + if ($database->getAdapter() instanceof SQLite) { + try { + $database->aggregate($collection, $bitwise); + $this->fail('SQLite has no bitwise aggregates'); + } catch (QueryException $exception) { + $this->assertSame('Aggregate bitAnd is not supported by this adapter', $exception->getMessage()); + } + $database->deleteCollection($collection); + + return; + } + + $expected = [ + 'extremes' => ['all_bits' => 0, 'any_bits' => -1, 'odd_bits' => -1], + 'mixed' => ['all_bits' => 10, 'any_bits' => -5, 'odd_bits' => -15], + 'negative' => ['all_bits' => -16, 'any_bits' => -1, 'odd_bits' => 15], + 'nulls' => ['all_bits' => null, 'any_bits' => null, 'odd_bits' => null], + 'positive' => ['all_bits' => 2, 'any_bits' => 11, 'odd_bits' => 9], + ]; + foreach ($expected as $series => $results) { + $rows = $database->aggregate($collection, [Query::equal('series', [$series]), ...$bitwise]); + $this->assertCount(1, $rows); + foreach ($results as $alias => $result) { + $this->assertSame($result, $rows[0][$alias], "{$alias} of the {$series} series"); + } + } + + $empty = $database->aggregate($collection, [Query::equal('series', ['none']), ...$bitwise]); + $this->assertSame(['all_bits' => null, 'any_bits' => null, 'odd_bits' => null], $empty[0]); + + $grouped = $database->aggregate($collection, [...$bitwise, Query::groupBy(['series']), Query::orderAsc('series')]); + $this->assertSame(\array_keys($expected), \array_column($grouped, 'series')); + foreach ($grouped as $row) { + $series = $row['series']; + $this->assertIsString($series); + foreach ($expected[$series] as $alias => $result) { + $this->assertSame($result, $row[$alias], "{$alias} of the {$series} group"); + } + } + + $database->deleteCollection($collection); + } + + /** + * MariaDB, MySQL and PostgreSQL sum integers exactly past the range of a 64-bit integer, which came back as an + * inexact float: two of PHP_INT_MAX summed to 1.8446744073709552E+19. An integer sum that does not fit is refused + * on every engine, as a sum past the range of a double is, and one that fits is answered exactly. + */ + public function testIntegerAggregateSumOverflowIsRefused(): void + { + $database = static::getDatabase(); + + $collection = 'agg_int_overflow'.$this->getAggSuffix(); + $database->createCollection(Collection::create( + id: $collection, + attributes: [ + Attribute::string(key: 'series', size: 20, required: true), + Attribute::bigInteger(key: 'amount', required: true), + ], + permissions: [Permission::create(Role::any()), Permission::read(Role::any())], + )); + $rows = [ + ['above', PHP_INT_MAX], ['above', PHP_INT_MAX], + ['below', PHP_INT_MIN], ['below', -1], + ['top', PHP_INT_MAX - 1], ['top', 1], + ['bottom', PHP_INT_MIN + 1], ['bottom', -1], + ]; + foreach ($rows as [$series, $amount]) { + $database->createDocument($collection, new Document([ + 'series' => $series, + 'amount' => $amount, + '$permissions' => [Permission::read(Role::any())], + ])); + } + + if ($this->refusesAggregation($database, $collection, [Query::sum('amount', 'total')])) { + return; + } + + $overflows = [ + [Query::equal('series', ['above']), Query::sum('amount', 'total')], + [Query::equal('series', ['below']), Query::sum('amount', 'total')], + [Query::sum('amount', 'total'), Query::groupBy(['series'])], + ]; + foreach ($overflows as $queries) { + try { + $database->aggregate($collection, $queries); + $this->fail('An integer sum past the range of a 64-bit integer is refused'); + } catch (LimitException) { + $this->addToAssertionCount(1); + } + } + + $top = $database->aggregate($collection, [Query::equal('series', ['top']), Query::sum('amount', 'total')]); + $this->assertSame(PHP_INT_MAX, $top[0]['total']); + $bottom = $database->aggregate($collection, [Query::equal('series', ['bottom']), Query::sum('amount', 'total')]); + $this->assertSame(PHP_INT_MIN, $bottom[0]['total']); + + $grouped = $database->aggregate($collection, [ + Query::equal('series', ['top', 'bottom']), + Query::sum('amount', 'total'), + Query::groupBy(['series']), + Query::orderAsc('series'), + ]); + $this->assertSame([['total' => PHP_INT_MIN, 'series' => 'bottom'], ['total' => PHP_INT_MAX, 'series' => 'top']], $grouped); + + $database->deleteCollection($collection); + } + + /** + * An adapter without aggregations refuses the queries as an invalid query, and the collection is dropped. + * + * @param array $queries + */ + private function refusesAggregation(Database $database, string $collection, array $queries): bool + { + if ($database->getAdapter()->supports(Capability::Aggregations)) { + return false; + } + + try { + $database->aggregate($collection, $queries); + $this->fail('An adapter without aggregations refuses an aggregate'); + } catch (QueryException) { + $this->addToAssertionCount(1); + } + $database->deleteCollection($collection); + + return true; + } + private function assertRejectedAsQueryShape(callable $call, string $message): void { $error = null; diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 5feafb992..274018761 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -2896,6 +2896,51 @@ public function testNegativeSumOverflowIsRefused(): void $database->deleteCollection($collection); } + /** + * An integer sum past the range of a 64-bit integer came back as an inexact float, or on Memory and Redis as a + * wrapped integer. It is refused on every adapter, and one that fits is answered exactly. + */ + public function testIntegerSumOverflowIsRefused(): void + { + $database = $this->getDatabase(); + $collection = 'sum_int_overflow_'.\substr(\uniqid(), -6); + $database->createCollection(Collection::create( + id: $collection, + attributes: [ + Attribute::string(key: 'series', size: 20, required: true), + Attribute::bigInteger(key: 'amount', required: true), + ], + permissions: [Permission::create(Role::any()), Permission::read(Role::any())], + )); + $rows = [ + ['above', PHP_INT_MAX], ['above', PHP_INT_MAX], + ['below', PHP_INT_MIN], ['below', -1], + ['top', PHP_INT_MAX - 1], ['top', 1], + ['bottom', PHP_INT_MIN + 1], ['bottom', -1], + ]; + foreach ($rows as [$series, $amount]) { + $database->createDocument($collection, new Document([ + 'series' => $series, + 'amount' => $amount, + '$permissions' => [Permission::read(Role::any())], + ])); + } + + foreach ([[[Query::equal('series', ['above'])], null], [[Query::equal('series', ['above'])], 5], [[Query::equal('series', ['below'])], null]] as [$queries, $max]) { + try { + $sum = $database->sum($collection, 'amount', $queries, $max); + $this->fail('An integer sum past the range of a 64-bit integer is refused, got '.\var_export($sum, true)); + } catch (LimitException) { + $this->addToAssertionCount(1); + } + } + + $this->assertSame(PHP_INT_MAX, $database->sum($collection, 'amount', [Query::equal('series', ['top'])])); + $this->assertSame(PHP_INT_MIN, $database->sum($collection, 'amount', [Query::equal('series', ['bottom'])])); + + $database->deleteCollection($collection); + } + public function testUpdateDocument(): void { $document = $this->initDocumentsFixture(); diff --git a/tests/unit/Adapter/SumResultTest.php b/tests/unit/Adapter/SumResultTest.php index f14dd3e8d..36f57b30b 100644 --- a/tests/unit/Adapter/SumResultTest.php +++ b/tests/unit/Adapter/SumResultTest.php @@ -8,11 +8,12 @@ use PHPUnit\Framework\TestCase; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Collection; +use Utopia\Database\Exception\Limit as LimitException; use Utopia\Database\Validator\Authorization; /** * The value of a sum() as the engine sends it over the text protocol: MariaDB and MySQL write a large double with an - * exponent and no decimal point, and a DECIMAL sum past the range of an integer in full. + * exponent and no decimal point, and a DECIMAL sum past the range of an integer in full, which is refused. */ final class SumResultTest extends TestCase { @@ -25,7 +26,8 @@ public static function sums(): iterable yield 'decimal' => ['10.50', 10.5]; yield 'double with an exponent' => ['1e308', 1e308]; yield 'negative double with an exponent' => ['-1.5e300', -1.5e300]; - yield 'decimal past the range of an integer' => ['18446744073709551614', 18446744073709551614.0]; + yield 'decimal at the top of the range of an integer' => ['9223372036854775807', PHP_INT_MAX]; + yield 'decimal at the bottom of the range of an integer' => ['-9223372036854775808', PHP_INT_MIN]; yield 'no rows' => [null, 0]; yield 'infinity' => ['INF', \INF]; yield 'negative infinity' => ['-INF', -\INF]; @@ -39,6 +41,24 @@ public function testASumIsReadAsTheNumberTheEngineSent(?string $sent, int|float $this->assertSame($expected, $this->sumOf($sent)); } + /** + * @return iterable + */ + public static function integerSumsPastTheRangeOfAnInteger(): iterable + { + yield 'above, as MySQL and PostgreSQL sum it' => ['18446744073709551614']; + yield 'above, as MariaDB sums it' => ['18446744073709551614.0000']; + yield 'one above' => ['9223372036854775808']; + yield 'one below' => ['-9223372036854775809']; + } + + #[DataProvider('integerSumsPastTheRangeOfAnInteger')] + public function testAnIntegerSumPastTheRangeOfAnIntegerIsRefused(string $sent): void + { + $this->expectException(LimitException::class); + $this->sumOf($sent); + } + public function testANotANumberSumIsReadAsNan(): void { $this->assertNan($this->sumOf('NAN')); diff --git a/tests/unit/AggregationTest.php b/tests/unit/AggregationTest.php index 5f3bca39d..6fe8e8d4d 100644 --- a/tests/unit/AggregationTest.php +++ b/tests/unit/AggregationTest.php @@ -14,6 +14,7 @@ use Utopia\Database\Collection; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Exception\Limit as LimitException; use Utopia\Database\Filter\Callback; use Utopia\Database\Permission; use Utopia\Database\Query; @@ -99,6 +100,50 @@ public function testWholeDecimalSumOfIntegersIsAnInteger(): void $this->assertSame(['units' => 7, 'rows' => 3], $aggregation->decode(['units' => '7.0000', 'rows' => '3'])); } + /** + * @return Iterator + */ + public static function integerSumsPastTheRangeOfAnInteger(): Iterator + { + yield 'above, as MySQL and PostgreSQL sum it' => ['18446744073709551614']; + yield 'above, as MariaDB sums it' => ['18446744073709551614.0000']; + yield 'one above' => ['9223372036854775808']; + yield 'one below' => ['-9223372036854775809']; + yield 'above, as a double' => [1.8446744073709552E+19]; + yield 'below, as a double' => [-1.8446744073709552E+19]; + } + + #[DataProvider('integerSumsPastTheRangeOfAnInteger')] + public function testIntegerSumPastTheRangeOfAnIntegerIsRefused(mixed $sum): void + { + $database = $this->database(true); + $aggregation = Aggregation::from($database, $database->getCollection(self::ITEMS), [], [Query::sum('quantity', 'units')]); + + $this->expectException(LimitException::class); + $aggregation->decode(['units' => $sum]); + } + + public function testIntegerSumAtTheEdgesOfTheRangeOfAnIntegerIsExact(): void + { + $database = $this->database(true); + $aggregation = Aggregation::from($database, $database->getCollection(self::ITEMS), [], [Query::sum('quantity', 'units')]); + + $this->assertSame(['units' => PHP_INT_MAX], $aggregation->decode(['units' => '9223372036854775807'])); + $this->assertSame(['units' => PHP_INT_MIN], $aggregation->decode(['units' => '-9223372036854775808.0000'])); + $this->assertSame(['units' => PHP_INT_MIN], $aggregation->decode(['units' => PHP_INT_MIN])); + } + + public function testBoundOfAnUnsignedBigIntegerPastTheRangeOfAnIntegerKeepsItsDigitsAsAReadDoes(): void + { + $collection = new Document(['attributes' => [Attribute::bigInteger('total', signed: false)->toDocument()]]); + $aggregation = Aggregation::from($this->database(true), $collection, [], [Query::max('total', 'largest'), Query::min('total', 'smallest')]); + + $this->assertSame( + ['largest' => '18446744073709551615', 'smallest' => 7], + $aggregation->decode(['largest' => '18446744073709551615', 'smallest' => '7']), + ); + } + #[DataProvider('fetches')] public function testAggregateOverNoRowsKeepsNull(bool $stringify): void { diff --git a/tests/unit/Builder/FloatOverflowTest.php b/tests/unit/Builder/FloatOverflowTest.php index 4a23dc5a5..3f2adc1da 100644 --- a/tests/unit/Builder/FloatOverflowTest.php +++ b/tests/unit/Builder/FloatOverflowTest.php @@ -38,6 +38,10 @@ public static function aggregates(): iterable yield 'count is an integer' => [Query::count('*', 'total'), 'COUNT(*) AS `total`']; yield 'min cannot overflow' => [Query::min('reading', 'total'), 'MIN(`reading`) AS `total`']; yield 'max cannot overflow' => [Query::max('reading', 'total'), 'MAX(`reading`) AS `total`']; + yield 'bitAnd is signed' => [Query::bitAnd('flags', 'bits'), 'CAST(BIT_AND(`flags`) AS SIGNED) AS `bits`']; + yield 'bitOr is signed' => [Query::bitOr('flags', 'bits'), 'CAST(BIT_OR(`flags`) AS SIGNED) AS `bits`']; + yield 'bitXor is signed' => [Query::bitXor('flags', 'bits'), 'CAST(BIT_XOR(`flags`) AS SIGNED) AS `bits`']; + yield 'unaliased bitOr keeps the name the engine gives it' => [Query::bitOr('flags'), 'CAST(BIT_OR(`flags`) AS SIGNED) AS `BIT_OR(``flags``)`']; } /** @@ -53,13 +57,14 @@ public static function compilations(): iterable } #[DataProvider('compilations')] - public function testAFloatingAggregateIsCompiledThroughTheEngineRangeCheck(Builder $builder, Query $query, string $sql): void + public function testAnAggregateIsCompiledAsEveryEngineAnswersIt(Builder $builder, Query $query, string $sql): void { $this->assertSame($sql, $builder->compileAggregate($query)); } - public function testPostgresRaisesAnOverflowItselfSoItsAggregatesAreLeftAsTheyAre(): void + public function testPostgresRaisesAnOverflowItselfAndSignsBitwiseAggregatesSoItsAggregatesAreLeftAsTheyAre(): void { + $this->assertSame('BIT_OR("flags") AS "bits"', (new Postgres())->compileAggregate(Query::bitOr('flags', 'bits'))); $this->assertSame('SUM("reading") AS "total"', (new Postgres())->compileAggregate(Query::sum('reading', 'total'))); $this->assertSame('STDDEV_POP("reading") AS "total"', (new Postgres())->compileAggregate(Query::stddevPop('reading', 'total'))); } diff --git a/tests/unit/EmptySetAggregateContractTest.php b/tests/unit/EmptySetAggregateContractTest.php index 6eb401766..1439467bd 100644 --- a/tests/unit/EmptySetAggregateContractTest.php +++ b/tests/unit/EmptySetAggregateContractTest.php @@ -18,12 +18,15 @@ * An aggregate over no input values is null, except a count, which is zero, on every engine. * * The statement the adapter builds is answered the way MySQL and MariaDB answer it: BIT_AND over - * no input values has every bit set and BIT_OR / BIT_XOR are zero, where PostgreSQL answers null. + * no input values has every bit set, -1 once cast to SIGNED, and BIT_OR / BIT_XOR are zero, where + * PostgreSQL answers null. */ final class EmptySetAggregateContractTest extends TestCase { private const string EVERY_BIT = '18446744073709551615'; + private const string EVERY_SIGNED_BIT = '-1'; + /** * @return iterable}> */ @@ -71,7 +74,7 @@ public function testBitwiseAggregatesOverInputValuesKeepTheirResult(string $adap ]); $this->assertSame([[ - 'all_bits' => self::EVERY_BIT, + 'all_bits' => $adapter === Postgres::class ? self::EVERY_BIT : self::EVERY_SIGNED_BIT, 'any_bits' => '0', 'odd_bits' => '0', ]], $rows); @@ -151,14 +154,14 @@ private function find(string $adapter, int $inputs, array $queries): array */ private function answer(string $sql, int $inputs): array { - \preg_match_all('/([A-Z_]+)\((?:DISTINCT )?[^()]*\)(?: \+ 0)? AS [`"]([^`"]+)[`"]/', $sql, $matches, PREG_SET_ORDER); + \preg_match_all('/(CAST\()?([A-Z_]+)\((?:DISTINCT )?[^()]*\)(?: \+ 0| AS SIGNED\))? AS [`"]([^`"]+)[`"]/', $sql, $matches, PREG_SET_ORDER); $this->assertNotSame([], $matches, 'no aggregate in: '.$sql); $row = []; - foreach ($matches as [, $function, $alias]) { + foreach ($matches as [, $cast, $function, $alias]) { $row[$alias] = match ($function) { 'COUNT' => (string) $inputs, - 'BIT_AND' => self::EVERY_BIT, + 'BIT_AND' => $cast === '' ? self::EVERY_BIT : self::EVERY_SIGNED_BIT, 'BIT_OR', 'BIT_XOR' => '0', default => $inputs === 0 ? null : '1', }; diff --git a/tests/unit/Joins/AggregateMinorsTest.php b/tests/unit/Joins/AggregateMinorsTest.php index dbeada4c5..11b48a86b 100644 --- a/tests/unit/Joins/AggregateMinorsTest.php +++ b/tests/unit/Joins/AggregateMinorsTest.php @@ -29,7 +29,8 @@ final class AggregateMinorsTest extends TestCase { /** - * MariaDB and MySQL answer BIT_AND over no values with every bit set and BIT_OR/BIT_XOR with 0. + * MariaDB and MySQL answer BIT_AND over no values with every bit set, -1 once cast to SIGNED, and BIT_OR/BIT_XOR + * with 0. * The contract is null for both, aliased or not. */ public function testUnaliasedBitwiseAggregateOfAnEmptySetFollowsTheContract(): void @@ -57,7 +58,7 @@ public function testUnaliasedBitwiseAggregateOfValuesKeepsItsValue(): void ); $this->assertSame( - [['BIT_AND(`flags`)' => '18446744073709551615', 'BIT_OR(`mask`)' => null, 'BIT_XOR(`_id`)' => '0']], + [['BIT_AND(`flags`)' => '-1', 'BIT_OR(`mask`)' => null, 'BIT_XOR(`_id`)' => '0']], $rows, ); } @@ -183,16 +184,16 @@ private function mariaDBFind(array $queries, array $inputs): array $statement->method('execute')->willReturn(true); $statement->method('closeCursor')->willReturn(true); $statement->method('fetchAll')->willReturnCallback(function () use (&$sql, $inputs): array { - \preg_match_all('/([A-Z_]+)\(`([^`]+)`\)(?: AS `([^`]+)`)?/', $sql, $matches, PREG_SET_ORDER); + \preg_match_all('/(CAST\()?(([A-Z_]+)\(`([^`]+)`\))(?: AS SIGNED\))?(?: AS `((?:[^`]|``)+)`)?/', $sql, $matches, PREG_SET_ORDER); $this->assertNotSame([], $matches, 'no aggregate in: '.$sql); $row = []; foreach ($matches as $match) { - [$expression, $function, $column] = $match; - $name = $match[3] ?? $expression; + [, $cast, $expression, $function, $column] = $match; + $name = isset($match[5]) ? \str_replace('``', '`', $match[5]) : $expression; $row[$name] = match ($function) { 'COUNT' => (string) ($inputs[$column] ?? 0), - 'BIT_AND' => '18446744073709551615', + 'BIT_AND' => $cast === '' ? '18446744073709551615' : '-1', default => '0', }; } diff --git a/tests/unit/Validator/BigIntTest.php b/tests/unit/Validator/BigIntTest.php index 2cb168d52..31f4ae659 100644 --- a/tests/unit/Validator/BigIntTest.php +++ b/tests/unit/Validator/BigIntTest.php @@ -16,6 +16,29 @@ public function testTypeIsTheBigIntegerColumnType(): void $this->assertSame(ColumnType::BigInteger->value, (new BigInt(false))->getType()); } + /** + * @return iterable + */ + public static function wholeNumbers(): iterable + { + yield 'integer' => ['18446744073709551614', '18446744073709551614']; + yield 'negative integer' => ['-9223372036854775809', '-9223372036854775809']; + yield 'decimal with a zero fraction' => ['7.0000', '7']; + yield 'negative decimal with a zero fraction' => ['-7.0', '-7']; + yield 'decimal with an empty fraction' => ['7.', '7']; + yield 'decimal with a fraction' => ['7.0500', null]; + yield 'exponent' => ['1.8446744073709552e19', null]; + yield 'infinity' => ['INF', null]; + yield 'empty' => ['', null]; + yield 'fraction alone' => ['.0', null]; + } + + #[DataProvider('wholeNumbers')] + public function testWholeReadsTheIntegerAWholeDecimalNames(string $value, ?string $whole): void + { + $this->assertSame($whole, BigInt::whole($value)); + } + public function testNegatingZeroGivesTheIntegerZero(): void { $this->assertSame(0, BigInt::negate(0));