diff --git a/src/Database/Adapter/Memory.php b/src/Database/Adapter/Memory.php index 792cfe8bb..d44e929f4 100644 --- a/src/Database/Adapter/Memory.php +++ b/src/Database/Adapter/Memory.php @@ -1867,7 +1867,8 @@ public function sum(Document $collection, string $attribute, array $queries = [] $rows = \array_slice($rows, 0, $max); } - $sum = 0; + $integers = 0; + $floats = 0.0; $isFloat = false; $column = $this->filter($attribute); foreach ($rows as $row) { @@ -1875,13 +1876,23 @@ public function sum(Document $collection, string $attribute, array $queries = [] if ($value === null || ! \is_numeric($value)) { continue; } - if (\is_float($value)) { + if (\is_int($value) || (\is_string($value) && BigInt::isIntegerString($value))) { + $integers = BigInt::add($integers, $value); + } else { $isFloat = true; + $floats += (float) $value; } - $sum += $value; } - return $isFloat ? (float) $sum : (int) $sum; + if ($isFloat) { + return $floats + (float) $integers; + } + + if (\is_string($integers)) { + throw new LimitException('Value out of range'); + } + + return $integers; } #[\Override] diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 687b9dd10..afd8aeaf7 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -2748,6 +2748,9 @@ public function sum(Document $collection, string $attribute, array $queries = [] } /** @var mixed $sumTotal */ $sumTotal = $sumFirstResult->total; + if (\is_float($sumTotal) && $this->sumsIntegers($collection, $attribute)) { + throw new LimitException('Value out of range'); + } if (\is_int($sumTotal) || \is_float($sumTotal)) { return $sumTotal; } @@ -2758,6 +2761,21 @@ public function sum(Document $collection, string $attribute, array $queries = [] } } + /** + * MongoDB sums integers into a long, and into a double only when the sum does not fit one. + */ + private function sumsIntegers(Document $collection, string $attribute): bool + { + $definitions = $collection->getAttribute('attributes', []); + foreach (\is_array($definitions) ? $definitions : [] as $definition) { + if ($definition instanceof Document && $definition->getId() === $attribute) { + return \in_array($definition->getAttribute('type'), [Attribute::storedType(ColumnType::Integer), Attribute::storedType(ColumnType::BigInteger)], true); + } + } + + return false; + } + /** * @param array $documents * @return array diff --git a/src/Database/Adapter/Redis.php b/src/Database/Adapter/Redis.php index f3d87e607..efcc16259 100644 --- a/src/Database/Adapter/Redis.php +++ b/src/Database/Adapter/Redis.php @@ -1536,22 +1536,31 @@ public function sum(Document $collection, string $attribute, array $queries = [] $documents = \array_slice($documents, 0, $max); } - $sum = 0; + $integers = 0; + $floats = 0.0; $isFloat = false; foreach ($documents as $document) { $value = $this->resolveDocumentAttribute($document, $attribute); - if ($value === null) { + if ($value === null || ! \is_numeric($value)) { continue; } - if (\is_float($value)) { + if (\is_int($value) || (\is_string($value) && BigInt::isIntegerString($value))) { + $integers = BigInt::add($integers, $value); + } else { $isFloat = true; + $floats += (float) $value; } - if (\is_numeric($value)) { - $sum += $value; - } } - return $isFloat ? (float) $sum : (int) $sum; + if ($isFloat) { + return $floats + (float) $integers; + } + + if (\is_string($integers)) { + throw new LimitException('Value out of range'); + } + + return $integers; }); } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 704515dd0..36ef14a21 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -36,6 +36,7 @@ use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Contention as ContentionException; use Utopia\Database\Exception\Duplicate as DuplicateException; +use Utopia\Database\Exception\Limit as LimitException; use Utopia\Database\Exception\NotFound as NotFoundException; use Utopia\Database\Exception\Query as QueryException; use Utopia\Database\Exception\Timeout as TimeoutException; @@ -2335,14 +2336,22 @@ private function countOf(array $row): int /** * The sum in a row, with SQLite's `INF`, `-INF` and `NAN` read as the doubles they name for Database::sum() to - * refuse. + * refuse. MariaDB, MySQL and PostgreSQL sum integers exactly past the range of a 64-bit integer, which is refused + * rather than read as an inexact double. * * @param array $row + * + * @throws LimitException When an integer sum does not fit a 64-bit integer */ private function sumOf(array $row): int|float { $sum = $row['sum'] ?? 0; + $whole = \is_string($sum) ? BigInt::whole($sum) : null; + if ($whole !== null && ! BigInt::fitsPhpInt($whole)) { + throw new LimitException('Value out of range'); + } + if (\is_numeric($sum)) { return $sum + 0; } diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index e9873a0a4..238dbe27e 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -1715,6 +1715,10 @@ protected function processException(PDOException $e): Exception return new DuplicateException('Attribute already exists', $e->getCode(), $e); } + if ($e->getCode() === 'HY000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1 && stripos($e->getMessage(), 'integer overflow') !== false) { + return new LimitException('Value out of range', $e->getCode(), $e); + } + // Duplicate - SQLite uses various error codes for constraint violations: // - Error code 19 is SQLITE_CONSTRAINT (includes UNIQUE violations) // - Error code 1 is also used for some duplicate cases diff --git a/src/Database/Aggregation.php b/src/Database/Aggregation.php index 28b9269f8..33890a076 100644 --- a/src/Database/Aggregation.php +++ b/src/Database/Aggregation.php @@ -3,7 +3,9 @@ namespace Utopia\Database; use Utopia\Database\Exception as DatabaseException; +use Utopia\Database\Exception\Limit as LimitException; use Utopia\Database\Exception\Schema; +use Utopia\Database\Validator\BigInt; use Utopia\Query\Method; use Utopia\Query\Schema\ColumnType; @@ -21,6 +23,8 @@ Method::BitXor, ]; + private const float INTEGER_LIMIT = 9223372036854775808.0; + private const array STATISTICS = [ Method::Avg, Method::Stddev, @@ -356,6 +360,7 @@ private static function scalar(ColumnType $type, mixed $value): mixed return match (true) { $type === ColumnType::Boolean => \is_bool($value) ? $value : \filter_var($value, FILTER_VALIDATE_BOOLEAN), $type === ColumnType::Id => \is_scalar($value) ? (string) $value : $value, + $type === ColumnType::BigInteger && \is_string($value) && BigInt::isIntegerString($value) && ! BigInt::fitsPhpInt($value) => $value, \in_array($type, self::INTEGERS, true) => self::integer($value), \in_array($type, self::FLOATS, true) => self::float($value), \in_array($type, self::DATETIMES, true) => \is_string($value) ? DateTime::formatTz($value) : $value, @@ -364,8 +369,11 @@ private static function scalar(ColumnType $type, mixed $value): mixed } /** - * MariaDB sums integers into DECIMAL, so a whole value can arrive as "7.0000"; one past the platform's integers - * stays a float rather than wrap. + * An integer aggregate, or a group or bound of an integer attribute, as the integer it is. MariaDB sums integers + * into DECIMAL, so a whole value can arrive as "7.0000"; it and MySQL and PostgreSQL sum integers exactly past + * the range of a 64-bit integer, and a value that does not fit one is refused rather than answered inexactly. + * + * @throws LimitException When the value is a whole number past the range of a 64-bit integer */ private static function integer(mixed $value): int|float { @@ -373,14 +381,25 @@ private static function integer(mixed $value): int|float return $value; } - $integer = \filter_var($value, FILTER_VALIDATE_INT); - if ($integer !== false) { - return $integer; + $whole = \is_string($value) ? BigInt::whole($value) : null; + if ($whole !== null) { + if (! BigInt::fitsPhpInt($whole)) { + throw new LimitException('Value out of range'); + } + + return (int) $whole; } $float = self::float($value); + if (\floor($float) !== $float) { + return $float; + } + + if ($float < -self::INTEGER_LIMIT || $float >= self::INTEGER_LIMIT) { + throw new LimitException('Value out of range'); + } - return \floor($float) === $float && \abs($float) < PHP_INT_MAX ? (int) $float : $float; + return (int) $float; } private static function float(mixed $value): float diff --git a/src/Database/Builder/CompilesAggregates.php b/src/Database/Builder/CompilesAggregates.php new file mode 100644 index 000000000..cb4cb49db --- /dev/null +++ b/src/Database/Builder/CompilesAggregates.php @@ -0,0 +1,53 @@ +getMethod(); + $variance = match ($method) { + Method::Stddev => Method::Variance, + Method::StddevPop => Method::VarPop, + Method::StddevSamp => Method::VarSamp, + default => null, + }; + + $bitwise = \in_array($method, self::BITWISE_AGGREGATES, true); + if ($variance === null && ! $bitwise && ! \in_array($method, self::CHECKED_AGGREGATES, true)) { + return parent::compileAggregate($query); + } + + $expression = parent::compileAggregate(new Query($method, $query->getAttribute())); + $compiled = match (true) { + $variance !== null => 'SQRT('.parent::compileAggregate(new Query($variance, $query->getAttribute())).' + 0)', + $bitwise => 'CAST('.$expression.' AS SIGNED)', + default => $expression.' + 0', + }; + + $alias = $query->getAlias(); + + return $compiled.' AS '.($alias === '' ? $this->quoteLiteral($expression) : $this->quote($alias)); + } +} diff --git a/src/Database/Builder/MariaDB.php b/src/Database/Builder/MariaDB.php index 745b7092f..6475ddcde 100644 --- a/src/Database/Builder/MariaDB.php +++ b/src/Database/Builder/MariaDB.php @@ -6,14 +6,14 @@ /** * The MariaDB builder, which also compiles filters on their own and prepares search terms, matches containsAll() - * as 7.x did and raises a floating aggregate that overflowed. + * as 7.x did, raises a floating aggregate that overflowed and signs a bitwise one. */ class MariaDB extends Base implements Filtering, Scoping { use CompilesFilters; use MatchesContainsAllPatterns; use PreparesSearchTerms; - use RaisesFloatOverflow; + use CompilesAggregates; use RefusesJoinedWrites; use ScopesCollections; } diff --git a/src/Database/Builder/MySQL.php b/src/Database/Builder/MySQL.php index 4ac3e57cc..0ff80e728 100644 --- a/src/Database/Builder/MySQL.php +++ b/src/Database/Builder/MySQL.php @@ -6,14 +6,14 @@ /** * The MySQL builder, which also compiles filters on their own and prepares search terms, matches containsAll() - * as 7.x did and raises a floating aggregate that overflowed. + * as 7.x did, raises a floating aggregate that overflowed and signs a bitwise one. */ class MySQL extends Base implements Filtering, Scoping { use CompilesFilters; use MatchesContainsAllPatterns; use PreparesSearchTerms; - use RaisesFloatOverflow; + use CompilesAggregates; use RefusesJoinedWrites; use ScopesCollections; } diff --git a/src/Database/Builder/RaisesFloatOverflow.php b/src/Database/Builder/RaisesFloatOverflow.php deleted file mode 100644 index bdd20c8ce..000000000 --- a/src/Database/Builder/RaisesFloatOverflow.php +++ /dev/null @@ -1,42 +0,0 @@ -getMethod(); - $variance = match ($method) { - Method::Stddev => Method::Variance, - Method::StddevPop => Method::VarPop, - Method::StddevSamp => Method::VarSamp, - default => null, - }; - - if ($variance === null && ! \in_array($method, self::CHECKED_AGGREGATES, true)) { - return parent::compileAggregate($query); - } - - $expression = parent::compileAggregate(new Query($method, $query->getAttribute())); - $checked = $variance === null - ? $expression.' + 0' - : 'SQRT('.parent::compileAggregate(new Query($variance, $query->getAttribute())).' + 0)'; - - $alias = $query->getAlias(); - - return $checked.' AS '.($alias === '' ? $this->quoteLiteral($expression) : $this->quote($alias)); - } -} diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 4cd80ed09..b95120b54 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -4760,7 +4760,8 @@ private function readCollection(Collection $collection, array $joins, array $joi * @return float|int The sum of the attribute values * * @throws DatabaseException - * @throws LimitException When the sum overflows the range of a double + * @throws LimitException When the sum overflows the range of a double, or an integer sum the range of a 64-bit + * integer * @throws QueryException When $max is not greater than 0, the adapter cannot read the queries, or the attribute is * not a column of the grouped or distinct rows */ @@ -5088,15 +5089,17 @@ private function pages( * Run aggregation queries (count, sum, avg, min, max, groupBy) and return one row per group. An aggregate without * an alias comes back under `_`, or `` for count('*'). * - * Counts and bitwise aggregates are integers, averages, deviations and variances floats, and a sum an integer over - * an integer attribute and a float otherwise; min() and max() are typed as their attribute. Each group is named as + * Counts and bitwise aggregates are integers, a bitwise one the signed 64-bit result on every engine, averages, + * deviations and variances floats, and a sum an integer over an integer attribute and a float otherwise; min() and + * max() are typed as their attribute. Each group is named as * a read names its attribute, a joined one qualified by its join alias, and typed and decoded as a read returns it. * * @param array $queries At least one aggregate or groupBy query * @return list> * * @throws DatabaseException - * @throws LimitException When a sum, average, deviation or variance overflows the range of a double + * @throws LimitException When a sum, average, deviation or variance overflows the range of a double, or an integer + * sum the range of a 64-bit integer * @throws QueryException When the queries hold no aggregate or groupBy */ public function aggregate(string $collection, array $queries): array diff --git a/src/Database/Validator/BigInt.php b/src/Database/Validator/BigInt.php index d7f31535f..3f95dfd36 100644 --- a/src/Database/Validator/BigInt.php +++ b/src/Database/Validator/BigInt.php @@ -67,6 +67,23 @@ public static function isIntegerString(string $value, bool $signed = true): bool return \preg_match($signed ? '/^-?\d+$/' : '/^\d+$/', $value) === 1; } + /** + * The integer a whole number written in decimal names, as "7" or as MariaDB writes a DECIMAL sum, "7.0000"; null + * for any other value. + */ + public static function whole(string $value): ?string + { + $dot = \strpos($value, '.'); + if ($dot !== false) { + if (\trim(\substr($value, $dot + 1), '0') !== '') { + return null; + } + $value = \substr($value, 0, $dot); + } + + return self::isIntegerString($value) ? $value : null; + } + public static function fitsPhpInt(string $value, bool $signed = true): bool { if (!self::isIntegerString($value, $signed)) { diff --git a/tests/e2e/Adapter/Scopes/AggregationTests.php b/tests/e2e/Adapter/Scopes/AggregationTests.php index 404a26be0..2a1f53708 100644 --- a/tests/e2e/Adapter/Scopes/AggregationTests.php +++ b/tests/e2e/Adapter/Scopes/AggregationTests.php @@ -1736,6 +1736,176 @@ public function testBitXorOfIntegerColumn(): void $database->deleteCollection('bit_xor'); } + /** + * MariaDB and MySQL answer BIT_AND, BIT_OR and BIT_XOR as BIGINT UNSIGNED, so a result with the sign bit set came + * back as a float past PHP_INT_MAX where PostgreSQL answered the signed integer. Every engine answers the signed + * 64-bit result; SQLite has no bitwise aggregates and refuses them. + */ + public function testBitwiseAggregatesAreSigned64BitIntegers(): void + { + $database = static::getDatabase(); + + $collection = 'agg_bitwise_sign'.$this->getAggSuffix(); + $database->createCollection(Collection::create( + id: $collection, + attributes: [ + Attribute::string(key: 'series', size: 20, required: true), + Attribute::bigInteger(key: 'amount'), + ], + permissions: [Permission::create(Role::any()), Permission::read(Role::any())], + )); + $rows = [ + ['positive', 10], ['positive', 3], + ['negative', -5], ['negative', -12], + ['mixed', 10], ['mixed', -5], + ['extremes', PHP_INT_MAX], ['extremes', PHP_INT_MIN], + ['nulls', null], ['nulls', null], + ]; + foreach ($rows as [$series, $amount]) { + $database->createDocument($collection, new Document([ + 'series' => $series, + 'amount' => $amount, + '$permissions' => [Permission::read(Role::any())], + ])); + } + + $bitwise = [Query::bitAnd('amount', 'all_bits'), Query::bitOr('amount', 'any_bits'), Query::bitXor('amount', 'odd_bits')]; + + if ($this->refusesAggregation($database, $collection, $bitwise)) { + return; + } + + if ($database->getAdapter() instanceof SQLite) { + try { + $database->aggregate($collection, $bitwise); + $this->fail('SQLite has no bitwise aggregates'); + } catch (QueryException $exception) { + $this->assertSame('Aggregate bitAnd is not supported by this adapter', $exception->getMessage()); + } + $database->deleteCollection($collection); + + return; + } + + $expected = [ + 'extremes' => ['all_bits' => 0, 'any_bits' => -1, 'odd_bits' => -1], + 'mixed' => ['all_bits' => 10, 'any_bits' => -5, 'odd_bits' => -15], + 'negative' => ['all_bits' => -16, 'any_bits' => -1, 'odd_bits' => 15], + 'nulls' => ['all_bits' => null, 'any_bits' => null, 'odd_bits' => null], + 'positive' => ['all_bits' => 2, 'any_bits' => 11, 'odd_bits' => 9], + ]; + foreach ($expected as $series => $results) { + $rows = $database->aggregate($collection, [Query::equal('series', [$series]), ...$bitwise]); + $this->assertCount(1, $rows); + foreach ($results as $alias => $result) { + $this->assertSame($result, $rows[0][$alias], "{$alias} of the {$series} series"); + } + } + + $empty = $database->aggregate($collection, [Query::equal('series', ['none']), ...$bitwise]); + $this->assertSame(['all_bits' => null, 'any_bits' => null, 'odd_bits' => null], $empty[0]); + + $grouped = $database->aggregate($collection, [...$bitwise, Query::groupBy(['series']), Query::orderAsc('series')]); + $this->assertSame(\array_keys($expected), \array_column($grouped, 'series')); + foreach ($grouped as $row) { + $series = $row['series']; + $this->assertIsString($series); + foreach ($expected[$series] as $alias => $result) { + $this->assertSame($result, $row[$alias], "{$alias} of the {$series} group"); + } + } + + $database->deleteCollection($collection); + } + + /** + * MariaDB, MySQL and PostgreSQL sum integers exactly past the range of a 64-bit integer, which came back as an + * inexact float: two of PHP_INT_MAX summed to 1.8446744073709552E+19. An integer sum that does not fit is refused + * on every engine, as a sum past the range of a double is, and one that fits is answered exactly. + */ + public function testIntegerAggregateSumOverflowIsRefused(): void + { + $database = static::getDatabase(); + + $collection = 'agg_int_overflow'.$this->getAggSuffix(); + $database->createCollection(Collection::create( + id: $collection, + attributes: [ + Attribute::string(key: 'series', size: 20, required: true), + Attribute::bigInteger(key: 'amount', required: true), + ], + permissions: [Permission::create(Role::any()), Permission::read(Role::any())], + )); + $rows = [ + ['above', PHP_INT_MAX], ['above', PHP_INT_MAX], + ['below', PHP_INT_MIN], ['below', -1], + ['top', PHP_INT_MAX - 1], ['top', 1], + ['bottom', PHP_INT_MIN + 1], ['bottom', -1], + ]; + foreach ($rows as [$series, $amount]) { + $database->createDocument($collection, new Document([ + 'series' => $series, + 'amount' => $amount, + '$permissions' => [Permission::read(Role::any())], + ])); + } + + if ($this->refusesAggregation($database, $collection, [Query::sum('amount', 'total')])) { + return; + } + + $overflows = [ + [Query::equal('series', ['above']), Query::sum('amount', 'total')], + [Query::equal('series', ['below']), Query::sum('amount', 'total')], + [Query::sum('amount', 'total'), Query::groupBy(['series'])], + ]; + foreach ($overflows as $queries) { + try { + $database->aggregate($collection, $queries); + $this->fail('An integer sum past the range of a 64-bit integer is refused'); + } catch (LimitException) { + $this->addToAssertionCount(1); + } + } + + $top = $database->aggregate($collection, [Query::equal('series', ['top']), Query::sum('amount', 'total')]); + $this->assertSame(PHP_INT_MAX, $top[0]['total']); + $bottom = $database->aggregate($collection, [Query::equal('series', ['bottom']), Query::sum('amount', 'total')]); + $this->assertSame(PHP_INT_MIN, $bottom[0]['total']); + + $grouped = $database->aggregate($collection, [ + Query::equal('series', ['top', 'bottom']), + Query::sum('amount', 'total'), + Query::groupBy(['series']), + Query::orderAsc('series'), + ]); + $this->assertSame([['total' => PHP_INT_MIN, 'series' => 'bottom'], ['total' => PHP_INT_MAX, 'series' => 'top']], $grouped); + + $database->deleteCollection($collection); + } + + /** + * An adapter without aggregations refuses the queries as an invalid query, and the collection is dropped. + * + * @param array $queries + */ + private function refusesAggregation(Database $database, string $collection, array $queries): bool + { + if ($database->getAdapter()->supports(Capability::Aggregations)) { + return false; + } + + try { + $database->aggregate($collection, $queries); + $this->fail('An adapter without aggregations refuses an aggregate'); + } catch (QueryException) { + $this->addToAssertionCount(1); + } + $database->deleteCollection($collection); + + return true; + } + private function assertRejectedAsQueryShape(callable $call, string $message): void { $error = null; diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 5feafb992..274018761 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -2896,6 +2896,51 @@ public function testNegativeSumOverflowIsRefused(): void $database->deleteCollection($collection); } + /** + * An integer sum past the range of a 64-bit integer came back as an inexact float, or on Memory and Redis as a + * wrapped integer. It is refused on every adapter, and one that fits is answered exactly. + */ + public function testIntegerSumOverflowIsRefused(): void + { + $database = $this->getDatabase(); + $collection = 'sum_int_overflow_'.\substr(\uniqid(), -6); + $database->createCollection(Collection::create( + id: $collection, + attributes: [ + Attribute::string(key: 'series', size: 20, required: true), + Attribute::bigInteger(key: 'amount', required: true), + ], + permissions: [Permission::create(Role::any()), Permission::read(Role::any())], + )); + $rows = [ + ['above', PHP_INT_MAX], ['above', PHP_INT_MAX], + ['below', PHP_INT_MIN], ['below', -1], + ['top', PHP_INT_MAX - 1], ['top', 1], + ['bottom', PHP_INT_MIN + 1], ['bottom', -1], + ]; + foreach ($rows as [$series, $amount]) { + $database->createDocument($collection, new Document([ + 'series' => $series, + 'amount' => $amount, + '$permissions' => [Permission::read(Role::any())], + ])); + } + + foreach ([[[Query::equal('series', ['above'])], null], [[Query::equal('series', ['above'])], 5], [[Query::equal('series', ['below'])], null]] as [$queries, $max]) { + try { + $sum = $database->sum($collection, 'amount', $queries, $max); + $this->fail('An integer sum past the range of a 64-bit integer is refused, got '.\var_export($sum, true)); + } catch (LimitException) { + $this->addToAssertionCount(1); + } + } + + $this->assertSame(PHP_INT_MAX, $database->sum($collection, 'amount', [Query::equal('series', ['top'])])); + $this->assertSame(PHP_INT_MIN, $database->sum($collection, 'amount', [Query::equal('series', ['bottom'])])); + + $database->deleteCollection($collection); + } + public function testUpdateDocument(): void { $document = $this->initDocumentsFixture(); diff --git a/tests/unit/Adapter/SumResultTest.php b/tests/unit/Adapter/SumResultTest.php index f14dd3e8d..36f57b30b 100644 --- a/tests/unit/Adapter/SumResultTest.php +++ b/tests/unit/Adapter/SumResultTest.php @@ -8,11 +8,12 @@ use PHPUnit\Framework\TestCase; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Collection; +use Utopia\Database\Exception\Limit as LimitException; use Utopia\Database\Validator\Authorization; /** * The value of a sum() as the engine sends it over the text protocol: MariaDB and MySQL write a large double with an - * exponent and no decimal point, and a DECIMAL sum past the range of an integer in full. + * exponent and no decimal point, and a DECIMAL sum past the range of an integer in full, which is refused. */ final class SumResultTest extends TestCase { @@ -25,7 +26,8 @@ public static function sums(): iterable yield 'decimal' => ['10.50', 10.5]; yield 'double with an exponent' => ['1e308', 1e308]; yield 'negative double with an exponent' => ['-1.5e300', -1.5e300]; - yield 'decimal past the range of an integer' => ['18446744073709551614', 18446744073709551614.0]; + yield 'decimal at the top of the range of an integer' => ['9223372036854775807', PHP_INT_MAX]; + yield 'decimal at the bottom of the range of an integer' => ['-9223372036854775808', PHP_INT_MIN]; yield 'no rows' => [null, 0]; yield 'infinity' => ['INF', \INF]; yield 'negative infinity' => ['-INF', -\INF]; @@ -39,6 +41,24 @@ public function testASumIsReadAsTheNumberTheEngineSent(?string $sent, int|float $this->assertSame($expected, $this->sumOf($sent)); } + /** + * @return iterable + */ + public static function integerSumsPastTheRangeOfAnInteger(): iterable + { + yield 'above, as MySQL and PostgreSQL sum it' => ['18446744073709551614']; + yield 'above, as MariaDB sums it' => ['18446744073709551614.0000']; + yield 'one above' => ['9223372036854775808']; + yield 'one below' => ['-9223372036854775809']; + } + + #[DataProvider('integerSumsPastTheRangeOfAnInteger')] + public function testAnIntegerSumPastTheRangeOfAnIntegerIsRefused(string $sent): void + { + $this->expectException(LimitException::class); + $this->sumOf($sent); + } + public function testANotANumberSumIsReadAsNan(): void { $this->assertNan($this->sumOf('NAN')); diff --git a/tests/unit/AggregationTest.php b/tests/unit/AggregationTest.php index 5f3bca39d..6fe8e8d4d 100644 --- a/tests/unit/AggregationTest.php +++ b/tests/unit/AggregationTest.php @@ -14,6 +14,7 @@ use Utopia\Database\Collection; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Exception\Limit as LimitException; use Utopia\Database\Filter\Callback; use Utopia\Database\Permission; use Utopia\Database\Query; @@ -99,6 +100,50 @@ public function testWholeDecimalSumOfIntegersIsAnInteger(): void $this->assertSame(['units' => 7, 'rows' => 3], $aggregation->decode(['units' => '7.0000', 'rows' => '3'])); } + /** + * @return Iterator + */ + public static function integerSumsPastTheRangeOfAnInteger(): Iterator + { + yield 'above, as MySQL and PostgreSQL sum it' => ['18446744073709551614']; + yield 'above, as MariaDB sums it' => ['18446744073709551614.0000']; + yield 'one above' => ['9223372036854775808']; + yield 'one below' => ['-9223372036854775809']; + yield 'above, as a double' => [1.8446744073709552E+19]; + yield 'below, as a double' => [-1.8446744073709552E+19]; + } + + #[DataProvider('integerSumsPastTheRangeOfAnInteger')] + public function testIntegerSumPastTheRangeOfAnIntegerIsRefused(mixed $sum): void + { + $database = $this->database(true); + $aggregation = Aggregation::from($database, $database->getCollection(self::ITEMS), [], [Query::sum('quantity', 'units')]); + + $this->expectException(LimitException::class); + $aggregation->decode(['units' => $sum]); + } + + public function testIntegerSumAtTheEdgesOfTheRangeOfAnIntegerIsExact(): void + { + $database = $this->database(true); + $aggregation = Aggregation::from($database, $database->getCollection(self::ITEMS), [], [Query::sum('quantity', 'units')]); + + $this->assertSame(['units' => PHP_INT_MAX], $aggregation->decode(['units' => '9223372036854775807'])); + $this->assertSame(['units' => PHP_INT_MIN], $aggregation->decode(['units' => '-9223372036854775808.0000'])); + $this->assertSame(['units' => PHP_INT_MIN], $aggregation->decode(['units' => PHP_INT_MIN])); + } + + public function testBoundOfAnUnsignedBigIntegerPastTheRangeOfAnIntegerKeepsItsDigitsAsAReadDoes(): void + { + $collection = new Document(['attributes' => [Attribute::bigInteger('total', signed: false)->toDocument()]]); + $aggregation = Aggregation::from($this->database(true), $collection, [], [Query::max('total', 'largest'), Query::min('total', 'smallest')]); + + $this->assertSame( + ['largest' => '18446744073709551615', 'smallest' => 7], + $aggregation->decode(['largest' => '18446744073709551615', 'smallest' => '7']), + ); + } + #[DataProvider('fetches')] public function testAggregateOverNoRowsKeepsNull(bool $stringify): void { diff --git a/tests/unit/Builder/FloatOverflowTest.php b/tests/unit/Builder/FloatOverflowTest.php index 4a23dc5a5..3f2adc1da 100644 --- a/tests/unit/Builder/FloatOverflowTest.php +++ b/tests/unit/Builder/FloatOverflowTest.php @@ -38,6 +38,10 @@ public static function aggregates(): iterable yield 'count is an integer' => [Query::count('*', 'total'), 'COUNT(*) AS `total`']; yield 'min cannot overflow' => [Query::min('reading', 'total'), 'MIN(`reading`) AS `total`']; yield 'max cannot overflow' => [Query::max('reading', 'total'), 'MAX(`reading`) AS `total`']; + yield 'bitAnd is signed' => [Query::bitAnd('flags', 'bits'), 'CAST(BIT_AND(`flags`) AS SIGNED) AS `bits`']; + yield 'bitOr is signed' => [Query::bitOr('flags', 'bits'), 'CAST(BIT_OR(`flags`) AS SIGNED) AS `bits`']; + yield 'bitXor is signed' => [Query::bitXor('flags', 'bits'), 'CAST(BIT_XOR(`flags`) AS SIGNED) AS `bits`']; + yield 'unaliased bitOr keeps the name the engine gives it' => [Query::bitOr('flags'), 'CAST(BIT_OR(`flags`) AS SIGNED) AS `BIT_OR(``flags``)`']; } /** @@ -53,13 +57,14 @@ public static function compilations(): iterable } #[DataProvider('compilations')] - public function testAFloatingAggregateIsCompiledThroughTheEngineRangeCheck(Builder $builder, Query $query, string $sql): void + public function testAnAggregateIsCompiledAsEveryEngineAnswersIt(Builder $builder, Query $query, string $sql): void { $this->assertSame($sql, $builder->compileAggregate($query)); } - public function testPostgresRaisesAnOverflowItselfSoItsAggregatesAreLeftAsTheyAre(): void + public function testPostgresRaisesAnOverflowItselfAndSignsBitwiseAggregatesSoItsAggregatesAreLeftAsTheyAre(): void { + $this->assertSame('BIT_OR("flags") AS "bits"', (new Postgres())->compileAggregate(Query::bitOr('flags', 'bits'))); $this->assertSame('SUM("reading") AS "total"', (new Postgres())->compileAggregate(Query::sum('reading', 'total'))); $this->assertSame('STDDEV_POP("reading") AS "total"', (new Postgres())->compileAggregate(Query::stddevPop('reading', 'total'))); } diff --git a/tests/unit/EmptySetAggregateContractTest.php b/tests/unit/EmptySetAggregateContractTest.php index 6eb401766..1439467bd 100644 --- a/tests/unit/EmptySetAggregateContractTest.php +++ b/tests/unit/EmptySetAggregateContractTest.php @@ -18,12 +18,15 @@ * An aggregate over no input values is null, except a count, which is zero, on every engine. * * The statement the adapter builds is answered the way MySQL and MariaDB answer it: BIT_AND over - * no input values has every bit set and BIT_OR / BIT_XOR are zero, where PostgreSQL answers null. + * no input values has every bit set, -1 once cast to SIGNED, and BIT_OR / BIT_XOR are zero, where + * PostgreSQL answers null. */ final class EmptySetAggregateContractTest extends TestCase { private const string EVERY_BIT = '18446744073709551615'; + private const string EVERY_SIGNED_BIT = '-1'; + /** * @return iterable}> */ @@ -71,7 +74,7 @@ public function testBitwiseAggregatesOverInputValuesKeepTheirResult(string $adap ]); $this->assertSame([[ - 'all_bits' => self::EVERY_BIT, + 'all_bits' => $adapter === Postgres::class ? self::EVERY_BIT : self::EVERY_SIGNED_BIT, 'any_bits' => '0', 'odd_bits' => '0', ]], $rows); @@ -151,14 +154,14 @@ private function find(string $adapter, int $inputs, array $queries): array */ private function answer(string $sql, int $inputs): array { - \preg_match_all('/([A-Z_]+)\((?:DISTINCT )?[^()]*\)(?: \+ 0)? AS [`"]([^`"]+)[`"]/', $sql, $matches, PREG_SET_ORDER); + \preg_match_all('/(CAST\()?([A-Z_]+)\((?:DISTINCT )?[^()]*\)(?: \+ 0| AS SIGNED\))? AS [`"]([^`"]+)[`"]/', $sql, $matches, PREG_SET_ORDER); $this->assertNotSame([], $matches, 'no aggregate in: '.$sql); $row = []; - foreach ($matches as [, $function, $alias]) { + foreach ($matches as [, $cast, $function, $alias]) { $row[$alias] = match ($function) { 'COUNT' => (string) $inputs, - 'BIT_AND' => self::EVERY_BIT, + 'BIT_AND' => $cast === '' ? self::EVERY_BIT : self::EVERY_SIGNED_BIT, 'BIT_OR', 'BIT_XOR' => '0', default => $inputs === 0 ? null : '1', }; diff --git a/tests/unit/Joins/AggregateMinorsTest.php b/tests/unit/Joins/AggregateMinorsTest.php index dbeada4c5..11b48a86b 100644 --- a/tests/unit/Joins/AggregateMinorsTest.php +++ b/tests/unit/Joins/AggregateMinorsTest.php @@ -29,7 +29,8 @@ final class AggregateMinorsTest extends TestCase { /** - * MariaDB and MySQL answer BIT_AND over no values with every bit set and BIT_OR/BIT_XOR with 0. + * MariaDB and MySQL answer BIT_AND over no values with every bit set, -1 once cast to SIGNED, and BIT_OR/BIT_XOR + * with 0. * The contract is null for both, aliased or not. */ public function testUnaliasedBitwiseAggregateOfAnEmptySetFollowsTheContract(): void @@ -57,7 +58,7 @@ public function testUnaliasedBitwiseAggregateOfValuesKeepsItsValue(): void ); $this->assertSame( - [['BIT_AND(`flags`)' => '18446744073709551615', 'BIT_OR(`mask`)' => null, 'BIT_XOR(`_id`)' => '0']], + [['BIT_AND(`flags`)' => '-1', 'BIT_OR(`mask`)' => null, 'BIT_XOR(`_id`)' => '0']], $rows, ); } @@ -183,16 +184,16 @@ private function mariaDBFind(array $queries, array $inputs): array $statement->method('execute')->willReturn(true); $statement->method('closeCursor')->willReturn(true); $statement->method('fetchAll')->willReturnCallback(function () use (&$sql, $inputs): array { - \preg_match_all('/([A-Z_]+)\(`([^`]+)`\)(?: AS `([^`]+)`)?/', $sql, $matches, PREG_SET_ORDER); + \preg_match_all('/(CAST\()?(([A-Z_]+)\(`([^`]+)`\))(?: AS SIGNED\))?(?: AS `((?:[^`]|``)+)`)?/', $sql, $matches, PREG_SET_ORDER); $this->assertNotSame([], $matches, 'no aggregate in: '.$sql); $row = []; foreach ($matches as $match) { - [$expression, $function, $column] = $match; - $name = $match[3] ?? $expression; + [, $cast, $expression, $function, $column] = $match; + $name = isset($match[5]) ? \str_replace('``', '`', $match[5]) : $expression; $row[$name] = match ($function) { 'COUNT' => (string) ($inputs[$column] ?? 0), - 'BIT_AND' => '18446744073709551615', + 'BIT_AND' => $cast === '' ? '18446744073709551615' : '-1', default => '0', }; } diff --git a/tests/unit/Validator/BigIntTest.php b/tests/unit/Validator/BigIntTest.php index 2cb168d52..31f4ae659 100644 --- a/tests/unit/Validator/BigIntTest.php +++ b/tests/unit/Validator/BigIntTest.php @@ -16,6 +16,29 @@ public function testTypeIsTheBigIntegerColumnType(): void $this->assertSame(ColumnType::BigInteger->value, (new BigInt(false))->getType()); } + /** + * @return iterable + */ + public static function wholeNumbers(): iterable + { + yield 'integer' => ['18446744073709551614', '18446744073709551614']; + yield 'negative integer' => ['-9223372036854775809', '-9223372036854775809']; + yield 'decimal with a zero fraction' => ['7.0000', '7']; + yield 'negative decimal with a zero fraction' => ['-7.0', '-7']; + yield 'decimal with an empty fraction' => ['7.', '7']; + yield 'decimal with a fraction' => ['7.0500', null]; + yield 'exponent' => ['1.8446744073709552e19', null]; + yield 'infinity' => ['INF', null]; + yield 'empty' => ['', null]; + yield 'fraction alone' => ['.0', null]; + } + + #[DataProvider('wholeNumbers')] + public function testWholeReadsTheIntegerAWholeDecimalNames(string $value, ?string $whole): void + { + $this->assertSame($whole, BigInt::whole($value)); + } + public function testNegatingZeroGivesTheIntegerZero(): void { $this->assertSame(0, BigInt::negate(0));