From 876972df4810db7d358a1761f843527c7318f3df Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 26 Jul 2026 11:35:07 +0545 Subject: [PATCH 1/5] chore(audit): bump utopia-php/query to 0.3.* Moves the audit package from utopia-php/query 0.1.* to the 0.3 line (locked at 0.3.3) and refreshes packages/audit/composer.lock. --- packages/audit/composer.json | 2 +- packages/audit/composer.lock | 127 +++++++++++++++++++---------------- 2 files changed, 69 insertions(+), 60 deletions(-) diff --git a/packages/audit/composer.json b/packages/audit/composer.json index 9e6572771..15510e36c 100644 --- a/packages/audit/composer.json +++ b/packages/audit/composer.json @@ -29,7 +29,7 @@ "php": ">=8.5", "utopia-php/database": "^6.0.0", "utopia-php/fetch": "^1.1", - "utopia-php/query": "0.1.*", + "utopia-php/query": "0.3.*", "utopia-php/validators": "0.3.*" }, "config": { diff --git a/packages/audit/composer.lock b/packages/audit/composer.lock index e54f2ade6..c3100efe2 100644 --- a/packages/audit/composer.lock +++ b/packages/audit/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "4a91c87b103bb97847d08d3c77925bea", + "content-hash": "3e5c0d05eb866f51d6e31c4edbb46546", "packages": [ { "name": "brick/math", @@ -144,23 +144,23 @@ }, { "name": "google/protobuf", - "version": "v4.33.6", + "version": "v5.35.1", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "84b008c23915ed94536737eae46f41ba3bccfe67" + "reference": "55bb4a7d6739b5af0927b96213c1371a3afb7cfb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/84b008c23915ed94536737eae46f41ba3bccfe67", - "reference": "84b008c23915ed94536737eae46f41ba3bccfe67", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/55bb4a7d6739b5af0927b96213c1371a3afb7cfb", + "reference": "55bb4a7d6739b5af0927b96213c1371a3afb7cfb", "shasum": "" }, "require": { - "php": ">=8.1.0" + "php": ">=8.2.0" }, "require-dev": { - "phpunit/phpunit": ">=10.5.62 <11.0.0" + "phpunit/phpunit": ">=11.5.0 <12.0.0" }, "suggest": { "ext-bcmath": "Need to support JSON deserialization" @@ -182,9 +182,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.6" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v5.35.1" }, - "time": "2026-03-18T17:32:05+00:00" + "time": "2026-06-11T21:19:23+00:00" }, { "name": "mongodb/mongodb", @@ -409,16 +409,16 @@ }, { "name": "open-telemetry/api", - "version": "1.9.0", + "version": "1.10.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "6f8d237ce2c304ca85f31970f788e7f074d147be" + "reference": "7c029c4a6fd457094a20569bf98f93d95e9a7559" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/6f8d237ce2c304ca85f31970f788e7f074d147be", - "reference": "6f8d237ce2c304ca85f31970f788e7f074d147be", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/7c029c4a6fd457094a20569bf98f93d95e9a7559", + "reference": "7c029c4a6fd457094a20569bf98f93d95e9a7559", "shasum": "" }, "require": { @@ -475,7 +475,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2026-02-25T13:24:05+00:00" + "time": "2026-07-06T12:28:04+00:00" }, { "name": "open-telemetry/context", @@ -602,20 +602,20 @@ }, { "name": "open-telemetry/gen-otlp-protobuf", - "version": "1.9.0", + "version": "1.10.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/gen-otlp-protobuf.git", - "reference": "a229cf161d42001d64c8f21e8f678581fe1c66b9" + "reference": "66f04d0e448ad333033bfc7baae1aa56330be088" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/gen-otlp-protobuf/zipball/a229cf161d42001d64c8f21e8f678581fe1c66b9", - "reference": "a229cf161d42001d64c8f21e8f678581fe1c66b9", + "url": "https://api.github.com/repos/opentelemetry-php/gen-otlp-protobuf/zipball/66f04d0e448ad333033bfc7baae1aa56330be088", + "reference": "66f04d0e448ad333033bfc7baae1aa56330be088", "shasum": "" }, "require": { - "google/protobuf": "^3.22 || ^4.0", + "google/protobuf": "^3.22 || ^4.0 || ^5.0", "php": "^8.0" }, "suggest": { @@ -661,20 +661,20 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-10-19T06:44:33+00:00" + "time": "2026-06-17T12:06:32+00:00" }, { "name": "open-telemetry/sdk", - "version": "1.14.0", + "version": "1.15.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "6e3d0ce93e76555dd5e2f1d19443ff45b990e410" + "reference": "77e1aa73850154abb86937d52a70883edc3b4547" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/6e3d0ce93e76555dd5e2f1d19443ff45b990e410", - "reference": "6e3d0ce93e76555dd5e2f1d19443ff45b990e410", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/77e1aa73850154abb86937d52a70883edc3b4547", + "reference": "77e1aa73850154abb86937d52a70883edc3b4547", "shasum": "" }, "require": { @@ -682,7 +682,7 @@ "nyholm/psr7-server": "^1.1", "open-telemetry/api": "^1.8", "open-telemetry/context": "^1.4", - "open-telemetry/sem-conv": "^1.0", + "open-telemetry/sem-conv": "^1.38.0", "php": "^8.1", "php-http/discovery": "^1.14", "psr/http-client": "^1.0", @@ -705,7 +705,10 @@ "spi": { "OpenTelemetry\\API\\Configuration\\ConfigEnv\\EnvComponentLoader": [ "OpenTelemetry\\API\\Instrumentation\\Configuration\\General\\ConfigEnv\\EnvComponentLoaderHttpConfig", - "OpenTelemetry\\API\\Instrumentation\\Configuration\\General\\ConfigEnv\\EnvComponentLoaderPeerConfig" + "OpenTelemetry\\API\\Instrumentation\\Configuration\\General\\ConfigEnv\\EnvComponentLoaderPeerConfig", + "OpenTelemetry\\SDK\\ConfigEnv\\Trace\\SpanSuppressionStrategySemConv", + "OpenTelemetry\\SDK\\ConfigEnv\\Trace\\SpanSuppressionStrategySpanKind", + "OpenTelemetry\\SDK\\ConfigEnv\\Distribution\\DistributionConfigurationSdk" ], "OpenTelemetry\\SDK\\Common\\Configuration\\Resolver\\ResolverInterface": [ "OpenTelemetry\\SDK\\Common\\Configuration\\Resolver\\SdkConfigurationResolver" @@ -715,7 +718,7 @@ ] }, "branch-alias": { - "dev-main": "1.12.x-dev" + "dev-main": "1.14.x-dev" } }, "autoload": { @@ -758,7 +761,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2026-03-21T11:50:01+00:00" + "time": "2026-07-14T13:09:54+00:00" }, { "name": "open-telemetry/sem-conv", @@ -2197,16 +2200,16 @@ }, { "name": "utopia-php/database", - "version": "6.4.3", + "version": "6.4.5", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "d5ebecc2f4518329e3e1c5ded3769638908c78ba" + "reference": "aa3fe328c1704f2dab4606b0b3a6316abeabbc56" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/d5ebecc2f4518329e3e1c5ded3769638908c78ba", - "reference": "d5ebecc2f4518329e3e1c5ded3769638908c78ba", + "url": "https://api.github.com/repos/utopia-php/database/zipball/aa3fe328c1704f2dab4606b0b3a6316abeabbc56", + "reference": "aa3fe328c1704f2dab4606b0b3a6316abeabbc56", "shasum": "" }, "require": { @@ -2251,9 +2254,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/6.4.3" + "source": "https://github.com/utopia-php/database/tree/6.4.5" }, - "time": "2026-07-13T13:24:23+00:00" + "time": "2026-07-23T09:53:11+00:00" }, { "name": "utopia-php/fetch", @@ -2297,16 +2300,16 @@ }, { "name": "utopia-php/mongo", - "version": "1.3.0", + "version": "1.3.2", "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "44eeb4db9bfcfddccd6e757668a519d9541e65c9" + "reference": "34d9ab406a87d2006b0e108796fdb6b5397853d3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/44eeb4db9bfcfddccd6e757668a519d9541e65c9", - "reference": "44eeb4db9bfcfddccd6e757668a519d9541e65c9", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/34d9ab406a87d2006b0e108796fdb6b5397853d3", + "reference": "34d9ab406a87d2006b0e108796fdb6b5397853d3", "shasum": "" }, "require": { @@ -2352,22 +2355,22 @@ ], "support": { "issues": "https://github.com/utopia-php/mongo/issues", - "source": "https://github.com/utopia-php/mongo/tree/1.3.0" + "source": "https://github.com/utopia-php/mongo/tree/1.3.2" }, - "time": "2026-07-08T04:09:01+00:00" + "time": "2026-07-20T09:02:13+00:00" }, { "name": "utopia-php/pools", - "version": "1.0.9", + "version": "1.1.0", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "13346168ca7097d9b220872a017c9ad8c5dac62f" + "reference": "fd5aa7e89685ffc74f9dff3d23c270ef30c7bbf3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/13346168ca7097d9b220872a017c9ad8c5dac62f", - "reference": "13346168ca7097d9b220872a017c9ad8c5dac62f", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/fd5aa7e89685ffc74f9dff3d23c270ef30c7bbf3", + "reference": "fd5aa7e89685ffc74f9dff3d23c270ef30c7bbf3", "shasum": "" }, "require": { @@ -2378,7 +2381,10 @@ "swoole/ide-helper": "6.*" }, "suggest": { - "ext-swoole": "Required to use the Swoole pool adapter" + "ext-mongodb": "Needed to support MongoDB database pools", + "ext-pdo": "Needed to support MariaDB, MySQL or SQLite database pools", + "ext-redis": "Needed to support Redis cache pools", + "ext-swoole": "Needed to support Swoole based pool adapter" }, "type": "library", "autoload": { @@ -2405,30 +2411,33 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/1.0.9" + "source": "https://github.com/utopia-php/pools/tree/1.1.0" }, - "time": "2026-06-26T10:48:18+00:00" + "time": "2026-07-17T10:19:09+00:00" }, { "name": "utopia-php/query", - "version": "0.1.1", + "version": "0.3.3", "source": { "type": "git", "url": "https://github.com/utopia-php/query.git", - "reference": "964a10ed3185490505f4c0062f2eb7b89287fb27" + "reference": "76cd71071cd59cb45ef6b30eb11449f3832bf18f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/query/zipball/964a10ed3185490505f4c0062f2eb7b89287fb27", - "reference": "964a10ed3185490505f4c0062f2eb7b89287fb27", + "url": "https://api.github.com/repos/utopia-php/query/zipball/76cd71071cd59cb45ef6b30eb11449f3832bf18f", + "reference": "76cd71071cd59cb45ef6b30eb11449f3832bf18f", "shasum": "" }, "require": { "php": ">=8.4" }, "require-dev": { + "brianium/paratest": "*", "laravel/pint": "*", + "mongodb/mongodb": "^2.0", "phpstan/phpstan": "*", + "phpunit/phpcov": "*", "phpunit/phpunit": "^12.0" }, "type": "library", @@ -2451,9 +2460,9 @@ ], "support": { "issues": "https://github.com/utopia-php/query/issues", - "source": "https://github.com/utopia-php/query/tree/0.1.1" + "source": "https://github.com/utopia-php/query/tree/0.3.3" }, - "time": "2026-03-03T09:05:14+00:00" + "time": "2026-07-06T04:55:56+00:00" }, { "name": "utopia-php/telemetry", @@ -2508,16 +2517,16 @@ }, { "name": "utopia-php/validators", - "version": "0.3.0", + "version": "0.3.1", "source": { "type": "git", "url": "https://github.com/utopia-php/validators.git", - "reference": "5a3ef67ce17c2e148da7895e8de699614f0afdfa" + "reference": "d9c2269ebd2596a09681ccd2fd133eeedfcdf9ba" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/validators/zipball/5a3ef67ce17c2e148da7895e8de699614f0afdfa", - "reference": "5a3ef67ce17c2e148da7895e8de699614f0afdfa", + "url": "https://api.github.com/repos/utopia-php/validators/zipball/d9c2269ebd2596a09681ccd2fd133eeedfcdf9ba", + "reference": "d9c2269ebd2596a09681ccd2fd133eeedfcdf9ba", "shasum": "" }, "require": { @@ -2542,9 +2551,9 @@ ], "support": { "issues": "https://github.com/utopia-php/validators/issues", - "source": "https://github.com/utopia-php/validators/tree/0.3.0" + "source": "https://github.com/utopia-php/validators/tree/0.3.1" }, - "time": "2026-07-13T09:46:33+00:00" + "time": "2026-07-14T11:55:48+00:00" } ], "packages-dev": [], From f6320efbcc734c0f741bd548b874a5af96b1b32b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 26 Jul 2026 11:35:17 +0545 Subject: [PATCH 2/5] refactor(audit): migrate the ClickHouse adapter to the query 0.3 builder setup() emits its DDL through Utopia\Query\Schema\ClickHouse instead of hand-assembled SQL: column types, LowCardinality/Nullable wrapping, bloom-filter indexes, engine, ORDER BY, PARTITION BY and SETTINGS all come from the schema builder. The retention MODIFY TTL / REMOVE TTL statements are unchanged. find(), count(), getById(), createBatch() and cleanup() build their SQL through Utopia\Query\Builder\ClickHouse. Positional bindings become typed {paramN:Type} placeholders derived from getAttributes(). createBatch() uses bulkInsert(Format::JSONEachRow, ...) for the INSERT envelope and body, and cleanup() uses a lightweight DELETE FROM. Query::getMethod() now returns the Utopia\Query\Method enum upstream, so Database::count() compares against enum cases; Utopia\Audit\Query keeps exposing the legacy TYPE_* string constants. Filter semantics are unchanged: contains/notContains stay substring matches, now compiled to position(col, ?) > 0 / = 0 instead of LIKE '%needle%', which also drops the wildcard escaping. Adds tests/Audit/Adapter/ClickHouseSqlSnapshotTest.php, a server-free SQL snapshot suite pinning the emitted DDL/INSERT/DELETE/SELECT shapes; it runs in the unit testsuite and is excluded from e2e. --- packages/audit/CHANGELOG.md | 26 + packages/audit/phpunit.xml | 3 + .../audit/src/Audit/Adapter/ClickHouse.php | 764 ++++++++---------- packages/audit/src/Audit/Adapter/Database.php | 9 +- packages/audit/src/Audit/Query.php | 78 +- .../Adapter/ClickHouseSqlSnapshotTest.php | 345 ++++++++ .../tests/Audit/Adapter/ClickHouseTest.php | 3 +- packages/audit/tests/Audit/AuditBase.php | 2 - packages/audit/tests/Audit/QueryTest.php | 30 +- 9 files changed, 812 insertions(+), 448 deletions(-) create mode 100644 packages/audit/tests/Audit/Adapter/ClickHouseSqlSnapshotTest.php diff --git a/packages/audit/CHANGELOG.md b/packages/audit/CHANGELOG.md index afb8bd847..8b6933918 100644 --- a/packages/audit/CHANGELOG.md +++ b/packages/audit/CHANGELOG.md @@ -2,6 +2,32 @@ All notable changes to `utopia-php/audit` are documented in this file. +## Unreleased + +### ClickHouse adapter — migrated to the utopia-php/query 0.3 builder + +#### Changed + +- `utopia-php/query` bumped from `0.1.*` to `0.3.*` (locked at 0.3.3). +- `setup()` builds its DDL through `Utopia\Query\Schema\ClickHouse` instead of + hand-assembled SQL. Column types, `LowCardinality(...)` / `Nullable(...)` + wrapping, bloom-filter indexes, engine, `ORDER BY`, `PARTITION BY` and + `SETTINGS` are all emitted by the schema builder. The retention `MODIFY TTL` + / `REMOVE TTL` statements are unchanged. +- `find()`, `count()`, `getById()`, `createBatch()` and `cleanup()` build their + SQL through `Utopia\Query\Builder\ClickHouse`. Positional bindings are + rewritten to typed `{paramN:Type}` ClickHouse placeholders from a column → + type map derived from `getAttributes()`. +- `createBatch()` uses `Builder\ClickHouse::bulkInsert(Format::JSONEachRow, …)` + to emit the `INSERT … FORMAT JSONEachRow` envelope and serialize the body. +- `Query::getMethod()` now returns the `Utopia\Query\Method` enum (upstream + 0.3 change). `Utopia\Audit\Query` continues to expose the legacy `TYPE_*` + string constants, which map to the same string values. + +Filter semantics are unchanged: `contains` / `notContains` remain substring +matches (now compiled to ClickHouse `position(col, ?) > 0` / `= 0` rather than +`LIKE '%needle%'`, which also removes the need for wildcard escaping). + ## 2.9.0 ### ClickHouse adapter — user-agent columns diff --git a/packages/audit/phpunit.xml b/packages/audit/phpunit.xml index 2a61dfa88..aee3fdc31 100755 --- a/packages/audit/phpunit.xml +++ b/packages/audit/phpunit.xml @@ -7,9 +7,12 @@ tests/Audit/QueryTest.php + + tests/Audit/Adapter/ClickHouseSqlSnapshotTest.php tests/Audit/Adapter + tests/Audit/Adapter/ClickHouseSqlSnapshotTest.php diff --git a/packages/audit/src/Audit/Adapter/ClickHouse.php b/packages/audit/src/Audit/Adapter/ClickHouse.php index d7c7e8134..4476d9e21 100644 --- a/packages/audit/src/Audit/Adapter/ClickHouse.php +++ b/packages/audit/src/Audit/Adapter/ClickHouse.php @@ -7,6 +7,14 @@ use Utopia\Audit\Query; use Utopia\Database\Database; use Utopia\Fetch\Client; +use Utopia\Query\Builder\ClickHouse as ClickHouseBuilder; +use Utopia\Query\Builder\ClickHouse\Format; +use Utopia\Query\Method; +use Utopia\Query\Query as BaseQuery; +use Utopia\Query\Schema\ClickHouse as ClickHouseSchema; +use Utopia\Query\Schema\ClickHouse\Engine as ClickHouseEngine; +use Utopia\Query\Schema\ClickHouse\IndexAlgorithm; +use Utopia\Query\Schema\ColumnType; use Utopia\Validator\Hostname; /** @@ -55,25 +63,25 @@ class ClickHouse extends SQL * arrays for these methods are rejected up front so they can't silently * compile into a "no filter applied" WHERE clause. * - * @var list + * @var list */ private const array VALUE_REQUIRED_METHODS = [ - Query::TYPE_EQUAL, - Query::TYPE_NOT_EQUAL, - Query::TYPE_LESSER, - Query::TYPE_LESSER_EQUAL, - Query::TYPE_GREATER, - Query::TYPE_GREATER_EQUAL, - Query::TYPE_BETWEEN, - Query::TYPE_NOT_BETWEEN, - Query::TYPE_CONTAINS, - Query::TYPE_NOT_CONTAINS, - Query::TYPE_STARTS_WITH, - Query::TYPE_NOT_STARTS_WITH, - Query::TYPE_ENDS_WITH, - Query::TYPE_NOT_ENDS_WITH, - Query::TYPE_REGEX, - Query::TYPE_SELECT, + Method::Equal, + Method::NotEqual, + Method::LessThan, + Method::LessThanEqual, + Method::GreaterThan, + Method::GreaterThanEqual, + Method::Between, + Method::NotBetween, + Method::Contains, + Method::NotContains, + Method::StartsWith, + Method::NotStartsWith, + Method::EndsWith, + Method::NotEndsWith, + Method::Regex, + Method::Select, ]; private readonly string $host; @@ -854,6 +862,54 @@ private function getTableName(): string return $tableName; } + /** + * Build the column → ClickHouse type map registered on `Builder\ClickHouse` + * so positional `?` bindings are emitted as typed `{paramN:Type}` placeholders. + * + * Derived from `getAttributes()` so the map stays in sync with the schema — + * DateTime attributes get `DateTime64(3)`, everything else gets `String`. + * `id` is added explicitly because it lives outside `getAttributes()`, and + * `tenant` is added only when shared-tables mode is on. `limit`, `offset` + * and `max` are pseudo-columns used by the count/find SQL wrappers. + * + * @return array + */ + private function getColumnTypeMap(): array + { + $map = ['id' => 'String']; + + foreach ($this->getAttributes() as $attribute) { + /** @var string $id */ + $id = $attribute['$id']; + $map[$id] = ($attribute['type'] ?? null) === Database::VAR_DATETIME + ? 'DateTime64(3)' + : 'String'; + } + + if ($this->sharedTables) { + $map['tenant'] = 'UInt64'; + } + + $map['limit'] = 'UInt64'; + $map['offset'] = 'UInt64'; + $map['max'] = 'UInt64'; + + return $map; + } + + /** + * Build a `Builder\ClickHouse` instance with the adapter's column type map + * pre-registered. Every adapter call site that produces SQL goes through + * here so positional `?` bindings can be rewritten to typed `{paramN:Type}` + * placeholders at `Statement` time. + */ + private function newBuilder(): ClickHouseBuilder + { + return new ClickHouseBuilder() + ->useNamedBindings() + ->withParamTypes($this->getColumnTypeMap()); + } + /** * Execute a ClickHouse query via HTTP interface using Fetch Client. * @@ -864,21 +920,23 @@ private function getTableName(): string * Parameters are referenced in SQL using syntax: {paramName:Type} * Example: SELECT * WHERE id = {id:String} * - * 2. **JSON body queries** (when $jsonRows is provided): - * Uses JSONEachRow format for optimal INSERT performance. - * SQL is sent via URL query string, JSON data as POST body. - * Each row is a JSON object on a separate line. + * 2. **Pre-serialized body queries** (when $rawBody is provided): + * Used for FORMAT-style INSERT operations (e.g. JSONEachRow). + * SQL envelope is sent via URL query string and the body is sent + * verbatim as the POST body. The caller (typically the typed + * `Builder\ClickHouse::bulkInsert()` entry point) is responsible for + * serializing rows into the format ClickHouse expects. * * ClickHouse handles all parameter escaping and type conversion internally, * making both approaches fully injection-safe. * * @param string $sql The SQL query to execute * @param array $params Key-value pairs for query parameters (for SELECT/UPDATE/DELETE) - * @param array>|null $jsonRows Array of rows for JSONEachRow INSERT operations + * @param string|null $rawBody Pre-serialized request body for FORMAT INSERT operations * @return string Response body * @throws Exception */ - private function query(string $sql, array $params = [], ?array $jsonRows = null): string + private function query(string $sql, array $params = [], ?string $rawBody = null): string { $scheme = $this->secure ? 'https' : 'http'; @@ -886,21 +944,10 @@ private function query(string $sql, array $params = [], ?array $jsonRows = null) $this->client->addHeader('X-ClickHouse-Database', $this->database); try { - if ($jsonRows !== null) { - // JSON body mode for INSERT operations with JSONEachRow format + if ($rawBody !== null) { + // Pre-serialized body mode for FORMAT INSERT operations $url = "{$scheme}://{$this->host}:{$this->port}/?query=" . urlencode($sql); - - // Build JSONEachRow body - each row on a separate line - $jsonLines = []; - foreach ($jsonRows as $row) { - try { - $encoded = json_encode($row, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR); - } catch (\JsonException $e) { - throw new Exception('Failed to encode row to JSON: ' . $e->getMessage(), $e->getCode(), $e); - } - $jsonLines[] = $encoded; - } - $body = implode("\n", $jsonLines); + $body = $rawBody; } else { // Parameterized query mode using multipart form data $url = "{$scheme}://{$this->host}:{$this->port}/"; @@ -987,59 +1034,69 @@ public function setup(): void { // Create database if not exists $escapedDatabase = $this->escapeIdentifier($this->database); - $createDbSql = "CREATE DATABASE IF NOT EXISTS {$escapedDatabase}"; - $this->query($createDbSql); + $this->query("CREATE DATABASE IF NOT EXISTS {$escapedDatabase}"); - // Build column definitions from base adapter schema - // Override time column to be NOT NULL since it's used in partition key - $columns = [ - 'id String', - ]; + $tableName = $this->getTableName(); + $qualifiedTable = $this->database . '.' . $tableName; + $escapedDatabaseAndTable = $escapedDatabase . '.' . $this->escapeIdentifier($tableName); + + // Build the DDL through Schema\ClickHouse so column types, LowCardinality + // wrappers, nullability, indexes, engine and settings all come from the + // typed builder rather than hand-assembled SQL fragments. + $schema = new ClickHouseSchema(); + $table = $schema->table($qualifiedTable); + $table->string('id')->primary(); foreach ($this->getAttributes() as $attribute) { /** @var string $id */ $id = $attribute['$id']; // Special handling for time column - must be NOT NULL for partition key - $columns[] = $id === 'time' ? 'time DateTime64(3)' : $this->getColumnDefinition($id); + if ($id === 'time') { + $table->datetime('time', precision: 3); + + continue; + } + + $column = $table->addColumn($id, $this->mapAttributeType($attribute)); + if (\in_array($id, self::LOW_CARDINALITY_COLUMNS, true)) { + $column->lowCardinality(); + } + if (empty($attribute['required'])) { + $column->nullable(); + } } // Add tenant column only if tables are shared across tenants if ($this->sharedTables) { - $columns[] = 'tenant Nullable(UInt64)'; // Supports 11-digit MySQL auto-increment IDs + // Supports 11-digit MySQL auto-increment IDs + $table->bigInteger('tenant')->unsigned()->nullable(); } - // Build indexes from base adapter schema - $indexes = []; foreach ($this->getIndexes() as $index) { /** @var string $indexName */ $indexName = $index['$id']; /** @var array $attributes */ $attributes = $index['attributes']; - // Escape each attribute name to prevent SQL injection - $escapedAttributes = array_map($this->escapeIdentifier(...), $attributes); - $attributeList = implode(', ', $escapedAttributes); - $indexes[] = "INDEX {$indexName} ({$attributeList}) TYPE bloom_filter GRANULARITY 1"; + $table->index( + columns: $attributes, + name: $indexName, + algorithm: IndexAlgorithm::BloomFilter, + granularity: 1, + ); } - $tableName = $this->getTableName(); - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $table->engine(ClickHouseEngine::MergeTree); + $table->orderBy($this->sharedTables ? ['tenant', 'time', 'id'] : ['time', 'id']); + $table->partitionBy('toYYYYMM(time)'); - $orderByExpr = $this->sharedTables ? '(tenant, time, id)' : '(time, id)'; - - // Create table with MergeTree engine for optimal performance - $createTableSql = " - CREATE TABLE IF NOT EXISTS {$escapedDatabaseAndTable} ( - " . implode(",\n ", $columns) . ', - ' . implode(",\n ", $indexes) . " - ) - ENGINE = MergeTree() - ORDER BY {$orderByExpr} - PARTITION BY toYYYYMM(time) - SETTINGS index_granularity = 8192" . ($this->sharedTables ? ', allow_nullable_key = 1' : '') . ' - '; + $settings = ['index_granularity' => '8192']; + if ($this->sharedTables) { + $settings['allow_nullable_key'] = '1'; + } + $table->settings($settings); - $this->query($createTableSql); + $this->query($table->createIfNotExists()->query); // Apply retention as a separate, idempotent ALTER. CREATE TABLE IF NOT // EXISTS won't add a TTL to a table that already exists, and MODIFY TTL @@ -1070,6 +1127,18 @@ public function setup(): void } } + /** + * Map an audit attribute descriptor to its `Schema\ColumnType`. + * + * @param array $attribute + */ + private function mapAttributeType(array $attribute): ColumnType + { + return ($attribute['type'] ?? null) === Database::VAR_DATETIME + ? ColumnType::Datetime + : ColumnType::String; + } + /** * Get column names from attributes. * Returns an array of column names excluding 'id' and 'tenant' which are handled separately. @@ -1198,19 +1267,23 @@ public function create(array $log): Log public function getById(string $id): ?Log { $tableName = $this->getTableName(); + $qualifiedTable = $this->database . '.' . $tableName; + + $builder = $this->newBuilder() + ->from($qualifiedTable) + ->selectRaw($this->getSelectColumns()) + ->filter([Query::equal('id', $id)]) + ->limit(1); + $tenantFilter = $this->getTenantFilter(); - $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); - $escapedId = $this->escapeIdentifier('id'); - - $sql = ' - SELECT ' . $this->getSelectColumns() . " - FROM {$escapedTable} - WHERE {$escapedId} = {id:String}{$tenantFilter} - LIMIT 1 - FORMAT JSON - "; - - $result = $this->query($sql, ['id' => $id]); + if ($tenantFilter !== '') { + $builder->whereRaw(ltrim($tenantFilter, ' AND')); + } + + $statement = $builder->build(); + $sql = $statement->query . ' FORMAT JSON'; + + $result = $this->query($sql, $statement->namedBindings ?? []); $logs = $this->parseJsonResults($result); return $logs[0] ?? null; @@ -1226,9 +1299,8 @@ public function getById(string $id): ?Log public function find(array $queries = []): array { $tableName = $this->getTableName(); - $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $qualifiedTable = $this->database . '.' . $tableName; - // Parse queries $parsed = $this->parseQueries($queries); // Random ordering can't combine with anything that asks for a @@ -1236,10 +1308,10 @@ public function find(array $queries = []): array // mixing column-based ORDER BY with rand() would silently drop the // column order. Reject loudly in both cases so the caller fixes the // query rather than getting unexpected results. - if (!empty($parsed['randomOrder']) && isset($parsed['cursor'])) { + if ($parsed['randomOrder'] && isset($parsed['cursor'])) { throw new Exception('Cursor pagination cannot be combined with orderRandom'); } - if (!empty($parsed['randomOrder']) && !empty($parsed['orderBy'])) { + if ($parsed['randomOrder'] && $parsed['orderAttributes'] !== []) { throw new Exception('orderRandom cannot be combined with orderAsc/orderDesc'); } @@ -1247,53 +1319,49 @@ public function find(array $queries = []): array // fall back to the full column list. $selectColumns = $this->buildProjection($parsed['select'] ?? null); - $filters = $parsed['filters']; - $params = $parsed['params']; - $orderAttributes = $parsed['orderAttributes'] ?? []; + $builder = $this->newBuilder() + ->from($qualifiedTable) + ->selectRaw($selectColumns) + ->filter($parsed['filters']); + + $tenantFilter = $this->getTenantFilter(); + if ($tenantFilter !== '') { + $builder->whereRaw(ltrim($tenantFilter, ' AND')); + } + $cursorDirection = $parsed['cursorDirection'] ?? null; + $orderAttributes = $parsed['orderAttributes']; + $cursorParams = []; if (isset($parsed['cursor'])) { $orderAttributes = $this->resolveCursorOrder($orderAttributes); - $cursorWhere = $this->buildCursorWhere($orderAttributes, $parsed['cursor'], $cursorDirection ?? 'after', $params); - $filters[] = $cursorWhere['clause']; - $params = $cursorWhere['params']; + $cursorWhere = $this->buildCursorWhere($orderAttributes, $parsed['cursor'], $cursorDirection ?? 'after', []); + $builder->whereRaw($cursorWhere['clause']); + $cursorParams = $cursorWhere['params']; } - // Build WHERE clause - $whereClause = ''; - $tenantFilter = $this->getTenantFilter(); - if (!empty($filters) || $tenantFilter) { - $conditions = $filters; - if ($tenantFilter !== '' && $tenantFilter !== '0') { - $conditions[] = ltrim($tenantFilter, ' AND'); - } - $whereClause = ' WHERE ' . implode(' AND ', $conditions); - } - - // Build ORDER BY clause. orderRandom is mutually exclusive with - // cursor and column ordering (rejected at the top of find()); when - // cursor is in play, rebuild from orderAttributes (always non-empty - // after resolveCursorOrder, which appends an id tiebreaker), - // flipping directions for `cursorBefore`. - $orderClause = ''; - if (!empty($parsed['randomOrder'])) { - $orderClause = ' ORDER BY rand()'; + // ORDER BY. orderRandom is mutually exclusive with cursor and column + // ordering (rejected above); when cursor is in play, rebuild from + // orderAttributes (always non-empty after resolveCursorOrder, which + // appends an id tiebreaker), flipping directions for `cursorBefore`. + if ($parsed['randomOrder']) { + $builder->sortRandom(); } elseif (isset($parsed['cursor'])) { - $orderSql = $this->buildOrderBySql($orderAttributes, flip: $cursorDirection === 'before'); - $orderClause = ' ORDER BY ' . implode(', ', $orderSql); - } elseif (!empty($parsed['orderBy'])) { - $orderClause = ' ORDER BY ' . implode(', ', $parsed['orderBy']); + $this->applyOrderBy($builder, $orderAttributes, flip: $cursorDirection === 'before'); + } else { + $this->applyOrderBy($builder, $orderAttributes); } - // Build LIMIT and OFFSET - $limitClause = isset($parsed['limit']) ? ' LIMIT {limit:UInt64}' : ''; - $offsetClause = isset($parsed['offset']) ? ' OFFSET {offset:UInt64}' : ''; + if (isset($parsed['limit'])) { + $builder->limit($parsed['limit']); + } + if (isset($parsed['offset'])) { + $builder->offset($parsed['offset']); + } - $sql = " - SELECT {$selectColumns} - FROM {$escapedTable}{$whereClause}{$orderClause}{$limitClause}{$offsetClause} - FORMAT JSON - "; + $statement = $builder->build(); + $sql = $statement->query . ' FORMAT JSON'; + $params = ($statement->namedBindings ?? []) + $cursorParams; $result = $this->query($sql, $params); $rows = $this->parseJsonResults($result); @@ -1369,43 +1437,32 @@ private function buildProjection(?array $select): string public function count(array $queries = [], ?int $max = null): int { $tableName = $this->getTableName(); - $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $qualifiedTable = $this->database . '.' . $tableName; - // Parse queries - we only need filters and params, not ordering/limit/offset/cursor + // Parse queries - we only need filters, not ordering/limit/offset/cursor $parsed = $this->parseQueries($queries); - // Build WHERE clause - $whereClause = ''; + $inner = $this->newBuilder() + ->from($qualifiedTable) + ->selectRaw($max !== null ? '1' : 'COUNT(*) AS count') + ->filter($parsed['filters']); + $tenantFilter = $this->getTenantFilter(); - if (!empty($parsed['filters']) || $tenantFilter) { - $conditions = $parsed['filters']; - if ($tenantFilter !== '' && $tenantFilter !== '0') { - $conditions[] = ltrim($tenantFilter, ' AND'); - } - $whereClause = ' WHERE ' . implode(' AND ', $conditions); + if ($tenantFilter !== '') { + $inner->whereRaw(ltrim($tenantFilter, ' AND')); } - // Remove limit and offset from params as they don't apply to count - $params = $parsed['params']; - unset($params['limit'], $params['offset']); - if ($max !== null) { - $params['max'] = $max; - $sql = " - SELECT COUNT(*) as count - FROM ( - SELECT 1 FROM {$escapedTable}{$whereClause} LIMIT {max:UInt64} - ) sub - FORMAT TabSeparated - "; - } else { - $sql = " - SELECT COUNT(*) as count - FROM {$escapedTable}{$whereClause} - FORMAT TabSeparated - "; + $inner->limit($max); } + $statement = $inner->build(); + $params = $statement->namedBindings ?? []; + + $sql = $max !== null + ? 'SELECT COUNT(*) AS count FROM (' . $statement->query . ') sub FORMAT TabSeparated' + : $statement->query . ' FORMAT TabSeparated'; + $result = $this->query($sql, $params); $trimmed = trim($result); @@ -1413,17 +1470,31 @@ public function count(array $queries = [], ?int $max = null): int } /** - * Parse Query objects into SQL components. + * Parse Query objects into builder-ready filters and auxiliary metadata. * - * @param array $queries - * @return array{filters: array, params: array, orderBy?: array, orderAttributes?: array, randomOrder?: bool, limit?: int, offset?: int, cursor?: array, cursorDirection?: string, select?: list} + * Returns the input filters as a list of `Utopia\Query\Query` instances — + * the caller hands them to `Builder\ClickHouse::filter()` which compiles + * them into typed `{paramN:Type}` placeholders via the column → type map + * registered on `newBuilder()`. Filter semantics are unchanged from the + * hand-written SQL this replaces: + * + * - `Equal` with multiple values compiles to `IN (...)`. + * - `Contains` / `NotContains` stay substring matches — the ClickHouse + * builder emits `position(col, ?) > 0` / `= 0`, which is equivalent to + * the previous `LIKE '%v%'` / `NOT LIKE '%v%'` form but needs no + * wildcard escaping. + * - `Regex` compiles to ClickHouse's re2 `match(col, ?)` predicate. + * - `time`-column values arriving as `\DateTimeInterface` are pre-formatted + * to ClickHouse's `Y-m-d H:i:s.v` literal so the HTTP layer doesn't see + * raw DateTime objects in `namedBindings`. + * + * @param array $queries + * @return array{filters: array, orderAttributes: array, randomOrder: bool, limit?: int, offset?: int, cursor?: array, cursorDirection?: string, select?: list} * @throws Exception */ private function parseQueries(array $queries): array { $filters = []; - $params = []; - $orderBy = []; $orderAttributes = []; $limit = null; $offset = null; @@ -1431,7 +1502,6 @@ private function parseQueries(array $queries): array $cursorDirection = null; $select = null; $randomOrder = false; - $paramCounter = 0; foreach ($queries as $query) { if (!$query instanceof Query) { @@ -1450,208 +1520,33 @@ private function parseQueries(array $queries): array // otherwise turn `Query::contains('attr', [])` into a full-table // match instead of an empty result. if (\in_array($method, self::VALUE_REQUIRED_METHODS, true) && $values === []) { - throw new \Exception(ucfirst($method) . ' queries require at least one value.'); + throw new \Exception(ucfirst($method->value) . ' queries require at least one value.'); } switch ($method) { - case Query::TYPE_EQUAL: - $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $chType = $this->getParamType($attribute); - - if (\count($values) > 1) { - $inParams = []; - foreach ($values as $value) { - $paramName = 'param_' . $paramCounter++; - $inParams[] = "{{$paramName}:{$chType}}"; - $params[$paramName] = $this->formatTypedValue($chType, $value); - } - $filters[] = "{$escapedAttr} IN (" . implode(', ', $inParams) . ')'; - } else { - $paramName = 'param_' . $paramCounter++; - $filters[] = "{$escapedAttr} = {{$paramName}:{$chType}}"; - $params[$paramName] = $this->formatTypedValue($chType, $values[0] ?? null); - } - break; - - case Query::TYPE_NOT_EQUAL: - $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $chType = $this->getParamType($attribute); - $paramName = 'param_' . $paramCounter++; - $filters[] = "{$escapedAttr} != {{$paramName}:{$chType}}"; - $params[$paramName] = $this->formatTypedValue($chType, $values[0] ?? null); - break; - - case Query::TYPE_LESSER: - $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $chType = $this->getParamType($attribute); - $paramName = 'param_' . $paramCounter++; - $filters[] = "{$escapedAttr} < {{$paramName}:{$chType}}"; - $params[$paramName] = $this->formatTypedValue($chType, $values[0] ?? null); - break; - - case Query::TYPE_LESSER_EQUAL: - $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $chType = $this->getParamType($attribute); - $paramName = 'param_' . $paramCounter++; - $filters[] = "{$escapedAttr} <= {{$paramName}:{$chType}}"; - $params[$paramName] = $this->formatTypedValue($chType, $values[0] ?? null); - break; - - case Query::TYPE_GREATER: - $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $chType = $this->getParamType($attribute); - $paramName = 'param_' . $paramCounter++; - $filters[] = "{$escapedAttr} > {{$paramName}:{$chType}}"; - $params[$paramName] = $this->formatTypedValue($chType, $values[0] ?? null); - break; - - case Query::TYPE_GREATER_EQUAL: - $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $chType = $this->getParamType($attribute); - $paramName = 'param_' . $paramCounter++; - $filters[] = "{$escapedAttr} >= {{$paramName}:{$chType}}"; - $params[$paramName] = $this->formatTypedValue($chType, $values[0] ?? null); - break; - - case Query::TYPE_BETWEEN: - $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $chType = $this->getParamType($attribute); - $paramName1 = 'param_' . $paramCounter++; - $paramName2 = 'param_' . $paramCounter++; - $filters[] = "{$escapedAttr} BETWEEN {{$paramName1}:{$chType}} AND {{$paramName2}:{$chType}}"; - $params[$paramName1] = $this->formatTypedValue($chType, $values[0] ?? null); - $params[$paramName2] = $this->formatTypedValue($chType, $values[1] ?? null); - break; - - case Query::TYPE_NOT_BETWEEN: - $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $chType = $this->getParamType($attribute); - $paramName1 = 'param_' . $paramCounter++; - $paramName2 = 'param_' . $paramCounter++; - $filters[] = "{$escapedAttr} NOT BETWEEN {{$paramName1}:{$chType}} AND {{$paramName2}:{$chType}}"; - $params[$paramName1] = $this->formatTypedValue($chType, $values[0] ?? null); - $params[$paramName2] = $this->formatTypedValue($chType, $values[1] ?? null); - break; - - case Query::TYPE_CONTAINS: - // Substring match, mirroring utopia-php/database: each - // value becomes `LIKE '%value%'`, OR'd together. - $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $conditions = []; - foreach ($values as $value) { - if (!\is_string($value)) { - throw new Exception("contains value must be a string for attribute '{$attribute}'"); - } - $paramName = 'param_' . $paramCounter++; - $conditions[] = "{$escapedAttr} LIKE {{$paramName}:String}"; - $params[$paramName] = '%' . $this->escapeLikeWildcards($value) . '%'; - } - $filters[] = '(' . implode(' OR ', $conditions) . ')'; - break; - - case Query::TYPE_NOT_CONTAINS: - // Negated substring match, mirroring utopia-php/database: - // each value becomes `NOT LIKE '%value%'`, AND'd together. - $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $conditions = []; - foreach ($values as $value) { - if (!\is_string($value)) { - throw new Exception("notContains value must be a string for attribute '{$attribute}'"); - } - $paramName = 'param_' . $paramCounter++; - $conditions[] = "{$escapedAttr} NOT LIKE {{$paramName}:String}"; - $params[$paramName] = '%' . $this->escapeLikeWildcards($value) . '%'; - } - $filters[] = '(' . implode(' AND ', $conditions) . ')'; - break; - - case Query::TYPE_IS_NULL: - $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $filters[] = "{$escapedAttr} IS NULL"; - break; - - case Query::TYPE_IS_NOT_NULL: - $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $filters[] = "{$escapedAttr} IS NOT NULL"; - break; - - case Query::TYPE_STARTS_WITH: + case Method::Equal: + case Method::NotEqual: + case Method::LessThan: + case Method::LessThanEqual: + case Method::GreaterThan: + case Method::GreaterThanEqual: + case Method::Between: + case Method::NotBetween: + case Method::Contains: + case Method::NotContains: + case Method::IsNull: + case Method::IsNotNull: + case Method::StartsWith: + case Method::NotStartsWith: + case Method::EndsWith: + case Method::NotEndsWith: + case Method::Regex: $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $needle = $values[0] ?? null; - if (!\is_string($needle)) { - throw new Exception("startsWith needle must be a string for attribute '{$attribute}'"); - } - $paramName = 'param_' . $paramCounter++; - $filters[] = "startsWith({$escapedAttr}, {{$paramName}:String})"; - $params[$paramName] = $needle; + $filters[] = new BaseQuery($method, $attribute, $this->normalizeFilterValues($attribute, $values)); break; - case Query::TYPE_NOT_STARTS_WITH: - $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $needle = $values[0] ?? null; - if (!\is_string($needle)) { - throw new Exception("notStartsWith needle must be a string for attribute '{$attribute}'"); - } - $paramName = 'param_' . $paramCounter++; - $filters[] = "NOT startsWith({$escapedAttr}, {{$paramName}:String})"; - $params[$paramName] = $needle; - break; - - case Query::TYPE_ENDS_WITH: - $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $needle = $values[0] ?? null; - if (!\is_string($needle)) { - throw new Exception("endsWith needle must be a string for attribute '{$attribute}'"); - } - $paramName = 'param_' . $paramCounter++; - $filters[] = "endsWith({$escapedAttr}, {{$paramName}:String})"; - $params[$paramName] = $needle; - break; - - case Query::TYPE_NOT_ENDS_WITH: - $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $needle = $values[0] ?? null; - if (!\is_string($needle)) { - throw new Exception("notEndsWith needle must be a string for attribute '{$attribute}'"); - } - $paramName = 'param_' . $paramCounter++; - $filters[] = "NOT endsWith({$escapedAttr}, {{$paramName}:String})"; - $params[$paramName] = $needle; - break; - - case Query::TYPE_REGEX: - $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $pattern = $values[0] ?? null; - if (!\is_string($pattern)) { - throw new Exception("regex pattern must be a string for attribute '{$attribute}'"); - } - $paramName = 'param_' . $paramCounter++; - // ClickHouse's `match(haystack, pattern)` is the re2-style - // regex predicate. Pattern is bound as a parameter, never - // interpolated, so it can't escape into the SQL. - $filters[] = "match({$escapedAttr}, {{$paramName}:String})"; - $params[$paramName] = $pattern; - break; - - case Query::TYPE_SELECT: - if (empty($values)) { + case Method::Select: + if ($values === []) { // VALUE_REQUIRED_METHODS already rejects empty values // earlier, but the explicit check keeps this branch safe // if the guard is ever bypassed. @@ -1672,45 +1567,39 @@ private function parseQueries(array $queries): array } break; - case Query::TYPE_ORDER_DESC: + case Method::OrderDesc: $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $orderBy[] = "{$escapedAttr} DESC"; $orderAttributes[] = ['attribute' => $attribute, 'direction' => 'DESC']; break; - case Query::TYPE_ORDER_ASC: + case Method::OrderAsc: $this->validateAttributeName($attribute); - $escapedAttr = $this->escapeIdentifier($attribute); - $orderBy[] = "{$escapedAttr} ASC"; $orderAttributes[] = ['attribute' => $attribute, 'direction' => 'ASC']; break; - case Query::TYPE_ORDER_RANDOM: + case Method::OrderRandom: // ClickHouse's rand() is the per-row PRNG used for random // sampling. Single emission across the result set — repeated // Query::orderRandom() calls collapse into one ORDER BY rand(). $randomOrder = true; break; - case Query::TYPE_LIMIT: + case Method::Limit: if (!\is_int($values[0])) { throw new \Exception('Invalid limit value. Expected int'); } $limit = $values[0]; - $params['limit'] = $limit; break; - case Query::TYPE_OFFSET: + case Method::Offset: if (!\is_int($values[0])) { throw new \Exception('Invalid offset value. Expected int'); } $offset = $values[0]; - $params['offset'] = $offset; break; - case Query::TYPE_CURSOR_AFTER: - case Query::TYPE_CURSOR_BEFORE: + case Method::CursorAfter: + case Method::CursorBefore: if ($cursor !== null) { // Keep the first cursor encountered (matches base groupByType semantics) break; @@ -1720,25 +1609,17 @@ private function parseQueries(array $queries): array break; // no-op cursor } $cursor = $this->normalizeCursorRow($rawCursor); - $cursorDirection = $method === Query::TYPE_CURSOR_AFTER ? 'after' : 'before'; + $cursorDirection = $method === Method::CursorAfter ? 'after' : 'before'; break; } } $result = [ 'filters' => $filters, - 'params' => $params, + 'orderAttributes' => $orderAttributes, + 'randomOrder' => $randomOrder, ]; - if ($orderBy !== []) { - $result['orderBy'] = $orderBy; - $result['orderAttributes'] = $orderAttributes; - } - - if ($randomOrder) { - $result['randomOrder'] = true; - } - if ($limit !== null) { $result['limit'] = $limit; } @@ -1759,6 +1640,59 @@ private function parseQueries(array $queries): array return $result; } + /** + * Normalize filter values so DateTime instances on the `time` column flow + * through `namedBindings` as ClickHouse-compatible strings rather than raw + * objects (the HTTP layer would otherwise serialise them as empty). + * + * @param array $values + * @return array + * + * @throws Exception + */ + private function normalizeFilterValues(string $attribute, array $values): array + { + if ($this->getParamType($attribute) !== 'DateTime64(3)') { + return $values; + } + + $normalized = []; + foreach ($values as $value) { + if ($value === null) { + $normalized[] = null; + + continue; + } + /** @var \DateTime|string $value */ + $normalized[] = $this->formatDateTime($value); + } + + return $normalized; + } + + /** + * Apply an ordered list of column directions to the builder via the + * canonical `sortAsc` / `sortDesc` API, optionally flipping each direction + * for `cursorBefore` pagination. + * + * @param array $orderAttributes + */ + private function applyOrderBy(ClickHouseBuilder $builder, array $orderAttributes, bool $flip = false): void + { + foreach ($orderAttributes as $entry) { + $direction = $entry['direction']; + if ($flip) { + $direction = $direction === 'DESC' ? 'ASC' : 'DESC'; + } + + if ($direction === 'DESC') { + $builder->sortDesc($entry['attribute']); + } else { + $builder->sortAsc($entry['attribute']); + } + } + } + /** * Normalize a user-supplied cursor row into a column-keyed array. * @@ -1836,18 +1770,6 @@ private function formatTypedValue(string $chType, mixed $value): string return $this->formatParamValue($value); } - /** - * Escape ClickHouse LIKE-pattern wildcards in a user-supplied needle. - * - * Backslash is escaped first so already-escaped characters aren't - * double-escaped. Keeps `contains('event', ['100%'])` a literal - * substring match instead of a wildcard. - */ - private function escapeLikeWildcards(string $value): string - { - return str_replace(['\\', '%', '_'], ['\\\\', '\\%', '\\_'], $value); - } - /** * Resolve the effective order attributes for cursor pagination. * @@ -1950,29 +1872,6 @@ private function buildCursorWhere(array $orderAttributes, array $cursor, string ]; } - /** - * Build the ORDER BY SQL fragment list, optionally flipping all directions. - * - * Used when cursor direction is `before` — we run the query in reverse to - * grab the previous-page rows, then `array_reverse` the result. - * - * @param array $orderAttributes - * @param bool $flip Whether to flip ASC↔DESC - * @return array - */ - private function buildOrderBySql(array $orderAttributes, bool $flip = false): array - { - $sql = []; - foreach ($orderAttributes as $entry) { - $direction = $entry['direction']; - if ($flip) { - $direction = $direction === 'DESC' ? 'ASC' : 'DESC'; - } - $sql[] = $this->escapeIdentifier($entry['attribute']) . ' ' . $direction; - } - return $sql; - } - /** * Create multiple audit log entries in batch. * @@ -1986,7 +1885,7 @@ public function createBatch(array $logs): bool } $tableName = $this->getTableName(); - $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $qualifiedTable = $this->database . '.' . $tableName; // Get all attribute column names $schemaColumns = $this->getColumnNames(); @@ -2087,9 +1986,25 @@ public function createBatch(array $logs): bool $rows[] = $row; } - $insertSql = "INSERT INTO {$escapedDatabaseAndTable} FORMAT JSONEachRow"; + $columns = ['id', 'time']; + foreach ($schemaColumns as $columnName) { + if ($columnName === 'time') { + continue; + } + $columns[] = $columnName; + } + if ($this->sharedTables) { + $columns[] = 'tenant'; + } + + // Builder\ClickHouse::bulkInsert() emits the `INSERT INTO ... FORMAT + // JSONEachRow` envelope and serialises the rows into the request body. + $statement = $this->newBuilder() + ->into($qualifiedTable) + ->bulkInsert(Format::JSONEachRow, $rows, $columns); + + $this->query($statement->query, [], $statement->body); - $this->query($insertSql, [], $rows); return true; } @@ -2507,17 +2422,24 @@ public function countByResourceAndEvents( public function cleanup(\DateTime $datetime): bool { $tableName = $this->getTableName(); - $tenantFilter = $this->getTenantFilter(); - $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); - + $qualifiedTable = $this->database . '.' . $tableName; + $escapedTimeColumn = $this->escapeIdentifier('time'); $datetimeString = $datetime->format('Y-m-d H:i:s.v'); - $settings = $this->asyncCleanup ? ' SETTINGS lightweight_deletes_sync = 0' : ''; + $builder = $this->newBuilder() + ->into($qualifiedTable) + ->whereRaw($escapedTimeColumn . ' < {datetime:DateTime64(3)}'); + + $tenantFilter = $this->getTenantFilter(); + if ($tenantFilter !== '') { + $builder->whereRaw(ltrim($tenantFilter, ' AND')); + } + + if ($this->asyncCleanup) { + $builder->settings(['lightweight_deletes_sync' => '0']); + } - $sql = " - DELETE FROM {$escapedTable} - WHERE time < {datetime:DateTime64(3)}{$tenantFilter}{$settings} - "; + $sql = $builder->delete()->query; $this->query($sql, ['datetime' => $datetimeString]); diff --git a/packages/audit/src/Audit/Adapter/Database.php b/packages/audit/src/Audit/Adapter/Database.php index 2f22765dd..796f5981e 100644 --- a/packages/audit/src/Audit/Adapter/Database.php +++ b/packages/audit/src/Audit/Adapter/Database.php @@ -10,6 +10,7 @@ use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Exception\Timeout; use Utopia\Database\Query; +use Utopia\Query\Method; /** * Database Adapter for Audit @@ -499,16 +500,16 @@ public function count(array $queries = [], ?int $max = null): int // Skip limit, offset, and cursor queries — they don't apply to count $method = $query->getMethod(); - if ($method === \Utopia\Audit\Query::TYPE_LIMIT) { + if ($method === Method::Limit) { continue; } - if ($method === \Utopia\Audit\Query::TYPE_OFFSET) { + if ($method === Method::Offset) { continue; } - if ($method === \Utopia\Audit\Query::TYPE_CURSOR_AFTER) { + if ($method === Method::CursorAfter) { continue; } - if ($method === \Utopia\Audit\Query::TYPE_CURSOR_BEFORE) { + if ($method === Method::CursorBefore) { continue; } diff --git a/packages/audit/src/Audit/Query.php b/packages/audit/src/Audit/Query.php index 4ae063b38..dfba1825c 100644 --- a/packages/audit/src/Audit/Query.php +++ b/packages/audit/src/Audit/Query.php @@ -2,6 +2,7 @@ namespace Utopia\Audit; +use Utopia\Query\Method; use Utopia\Query\Query as BaseQuery; /** @@ -12,23 +13,90 @@ * `parse()` validation — while keeping audit's lenient single-value factory * signatures (the base requires arrays / scalars only; audit accepts mixed * including `DateTime` for the `time` column). + * + * Also re-exposes the legacy `TYPE_*` string constants the audit adapter and + * its tests have always used. The base library moved to a `Method` enum in + * 0.3.x; the constants here map to the same string values (`equal`, + * `lessThan`, etc.) so existing call sites keep working. */ class Query extends BaseQuery { + public const TYPE_EQUAL = 'equal'; + + public const TYPE_NOT_EQUAL = 'notEqual'; + + public const TYPE_LESSER = 'lessThan'; + + public const TYPE_LESSER_EQUAL = 'lessThanEqual'; + + public const TYPE_GREATER = 'greaterThan'; + + public const TYPE_GREATER_EQUAL = 'greaterThanEqual'; + + public const TYPE_BETWEEN = 'between'; + + public const TYPE_NOT_BETWEEN = 'notBetween'; + + public const TYPE_CONTAINS = 'contains'; + + public const TYPE_NOT_CONTAINS = 'notContains'; + + public const TYPE_IS_NULL = 'isNull'; + + public const TYPE_IS_NOT_NULL = 'isNotNull'; + + public const TYPE_STARTS_WITH = 'startsWith'; + + public const TYPE_NOT_STARTS_WITH = 'notStartsWith'; + + public const TYPE_ENDS_WITH = 'endsWith'; + + public const TYPE_NOT_ENDS_WITH = 'notEndsWith'; + + public const TYPE_REGEX = 'regex'; + + public const TYPE_SELECT = 'select'; + + public const TYPE_ORDER_DESC = 'orderDesc'; + + public const TYPE_ORDER_ASC = 'orderAsc'; + + public const TYPE_ORDER_RANDOM = 'orderRandom'; + + public const TYPE_LIMIT = 'limit'; + + public const TYPE_OFFSET = 'offset'; + + public const TYPE_CURSOR_AFTER = 'cursorAfter'; + + public const TYPE_CURSOR_BEFORE = 'cursorBefore'; + + /** + * Construct a query with a string method name (legacy `TYPE_*` constants) + * or a `Method` enum case (new 0.3.x API). + * + * @param array $values + */ + public function __construct(Method|string $method, string $attribute = '', array $values = []) + { + parent::__construct($method, $attribute, $values); + } + /** * Filter by equal condition. * * Accepts a single scalar/object/array value and stores it as the values * array. Matches the legacy audit signature. * - * @param mixed $value Single value or array of values + * @param mixed $value Single value or array of values */ #[\Override] public static function equal(string $attribute, mixed $value): static { /** @var array $values */ $values = \is_array($value) ? $value : [$value]; - return new static(self::TYPE_EQUAL, $attribute, $values); + + return new static(Method::Equal, $attribute, $values); } /** @@ -40,7 +108,7 @@ public static function equal(string $attribute, mixed $value): static #[\Override] public static function lessThan(string $attribute, mixed $value): static { - return new static(self::TYPE_LESSER, $attribute, [$value]); + return new static(Method::LessThan, $attribute, [$value]); } /** @@ -49,7 +117,7 @@ public static function lessThan(string $attribute, mixed $value): static #[\Override] public static function greaterThan(string $attribute, mixed $value): static { - return new static(self::TYPE_GREATER, $attribute, [$value]); + return new static(Method::GreaterThan, $attribute, [$value]); } /** @@ -58,6 +126,6 @@ public static function greaterThan(string $attribute, mixed $value): static #[\Override] public static function between(string $attribute, mixed $start, mixed $end): static { - return new static(self::TYPE_BETWEEN, $attribute, [$start, $end]); + return new static(Method::Between, $attribute, [$start, $end]); } } diff --git a/packages/audit/tests/Audit/Adapter/ClickHouseSqlSnapshotTest.php b/packages/audit/tests/Audit/Adapter/ClickHouseSqlSnapshotTest.php new file mode 100644 index 000000000..a701792a2 --- /dev/null +++ b/packages/audit/tests/Audit/Adapter/ClickHouseSqlSnapshotTest.php @@ -0,0 +1,345 @@ + + */ + private function auditTypeMap(): array + { + return [ + 'id' => 'String', + 'actorId' => 'String', + 'actorType' => 'String', + 'actorInternalId' => 'String', + 'event' => 'String', + 'resource' => 'String', + 'userAgent' => 'String', + 'ip' => 'String', + 'time' => 'DateTime64(3)', + 'data' => 'String', + 'tenant' => 'UInt64', + ]; + } + + private function newAuditBuilder(): ClickHouseBuilder + { + return new ClickHouseBuilder() + ->useNamedBindings() + ->withParamTypes($this->auditTypeMap()); + } + + public function testSetupCreateTableSnapshot(): void + { + $schema = new ClickHouseSchema(); + $table = $schema->table('default.audits'); + $table->string('id')->primary(); + $table->string('actorId')->nullable(); + $table->string('actorType')->lowCardinality(); + $table->string('actorInternalId')->nullable(); + $table->string('event')->lowCardinality(); + $table->string('resource')->nullable(); + $table->string('userAgent'); + $table->string('ip'); + // Optional low-cardinality dimension, as emitted for sdk / country / + // the premium-geo and user-agent columns added in audit 2.7–2.9. + $table->addColumn('sdk', ColumnType::String)->lowCardinality()->nullable(); + $table->datetime('time', precision: 3); + $table->addColumn('data', ColumnType::String)->nullable(); + + $table->index( + columns: ['event'], + name: 'idx_event', + algorithm: IndexAlgorithm::BloomFilter, + granularity: 1, + ); + $table->index( + columns: ['actorId', 'event'], + name: 'idx_actorId_event', + algorithm: IndexAlgorithm::BloomFilter, + granularity: 1, + ); + $table->index( + columns: ['actorType'], + name: '_key_actor_type', + algorithm: IndexAlgorithm::BloomFilter, + granularity: 1, + ); + + $table->engine(ClickHouseEngine::MergeTree); + $table->orderBy(['time', 'id']); + $table->partitionBy('toYYYYMM(time)'); + $table->settings(['index_granularity' => '8192']); + + $sql = $table->createIfNotExists()->query; + + $this->assertStringContainsString('CREATE TABLE IF NOT EXISTS `default`.`audits`', $sql); + $this->assertStringContainsString('`id` String', $sql); + $this->assertStringContainsString('`actorId` Nullable(String)', $sql); + $this->assertStringContainsString('`actorType` LowCardinality(String)', $sql); + $this->assertStringContainsString('`actorInternalId` Nullable(String)', $sql); + $this->assertStringContainsString('`event` LowCardinality(String)', $sql); + $this->assertStringContainsString('`sdk` LowCardinality(Nullable(String))', $sql); + $this->assertStringContainsString('`time` DateTime64(3)', $sql); + $this->assertStringNotContainsString('`location`', $sql); + $this->assertStringNotContainsString('`userId`', $sql); + $this->assertStringContainsString('INDEX `idx_event` `event` TYPE bloom_filter GRANULARITY 1', $sql); + $this->assertStringContainsString('INDEX `idx_actorId_event` (`actorId`, `event`) TYPE bloom_filter GRANULARITY 1', $sql); + $this->assertStringContainsString('INDEX `_key_actor_type` `actorType` TYPE bloom_filter GRANULARITY 1', $sql); + $this->assertStringContainsString('ENGINE = MergeTree()', $sql); + $this->assertStringContainsString('PARTITION BY toYYYYMM(time)', $sql); + $this->assertStringContainsString('ORDER BY (`time`, `id`)', $sql); + $this->assertStringContainsString('SETTINGS index_granularity = 8192', $sql); + } + + public function testBulkInsertJsonEachRowSnapshot(): void + { + $columns = ['id', 'time', 'actorId', 'actorType', 'event', 'data']; + $rows = [ + [ + 'id' => 'log-1', + 'time' => '2025-01-02 03:04:05.678', + 'actorId' => 'u1', + 'actorType' => 'users', + 'event' => 'users.create', + 'data' => '{"foo":"bar"}', + ], + [ + 'id' => 'log-2', + 'time' => '2025-01-02 03:04:06.000', + 'actorId' => 'u2', + 'actorType' => 'users', + 'event' => 'users.delete', + 'data' => '{"foo":"baz"}', + ], + ]; + + $statement = new ClickHouseBuilder() + ->into('default.audits') + ->bulkInsert(Format::JSONEachRow, $rows, $columns); + + $this->assertSame( + 'INSERT INTO `default`.`audits` (`id`, `time`, `actorId`, `actorType`, `event`, `data`) FORMAT JSONEachRow', + $statement->query, + ); + $this->assertSame('JSONEachRow', $statement->format); + $this->assertSame($columns, $statement->columns); + $this->assertSame( + '{"id":"log-1","time":"2025-01-02 03:04:05.678","actorId":"u1","actorType":"users","event":"users.create","data":"{\"foo\":\"bar\"}"}' . "\n" + . '{"id":"log-2","time":"2025-01-02 03:04:06.000","actorId":"u2","actorType":"users","event":"users.delete","data":"{\"foo\":\"baz\"}"}', + $statement->body, + ); + } + + public function testAsyncCleanupDeleteEmitsSettingsClause(): void + { + $sql = new ClickHouseBuilder() + ->into('default.audits') + ->whereRaw('`time` < {datetime:DateTime64(3)}') + ->settings(['lightweight_deletes_sync' => '0']) + ->delete() + ->query; + + $this->assertSame( + 'DELETE FROM `default`.`audits` WHERE `time` < {datetime:DateTime64(3)} SETTINGS lightweight_deletes_sync=0', + $sql, + ); + } + + public function testSyncCleanupDeleteOmitsSettingsClause(): void + { + $sql = new ClickHouseBuilder() + ->into('default.audits') + ->whereRaw('`time` < {datetime:DateTime64(3)}') + ->delete() + ->query; + + $this->assertSame( + 'DELETE FROM `default`.`audits` WHERE `time` < {datetime:DateTime64(3)}', + $sql, + ); + } + + public function testFindEmitsTypedNamedBindings(): void + { + $statement = $this->newAuditBuilder() + ->from('default.audits') + ->selectRaw('`id`, `event`, `time`') + ->filter([ + Query::equal('actorId', ['u1']), + Query::between('time', '2025-01-01 00:00:00.000', '2025-12-31 00:00:00.000'), + ]) + ->sortDesc('time') + ->limit(25) + ->build(); + + $expectedSql = 'SELECT `id`, `event`, `time` FROM `default`.`audits` ' + . 'WHERE `actorId` IN ({param0:String}) ' + . 'AND `time` BETWEEN {param1:DateTime64(3)} AND {param2:DateTime64(3)} ' + . 'ORDER BY `time` DESC ' + . 'LIMIT {param3:Int64}'; + + $this->assertSame($expectedSql, $statement->query); + $this->assertSame( + [ + 'param0' => 'u1', + 'param1' => '2025-01-01 00:00:00.000', + 'param2' => '2025-12-31 00:00:00.000', + 'param3' => 25, + ], + $statement->namedBindings, + ); + } + + public function testEqualMultiValueEmitsTypedIn(): void + { + $statement = $this->newAuditBuilder() + ->from('default.audits') + ->selectRaw('`id`, `event`, `time`') + ->filter([ + Query::notEqual('event', ['users.delete', 'projects.delete']), + ]) + ->limit(25) + ->build(); + + $expectedSql = 'SELECT `id`, `event`, `time` FROM `default`.`audits` ' + . 'WHERE `event` NOT IN ({param0:String}, {param1:String}) ' + . 'LIMIT {param2:Int64}'; + + $this->assertSame($expectedSql, $statement->query); + $this->assertSame( + [ + 'param0' => 'users.delete', + 'param1' => 'projects.delete', + 'param2' => 25, + ], + $statement->namedBindings, + ); + } + + /** + * `contains` is a substring match on ClickHouse — the builder emits + * `position(col, ?) > 0`, which replaces the adapter's previous + * hand-written `LIKE '%needle%'` (and needs no wildcard escaping). + */ + public function testContainsEmitsPositionPredicate(): void + { + $statement = $this->newAuditBuilder() + ->from('default.audits') + ->selectRaw('`id`, `event`, `time`') + ->filter([Query::contains('event', ['dat'])]) + ->build(); + + $this->assertSame( + 'SELECT `id`, `event`, `time` FROM `default`.`audits` ' + . 'WHERE position(`event`, {param0:String}) > 0', + $statement->query, + ); + $this->assertSame(['param0' => 'dat'], $statement->namedBindings); + } + + public function testContainsMultiValueOrsPositionPredicates(): void + { + $statement = $this->newAuditBuilder() + ->from('default.audits') + ->selectRaw('`id`') + ->filter([Query::contains('event', ['dat', 'ins'])]) + ->build(); + + $this->assertSame( + 'SELECT `id` FROM `default`.`audits` ' + . 'WHERE (position(`event`, {param0:String}) > 0 OR position(`event`, {param1:String}) > 0)', + $statement->query, + ); + $this->assertSame(['param0' => 'dat', 'param1' => 'ins'], $statement->namedBindings); + } + + public function testNotContainsMultiValueAndsNegatedPositionPredicates(): void + { + $statement = $this->newAuditBuilder() + ->from('default.audits') + ->selectRaw('`id`') + ->filter([Query::notContains('event', ['update', 'delete'])]) + ->build(); + + $this->assertSame( + 'SELECT `id` FROM `default`.`audits` ' + . 'WHERE (position(`event`, {param0:String}) = 0 AND position(`event`, {param1:String}) = 0)', + $statement->query, + ); + $this->assertSame(['param0' => 'update', 'param1' => 'delete'], $statement->namedBindings); + } + + public function testFindCursorRawFragmentMergesWithTypedBindings(): void + { + $cursorClause = '((`time` < {cursor_cmp_0:DateTime64(3)}) ' + . 'OR (`time` = {cursor_eq_1_0:DateTime64(3)} AND `id` < {cursor_cmp_1:String}))'; + + $statement = $this->newAuditBuilder() + ->from('default.audits') + ->selectRaw('`id`, `event`, `time`') + ->filter([Query::equal('actorId', ['u1'])]) + ->whereRaw($cursorClause) + ->sortDesc('time') + ->sortDesc('id') + ->limit(25) + ->build(); + + $expectedSql = 'SELECT `id`, `event`, `time` FROM `default`.`audits` ' + . 'WHERE `actorId` IN ({param0:String}) ' + . 'AND ' . $cursorClause . ' ' + . 'ORDER BY `time` DESC, `id` DESC ' + . 'LIMIT {param1:Int64}'; + + $this->assertSame($expectedSql, $statement->query); + $this->assertSame( + [ + 'param0' => 'u1', + 'param1' => 25, + ], + $statement->namedBindings, + ); + } + + public function testCountWithMaxWrapsInnerSelect(): void + { + $inner = $this->newAuditBuilder() + ->from('default.audits') + ->selectRaw('1') + ->filter([Query::equal('actorId', ['u1'])]) + ->limit(5000) + ->build(); + + $sql = 'SELECT COUNT(*) AS count FROM (' . $inner->query . ') sub FORMAT TabSeparated'; + + $this->assertSame( + 'SELECT COUNT(*) AS count FROM (' + . 'SELECT 1 FROM `default`.`audits` WHERE `actorId` IN ({param0:String}) LIMIT {param1:Int64}' + . ') sub FORMAT TabSeparated', + $sql, + ); + $this->assertSame( + ['param0' => 'u1', 'param1' => 5000], + $inner->namedBindings, + ); + } +} diff --git a/packages/audit/tests/Audit/Adapter/ClickHouseTest.php b/packages/audit/tests/Audit/Adapter/ClickHouseTest.php index b95a8cdd4..16e2f2100 100644 --- a/packages/audit/tests/Audit/Adapter/ClickHouseTest.php +++ b/packages/audit/tests/Audit/Adapter/ClickHouseTest.php @@ -1327,8 +1327,9 @@ public function testSharedTableSortKeyLeadsWithTenant(): void $url = "{$scheme}://{$host}:{$port}/?database=" . rawurlencode($database) . '&user=' . rawurlencode($username) . '&password=' . rawurlencode($password); + /** @var array $params */ foreach ($params as $key => $value) { - $url .= '¶m_' . rawurlencode((string) $key) . '=' . rawurlencode((string) $value); + $url .= '¶m_' . rawurlencode($key) . '=' . rawurlencode($value); } $ctx = stream_context_create(['http' => [ 'method' => 'POST', diff --git a/packages/audit/tests/Audit/AuditBase.php b/packages/audit/tests/Audit/AuditBase.php index 6463b3b47..e0dcb10f4 100644 --- a/packages/audit/tests/Audit/AuditBase.php +++ b/packages/audit/tests/Audit/AuditBase.php @@ -245,8 +245,6 @@ public function testLogByBatch(): void ], ]; - $batchEvents = $this->applyRequiredAttributesToBatch($batchEvents); - // Test batch insertion $batchEvents = $this->applyRequiredAttributesToBatch($batchEvents); diff --git a/packages/audit/tests/Audit/QueryTest.php b/packages/audit/tests/Audit/QueryTest.php index 7c2971225..f1d59fd3e 100644 --- a/packages/audit/tests/Audit/QueryTest.php +++ b/packages/audit/tests/Audit/QueryTest.php @@ -16,55 +16,55 @@ public function testQueryStaticFactoryMethods(): void { // Test equal $query = Query::equal('userId', '123'); - $this->assertSame(Query::TYPE_EQUAL, $query->getMethod()); + $this->assertSame(Query::TYPE_EQUAL, $query->getMethod()->value); $this->assertSame('userId', $query->getAttribute()); $this->assertSame(['123'], $query->getValues()); // Test lessThan $query = Query::lessThan('time', '2024-01-01'); - $this->assertSame(Query::TYPE_LESSER, $query->getMethod()); + $this->assertSame(Query::TYPE_LESSER, $query->getMethod()->value); $this->assertSame('time', $query->getAttribute()); $this->assertSame(['2024-01-01'], $query->getValues()); // Test greaterThan $query = Query::greaterThan('time', '2023-01-01'); - $this->assertSame(Query::TYPE_GREATER, $query->getMethod()); + $this->assertSame(Query::TYPE_GREATER, $query->getMethod()->value); $this->assertSame('time', $query->getAttribute()); $this->assertSame(['2023-01-01'], $query->getValues()); // Test between $query = Query::between('time', '2023-01-01', '2024-01-01'); - $this->assertSame(Query::TYPE_BETWEEN, $query->getMethod()); + $this->assertSame(Query::TYPE_BETWEEN, $query->getMethod()->value); $this->assertSame('time', $query->getAttribute()); $this->assertSame(['2023-01-01', '2024-01-01'], $query->getValues()); // Test contains $query = Query::contains('event', ['create', 'update', 'delete']); - $this->assertSame(Query::TYPE_CONTAINS, $query->getMethod()); + $this->assertSame(Query::TYPE_CONTAINS, $query->getMethod()->value); $this->assertSame('event', $query->getAttribute()); $this->assertSame(['create', 'update', 'delete'], $query->getValues()); // Test orderDesc $query = Query::orderDesc('time'); - $this->assertSame(Query::TYPE_ORDER_DESC, $query->getMethod()); + $this->assertSame(Query::TYPE_ORDER_DESC, $query->getMethod()->value); $this->assertSame('time', $query->getAttribute()); $this->assertSame([], $query->getValues()); // Test orderAsc $query = Query::orderAsc('userId'); - $this->assertSame(Query::TYPE_ORDER_ASC, $query->getMethod()); + $this->assertSame(Query::TYPE_ORDER_ASC, $query->getMethod()->value); $this->assertSame('userId', $query->getAttribute()); $this->assertSame([], $query->getValues()); // Test limit $query = Query::limit(10); - $this->assertSame(Query::TYPE_LIMIT, $query->getMethod()); + $this->assertSame(Query::TYPE_LIMIT, $query->getMethod()->value); $this->assertSame('', $query->getAttribute()); $this->assertSame([10], $query->getValues()); // Test offset $query = Query::offset(5); - $this->assertSame(Query::TYPE_OFFSET, $query->getMethod()); + $this->assertSame(Query::TYPE_OFFSET, $query->getMethod()->value); $this->assertSame('', $query->getAttribute()); $this->assertSame([5], $query->getValues()); } @@ -77,7 +77,7 @@ public function testQueryParseAndToString(): void // Test parsing equal query $json = '{"method":"equal","attribute":"userId","values":["123"]}'; $query = Query::parse($json); - $this->assertSame(Query::TYPE_EQUAL, $query->getMethod()); + $this->assertSame(Query::TYPE_EQUAL, $query->getMethod()->value); $this->assertSame('userId', $query->getAttribute()); $this->assertSame(['123'], $query->getValues()); @@ -87,7 +87,7 @@ public function testQueryParseAndToString(): void $this->assertJson($json); $parsed = Query::parse($json); - $this->assertSame(Query::TYPE_EQUAL, $parsed->getMethod()); + $this->assertSame(Query::TYPE_EQUAL, $parsed->getMethod()->value); $this->assertSame('event', $parsed->getAttribute()); $this->assertSame(['create'], $parsed->getValues()); @@ -119,9 +119,9 @@ public function testQueryParseQueries(): void $this->assertInstanceOf(Query::class, $parsed[1]); $this->assertInstanceOf(Query::class, $parsed[2]); - $this->assertSame(Query::TYPE_EQUAL, $parsed[0]->getMethod()); - $this->assertSame(Query::TYPE_GREATER, $parsed[1]->getMethod()); - $this->assertSame(Query::TYPE_LIMIT, $parsed[2]->getMethod()); + $this->assertSame(Query::TYPE_EQUAL, $parsed[0]->getMethod()->value); + $this->assertSame(Query::TYPE_GREATER, $parsed[1]->getMethod()->value); + $this->assertSame(Query::TYPE_LIMIT, $parsed[2]->getMethod()->value); } /** @@ -220,7 +220,7 @@ public function testQueryToStringWithComplexValues(): void $this->assertJson($json); $parsed = Query::parse($json); - $this->assertSame(Query::TYPE_BETWEEN, $parsed->getMethod()); + $this->assertSame(Query::TYPE_BETWEEN, $parsed->getMethod()->value); $this->assertSame('time', $parsed->getAttribute()); $this->assertSame(['2023-01-01', '2024-12-31'], $parsed->getValues()); } From 9305cbdd313c79fe8c98b5cfadc33425d1230f82 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 19 Aug 2026 05:46:07 +0545 Subject: [PATCH 3/5] test(audit): say what the SQL snapshots do and do not cover Review flagged that these snapshots reconstruct the schema and builder calls instead of invoking the adapter, so an adapter-level mistake would not turn them red. That is accurate, and the class docblock claimed more than the tests deliver by describing them as pinning 'the SQL emitted by the adapter paths'. The coverage itself is not missing: ClickHouseTest drives the real adapter - setup, log, find, count, cleanup - against a live ClickHouse in CI, 76 tests across it and the AuditBase trait. The docblock now states the split explicitly so the next reader does not mistake a green snapshot run for adapter coverage. --- .../Audit/Adapter/ClickHouseSqlSnapshotTest.php | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/audit/tests/Audit/Adapter/ClickHouseSqlSnapshotTest.php b/packages/audit/tests/Audit/Adapter/ClickHouseSqlSnapshotTest.php index a701792a2..0bd22b7a5 100644 --- a/packages/audit/tests/Audit/Adapter/ClickHouseSqlSnapshotTest.php +++ b/packages/audit/tests/Audit/Adapter/ClickHouseSqlSnapshotTest.php @@ -14,10 +14,17 @@ use Utopia\Query\Schema\ColumnType; /** - * Snapshot tests for the SQL emitted by the migrated ClickHouse adapter - * paths. These pin the exact shape produced by `Schema\ClickHouse` and - * `Builder\ClickHouse` for the audit DDL/INSERT/DELETE/SELECT surfaces so - * a query-lib upgrade can't quietly change adapter SQL. + * Snapshots of the SQL that `Schema\ClickHouse` and `Builder\ClickHouse` + * produce for the audit DDL/INSERT/DELETE/SELECT surfaces, so a query-library + * upgrade cannot quietly change the emitted SQL. + * + * These rebuild the schema and builder calls rather than invoking the adapter, + * so they deliberately do NOT cover the adapter's own use of the library: a + * missing column or a wrong argument in `ClickHouse::setup()` would not show up + * here. That is `ClickHouseTest`, which drives the real adapter - setup, log, + * find, count and cleanup - against a live ClickHouse in CI. The two are + * complementary: this file pins the library's output shape, that one pins the + * adapter's behaviour. */ final class ClickHouseSqlSnapshotTest extends TestCase { From 2db30c61ace19ebb3610c571dca28a1ba9acfbec Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 25 Aug 2026 07:15:15 +0545 Subject: [PATCH 4/5] chore(audit): move to utopia-php/query 0.6 The branch was pinned at 0.3.* while query had released 0.4, 0.5 and 0.6. The adapter needed no changes: 0.4's Parser -> Classifier rename and its UnsupportedException removal are symbols this package never used, 0.5's nested join conditions are unused, and 0.6's Schema\Order left the index() call compatible. The one thing 0.6 did surface was Query::contains() being deprecated in favour of containsString() for substring matching and containsAny() for array attributes. Only tests called the deprecated factory - the adapter switches on Method::Contains, which is unchanged - and containsString() returns that same Method, so the swap is behaviour-preserving. Verified rather than assumed: the SQL snapshots hold at the same 37 assertions and the deprecation notices are gone. bin/monorepo check audit: pint, phpstan [OK] No errors, rector [OK]. Snapshot and query suites green. The ClickHouse integration suite needs a live server and runs in CI. --- packages/audit/composer.json | 2 +- packages/audit/composer.lock | 14 +++++++------- .../Audit/Adapter/ClickHouseSqlSnapshotTest.php | 4 ++-- .../audit/tests/Audit/Adapter/ClickHouseTest.php | 10 +++++----- packages/audit/tests/Audit/QueryTest.php | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/audit/composer.json b/packages/audit/composer.json index c6ea1f078..9e021c7cf 100644 --- a/packages/audit/composer.json +++ b/packages/audit/composer.json @@ -29,7 +29,7 @@ "php": ">=8.5", "utopia-php/database": "^7.0.0", "utopia-php/fetch": "^1.1", - "utopia-php/query": "0.3.*", + "utopia-php/query": "0.6.*", "utopia-php/validators": "^0.5" }, "config": { diff --git a/packages/audit/composer.lock b/packages/audit/composer.lock index 84e733679..76b01b54e 100644 --- a/packages/audit/composer.lock +++ b/packages/audit/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "2fd8388c40d298f6601ecbfc9ec51a71", + "content-hash": "0c3378591a7b499d440f28324d6df869", "packages": [ { "name": "brick/math", @@ -2419,16 +2419,16 @@ }, { "name": "utopia-php/query", - "version": "0.3.3", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/query.git", - "reference": "76cd71071cd59cb45ef6b30eb11449f3832bf18f" + "reference": "abaebb2f3426bdbc6f44bab04254a668de937148" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/query/zipball/76cd71071cd59cb45ef6b30eb11449f3832bf18f", - "reference": "76cd71071cd59cb45ef6b30eb11449f3832bf18f", + "url": "https://api.github.com/repos/utopia-php/query/zipball/abaebb2f3426bdbc6f44bab04254a668de937148", + "reference": "abaebb2f3426bdbc6f44bab04254a668de937148", "shasum": "" }, "require": { @@ -2462,9 +2462,9 @@ ], "support": { "issues": "https://github.com/utopia-php/query/issues", - "source": "https://github.com/utopia-php/query/tree/0.3.3" + "source": "https://github.com/utopia-php/query/tree/0.6.0" }, - "time": "2026-07-06T04:55:56+00:00" + "time": "2026-08-21T11:03:36+00:00" }, { "name": "utopia-php/telemetry", diff --git a/packages/audit/tests/Audit/Adapter/ClickHouseSqlSnapshotTest.php b/packages/audit/tests/Audit/Adapter/ClickHouseSqlSnapshotTest.php index 0bd22b7a5..3a52cf9ef 100644 --- a/packages/audit/tests/Audit/Adapter/ClickHouseSqlSnapshotTest.php +++ b/packages/audit/tests/Audit/Adapter/ClickHouseSqlSnapshotTest.php @@ -253,7 +253,7 @@ public function testContainsEmitsPositionPredicate(): void $statement = $this->newAuditBuilder() ->from('default.audits') ->selectRaw('`id`, `event`, `time`') - ->filter([Query::contains('event', ['dat'])]) + ->filter([Query::containsString('event', ['dat'])]) ->build(); $this->assertSame( @@ -269,7 +269,7 @@ public function testContainsMultiValueOrsPositionPredicates(): void $statement = $this->newAuditBuilder() ->from('default.audits') ->selectRaw('`id`') - ->filter([Query::contains('event', ['dat', 'ins'])]) + ->filter([Query::containsString('event', ['dat', 'ins'])]) ->build(); $this->assertSame( diff --git a/packages/audit/tests/Audit/Adapter/ClickHouseTest.php b/packages/audit/tests/Audit/Adapter/ClickHouseTest.php index 16e2f2100..cc0b06192 100644 --- a/packages/audit/tests/Audit/Adapter/ClickHouseTest.php +++ b/packages/audit/tests/Audit/Adapter/ClickHouseTest.php @@ -1009,7 +1009,7 @@ public function testContainsSubstringQuery(): void // Contains is a substring match (like utopia-php/database), not an // exact IN match — 'dat' matches only the 'update' logs $logs = $this->audit->find([ - Query::contains('event', ['dat']), + Query::containsString('event', ['dat']), ]); $this->assertCount(2, $logs); foreach ($logs as $log) { @@ -1018,7 +1018,7 @@ public function testContainsSubstringQuery(): void // Multiple needles OR together: 'dat' (update) + 'ins' (insert) $logs = $this->audit->find([ - Query::contains('event', ['dat', 'ins']), + Query::containsString('event', ['dat', 'ins']), ]); $this->assertCount(3, $logs); } @@ -1028,12 +1028,12 @@ public function testContainsEscapesLikeWildcards(): void // '%' and '_' in needles are literals, not LIKE wildcards — // no fixture event contains a literal '%' $logs = $this->audit->find([ - Query::contains('event', ['%']), + Query::containsString('event', ['%']), ]); $this->assertCount(0, $logs); $logs = $this->audit->find([ - Query::contains('event', ['u_date']), + Query::containsString('event', ['u_date']), ]); $this->assertCount(0, $logs); } @@ -1135,7 +1135,7 @@ public function testContainsRejectsEmptyValues(): void $this->expectExceptionMessage('Contains queries require at least one value.'); $this->audit->find([ - Query::contains('event', []), + Query::containsString('event', []), ]); } diff --git a/packages/audit/tests/Audit/QueryTest.php b/packages/audit/tests/Audit/QueryTest.php index f1d59fd3e..0232bf63f 100644 --- a/packages/audit/tests/Audit/QueryTest.php +++ b/packages/audit/tests/Audit/QueryTest.php @@ -39,7 +39,7 @@ public function testQueryStaticFactoryMethods(): void $this->assertSame(['2023-01-01', '2024-01-01'], $query->getValues()); // Test contains - $query = Query::contains('event', ['create', 'update', 'delete']); + $query = Query::containsString('event', ['create', 'update', 'delete']); $this->assertSame(Query::TYPE_CONTAINS, $query->getMethod()->value); $this->assertSame('event', $query->getAttribute()); $this->assertSame(['create', 'update', 'delete'], $query->getValues()); From f772c93cb91186f89b5159d08b7c2e7b23de4d41 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 25 Aug 2026 11:41:22 +0545 Subject: [PATCH 5/5] =?UTF-8?q?refactor(audit):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20inline=20a=20single-use=20helper,=20unify=20the=20t?= =?UTF-8?q?ype=20maps,=20bind=20the=20tenant=20filter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four of Chirag's five comments were actionable. mapAttributeType() had one call site and held a one-line ternary. AGENTS.md is explicit that a helper whose only job is to stop a short expression appearing twice should be inlined, so it is. getParamType() and getColumnTypeMap() were two type tables that could disagree. getColumnTypeMap() derives from getAttributes(), typing every VAR_DATETIME attribute as DateTime64(3); getParamType() hardcoded a match that recognised only 'time'. A second datetime column would have been bound as String by one and declared DateTime64(3) by the other. getParamType() now reads the same map. getTenantFilter() returned a raw ' AND `tenant` = 42' fragment that four call sites re-attached with $builder->whereRaw(ltrim($fragment, ' AND')). Two problems: the tenant was interpolated rather than bound, alone among this adapter's predicates, and ltrim() with ' AND' trims a character set - any leading space, A, N or D - not a prefix, so it only worked because of what followed it. Replaced with applyTenantFilter($builder), which pushes Query::equal('tenant', [...]) through the builder like every other filter. CHANGELOG said 'query 0.3 builder' and 'to 0.3.* (locked at 0.3.3)' after the bump to 0.6 - the file a consumer reads to decide whether an upgrade breaks them. The fifth comment asked whether the empty-values guard is needed. It is, and it is answered on the thread. bin/monorepo check audit: pint, phpstan [OK] No errors, rector [OK]. Snapshot and query suites 22 tests / 106 assertions. The tenant path is covered by the live-server shared-tables test in CI. --- packages/audit/CHANGELOG.md | 4 +- .../audit/src/Audit/Adapter/ClickHouse.php | 53 ++++++------------- 2 files changed, 18 insertions(+), 39 deletions(-) diff --git a/packages/audit/CHANGELOG.md b/packages/audit/CHANGELOG.md index 8b6933918..8290eba15 100644 --- a/packages/audit/CHANGELOG.md +++ b/packages/audit/CHANGELOG.md @@ -4,11 +4,11 @@ All notable changes to `utopia-php/audit` are documented in this file. ## Unreleased -### ClickHouse adapter — migrated to the utopia-php/query 0.3 builder +### ClickHouse adapter — migrated to the utopia-php/query 0.6 builder #### Changed -- `utopia-php/query` bumped from `0.1.*` to `0.3.*` (locked at 0.3.3). +- `utopia-php/query` bumped from `0.1.*` to `0.6.*` (locked at 0.6.0). - `setup()` builds its DDL through `Utopia\Query\Schema\ClickHouse` instead of hand-assembled SQL. Column types, `LowCardinality(...)` / `Nullable(...)` wrapping, bloom-filter indexes, engine, `ORDER BY`, `PARTITION BY` and diff --git a/packages/audit/src/Audit/Adapter/ClickHouse.php b/packages/audit/src/Audit/Adapter/ClickHouse.php index 4476d9e21..7b3bdbe08 100644 --- a/packages/audit/src/Audit/Adapter/ClickHouse.php +++ b/packages/audit/src/Audit/Adapter/ClickHouse.php @@ -1058,7 +1058,9 @@ public function setup(): void continue; } - $column = $table->addColumn($id, $this->mapAttributeType($attribute)); + $column = $table->addColumn($id, ($attribute['type'] ?? null) === Database::VAR_DATETIME + ? ColumnType::Datetime + : ColumnType::String); if (\in_array($id, self::LOW_CARDINALITY_COLUMNS, true)) { $column->lowCardinality(); } @@ -1127,17 +1129,6 @@ public function setup(): void } } - /** - * Map an audit attribute descriptor to its `Schema\ColumnType`. - * - * @param array $attribute - */ - private function mapAttributeType(array $attribute): ColumnType - { - return ($attribute['type'] ?? null) === Database::VAR_DATETIME - ? ColumnType::Datetime - : ColumnType::String; - } /** * Get column names from attributes. @@ -1275,10 +1266,7 @@ public function getById(string $id): ?Log ->filter([Query::equal('id', $id)]) ->limit(1); - $tenantFilter = $this->getTenantFilter(); - if ($tenantFilter !== '') { - $builder->whereRaw(ltrim($tenantFilter, ' AND')); - } + $this->applyTenantFilter($builder); $statement = $builder->build(); $sql = $statement->query . ' FORMAT JSON'; @@ -1324,10 +1312,7 @@ public function find(array $queries = []): array ->selectRaw($selectColumns) ->filter($parsed['filters']); - $tenantFilter = $this->getTenantFilter(); - if ($tenantFilter !== '') { - $builder->whereRaw(ltrim($tenantFilter, ' AND')); - } + $this->applyTenantFilter($builder); $cursorDirection = $parsed['cursorDirection'] ?? null; $orderAttributes = $parsed['orderAttributes']; @@ -1447,10 +1432,7 @@ public function count(array $queries = [], ?int $max = null): int ->selectRaw($max !== null ? '1' : 'COUNT(*) AS count') ->filter($parsed['filters']); - $tenantFilter = $this->getTenantFilter(); - if ($tenantFilter !== '') { - $inner->whereRaw(ltrim($tenantFilter, ' AND')); - } + $this->applyTenantFilter($inner); if ($max !== null) { $inner->limit($max); @@ -1740,11 +1722,12 @@ private function normalizeCursorRow(mixed $rawCursor): array */ private function getParamType(string $attribute): string { - return match (true) { - $attribute === 'time' => 'DateTime64(3)', - $attribute === 'tenant' && $this->sharedTables => 'UInt64', - default => 'String', - }; + // Derived from the same map the builder is given, rather than a second + // hardcoded list. The two used to be able to disagree: this matched only + // `time`, while getColumnTypeMap() types every VAR_DATETIME attribute, so + // a second datetime column would have been bound as String here and + // declared DateTime64(3) there. + return $this->getColumnTypeMap()[$attribute] ?? 'String'; } /** @@ -2125,14 +2108,13 @@ private function getSelectColumns(): string * Build tenant filter clause based on current tenant context. * Escapes column name to prevent SQL injection. */ - private function getTenantFilter(): string + private function applyTenantFilter(ClickHouseBuilder $builder): void { if (!$this->sharedTables || $this->tenant === null) { - return ''; + return; } - $escapedTenant = $this->escapeIdentifier('tenant'); - return " AND {$escapedTenant} = {$this->tenant}"; + $builder->filter([Query::equal('tenant', [$this->tenant])]); } /** @@ -2430,10 +2412,7 @@ public function cleanup(\DateTime $datetime): bool ->into($qualifiedTable) ->whereRaw($escapedTimeColumn . ' < {datetime:DateTime64(3)}'); - $tenantFilter = $this->getTenantFilter(); - if ($tenantFilter !== '') { - $builder->whereRaw(ltrim($tenantFilter, ' AND')); - } + $this->applyTenantFilter($builder); if ($this->asyncCleanup) { $builder->settings(['lightweight_deletes_sync' => '0']);