diff --git a/Justfile b/Justfile index ef884f47a1..2d4e741659 100644 --- a/Justfile +++ b/Justfile @@ -138,9 +138,7 @@ docs-api: # Regenerate Parquet Thrift PHP classes. gen-thrift: && fix - grep -q 'namespace php Flow.Parquet.ThriftModel' src/lib/parquet/src/Flow/Parquet/Resources/Thrift/parquet.thrift || { echo "Flow php namespace not found in thrift definition!"; exit 1; } - rm src/lib/parquet/src/Flow/Parquet/ThriftModel/*.php - thrift --gen php --out src/lib/parquet/src src/lib/parquet/src/Flow/Parquet/Resources/Thrift/parquet.thrift + php bin/thrift.php parquet # Regenerate PostgreSQL pg_query Protobuf PHP classes. gen-protobuf-pg: && fix diff --git a/bin/docs.php b/bin/docs.php index 833a0db5f7..0ac9d5ff8e 100755 --- a/bin/docs.php +++ b/bin/docs.php @@ -73,6 +73,7 @@ public function execute(InputInterface $input, OutputInterface $output): int __DIR__ . '/../src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/functions.php', __DIR__ . '/../src/lib/filesystem/src/Flow/Filesystem/DSL/functions.php', __DIR__ . '/../src/lib/types/src/Flow/Types/DSL/functions.php', + __DIR__ . '/../src/lib/parquet/src/Flow/Parquet/DSL/functions.php', __DIR__ . '/../src/lib/postgresql/src/Flow/PostgreSql/DSL/schema.php', __DIR__ . '/../src/lib/postgresql/src/Flow/PostgreSql/DSL/query.php', __DIR__ . '/../src/lib/postgresql/src/Flow/PostgreSql/DSL/condition.php', diff --git a/bin/thrift.php b/bin/thrift.php new file mode 100755 index 0000000000..241dff0b5b --- /dev/null +++ b/bin/thrift.php @@ -0,0 +1,100 @@ +#!/usr/bin/env php +addCommand(new class extends Command { + public function configure(): void + { + $this + ->setName('parquet') + ->setDescription('Regenerates the Parquet ThriftModel classes from parquet.thrift.') + ->addOption( + 'thrift', + null, + InputOption::VALUE_REQUIRED, + 'Thrift compiler binary, found in PATH when omitted', + ); + } + + public function execute(InputInterface $input, OutputInterface $output): int + { + $source = __DIR__ . '/../src/lib/parquet/src'; + $idl = $source . '/Flow/Parquet/Resources/Thrift/parquet.thrift'; + $models = $source . '/Flow/Parquet/ThriftModel'; + + $thrift = $input->getOption('thrift') === null + ? (new ExecutableFinder())->find('thrift') + : type_string()->assert($input->getOption('thrift')); + + if (!is_string($thrift) || !is_executable($thrift)) { + $output->writeln('Thrift compiler not found, pass an executable with --thrift'); + + return Command::FAILURE; + } + + if (!str_contains((string) file_get_contents($idl), 'namespace php Flow.Parquet.ThriftModel')) { + $output->writeln('Flow php namespace not found in thrift definition!'); + + return Command::FAILURE; + } + + foreach (glob($models . '/*.php') ?: [] as $file) { + unlink($file); + } + + (new Process([$thrift, '--gen', 'php', '--out', $source, $idl]))->mustRun( + static fn(string $type, string $buffer) => $output->write($buffer), + ); + + // the Thrift compiler types every field non-null; a field the IDL does not require is null when a file leaves it out + foreach (glob($models . '/*.php') ?: [] as $file) { + $class = 'Flow\\Parquet\\ThriftModel\\' . basename($file, '.php'); + + if (!class_exists($class) || !property_exists($class, '_TSPEC')) { + continue; + } + + /** @var array $spec */ + $spec = (new ReflectionProperty($class, '_TSPEC'))->getValue(); + $model = (string) file_get_contents($file); + $rewritten = $model; + + foreach ($spec as $field) { + if ($field['isRequired']) { + continue; + } + + $rewritten = (string) preg_replace( + '/(\* @var )(?!null\|)([^\n]+)(\n\s*\*\/\n\s*public \$' + . preg_quote($field['var'], '/') + . ' = null;)/', + '$1null|$2$3', + $rewritten, + ); + } + + if ($rewritten !== $model) { + file_put_contents($file, $rewritten); + } + } + + return Command::SUCCESS; + } +}); + +$application->run(); diff --git a/composer.json b/composer.json index 48de3ffb0d..1bca0798ee 100644 --- a/composer.json +++ b/composer.json @@ -260,6 +260,7 @@ "src/lib/filesystem/src/Flow/Filesystem/DSL/functions.php", "src/lib/parquet/src/Flow/Parquet/functions.php", "src/lib/parquet/src/Flow/Parquet/Binary/functions.php", + "src/lib/parquet/src/Flow/Parquet/DSL/functions.php", "src/lib/parquet/src/stubs.php", "src/lib/postgresql/src/Flow/PostgreSql/DSL/client.php", "src/lib/postgresql/src/Flow/PostgreSql/DSL/condition.php", diff --git a/documentation/components/adapters/parquet.md b/documentation/components/adapters/parquet.md index 862480ec79..06d5153c8b 100644 --- a/documentation/components/adapters/parquet.md +++ b/documentation/components/adapters/parquet.md @@ -73,3 +73,72 @@ data_frame() ->write(to_output()) ->run(); ``` + +A `UINT_64` column reads as `numeric_string` (`'18446744073709551615'` is past `PHP_INT_MAX`). A `numeric_string` +column is written as a parquet `STRING`; write `UINT_64` through the lib with an explicit schema: + +```php +write( + __DIR__ . '/ids.parquet', + Schema::with(new FlatColumn('id', PhysicalType::INT64, logicalType: LogicalType::integer(64, false))), + [['id' => 7], ['id' => '18446744073709551615']], +); +``` + +## Filter pushdown + +A `filter()` directly over `from_parquet()` moves into the reader, which returns only matching rows: row groups whose +statistics prove that no row matches are skipped, and a `limit()` after the filter stops the read at that many +matching rows: + +```php +read(from_parquet(__DIR__ . '/orders/*.parquet')) + ->filter(ref('id')->greaterThan(lit(1000))->and(ref('status')->equals(lit('paid')))) + ->limit(10) + ->write(to_output()) + ->run(); +``` + +Each part of an `and()` is pushed on its own when it compares a file column with a literal it can compare exactly. + +Pushed: + +- `equals`, `notEquals`, `same`, `notSame`, `lessThan`, `lessThanEqual`, `greaterThan`, `greaterThanEqual`, + `between`, `isIn` (literal list), `startsWith`, `isNull`, `isNotNull`, and `not()`/`and()`/`or()` of those +- an int literal on a float column up to 2^53, an integral float on an int column below 2^53 +- a `date` literal on a `datetime` column and the reverse +- a digit string (`lit('1000')`) on a `numeric_string` (`UINT_64`) column + +Evaluated on the rows read: + +- other functions, two columns compared, a literal of another type +- `json` columns (except `isNull`/`isNotNull`), ordering on `uuid` +- a column the file stores as another type than the read + +Partition columns list fewer files instead ([partitioning](/documentation/components/core/partitioning.md)). + +`explain()` shows the filter and the limit under `Read`, not what the reader skipped: + +``` +#3 Result +│ Rows this plan hands out: to the trigger, or to the node reading it +└─ #2 Limit + │ Limit: 10 + └─ #1 Read + Extractor: ParquetExtractor + Source: file:///orders/*.parquet + Limit: 10 + Filter: All +``` diff --git a/documentation/components/core/core.md b/documentation/components/core/core.md index 02254f9456..01d6a5f377 100644 --- a/documentation/components/core/core.md +++ b/documentation/components/core/core.md @@ -67,7 +67,7 @@ Outputs ├─ #4 Result │ │ Rows this plan hands out: to the trigger, or to the node reading it │ └─ #3 Select -│ │ Columns: id, email +│ │ Keeps only: id, email │ └─ #2 Filter │ │ Condition: IsNotNull │ └─ #1 Read @@ -129,7 +129,7 @@ echo $dataFrame->explain()->toString(format: Format::flow); └─ #2 Filter │ Condition: IsNotNull ├─ #3 Select - │ │ Columns: id, email + │ │ Keeps only: id, email │ └─ #4 Result │ Rows this plan hands out: to the trigger, or to the node reading it └─ #5 Write @@ -154,7 +154,7 @@ echo $dataFrame->explain()->toString(format: Format::boxes); ┌─────────────┴─────────────┐┌─────────────┴─────────────┐ │ #3 Select ││ #2 Filter │ │ ──────────────────── ││ (shared) │ -│ Columns: id, email ││ │ +│ Keeps only: id, email ││ │ └─────────────┬─────────────┘└───────────────────────────┘ ┌─────────────┴─────────────┐ │ #2 Filter │ diff --git a/documentation/components/core/partitioning.md b/documentation/components/core/partitioning.md index 186cc68419..feb42eb533 100644 --- a/documentation/components/core/partitioning.md +++ b/documentation/components/core/partitioning.md @@ -268,8 +268,10 @@ it from the file schema (Parquet, Floe), so `filter(ref('date')->between(lit($fr ### Partition Pruning -The optimizer pushes a `filter()` that reads only partition columns into the source, so whole directories -are skipped; a filter on a body column still reads everything and then discards. A partition column renamed +The optimizer moves a `filter()` into the file source, which returns only matching rows. A filter that reads only +partition columns skips whole directories; a filter on a body column reads every file and drops the rows it does not +match (Parquet skips row groups, see [filter pushdown](/documentation/components/adapters/parquet.md#filter-pushdown)). +A `limit()` after the filter reaches the source with it ([filter pushdown](/documentation/components/adapters/parquet.md#filter-pushdown)). A partition column renamed with `rename()` or copied with `withEntry('day', ref('date'))` is still pruned; any other redefinition of it stops the push. A filter whose answer can change between two evaluations - `call()`, `now()`, `uuid()`, `to_date_time()` with a format that leaves fields out (`'Y-m-d'`) - still filters rows but prunes nothing. @@ -283,6 +285,7 @@ use function Flow\ETL\DSL\{data_frame, lit, ref, to_output}; data_frame() ->read(from_csv(__DIR__ . '/output/date=*/department=*/*.csv')) ->filter(ref('date')->greaterThanEqual(lit('2024-01-01'))) + ->limit(10) ->write(to_output()) ->run(); ``` diff --git a/documentation/components/extensions/arrow-ext.md b/documentation/components/extensions/arrow-ext.md index f79d40880f..1c2dcdc35d 100644 --- a/documentation/components/extensions/arrow-ext.md +++ b/documentation/components/extensions/arrow-ext.md @@ -92,18 +92,23 @@ use Flow\Parquet\Engine\{RustParquetFileReader, RustParquetFileWriter}; use Flow\Parquet\Engine\Arrow\{OptionsConverter, SchemaConverter}; use Flow\Parquet\Options; use Flow\Parquet\ParquetFile\Compressions; +use Flow\Parquet\ParquetFile\Index\StreamIndexes; +use Flow\Parquet\Reader\ReadPlan; use function Flow\Filesystem\DSL\{native_local_filesystem, path}; $filesystem = native_local_filesystem(); $file = new RustParquetFileReader($filesystem->readFrom(path('data.parquet'))); // Flow\Parquet\ParquetFileReader, one footer read -$file->rowsNumber(); -$file->schema(); // Flow\Parquet\ParquetFile\Schema -$file->metadata(); // Flow\Parquet\ParquetFile\Metadata -$file->thrift(); // Flow\Parquet\ThriftModel\FileMetaData - -foreach ($file->readColumns(['id', 'name'], 1_000, null, null) as $chunk) { // RustColumnsReader, an Iterator that streams once +$footer = $file->footer(); // Flow\Parquet\Engine\RustParquetFooter, readable after close() +$footer->rowsNumber(); +$footer->schema(); // Flow\Parquet\ParquetFile\Schema +$footer->metadata(); // Flow\Parquet\ParquetFile\Metadata +$footer->thrift(); // Flow\Parquet\ThriftModel\FileMetaData +$indexes = new StreamIndexes($footer, $filesystem->readFrom(path('data.parquet'))); // page indexes, read on demand +$plan = ReadPlan::window($footer->rowsNumber(), $indexes, offset: 10, limit: 100); // rows 10-109 + +foreach ($file->readColumns(['id', 'name'], 1_000, $plan) as $chunk) { // RustColumnsReader, an Iterator that streams once // ['id' => [1, 2, ...], 'name' => ['a', 'b', ...]] } @@ -118,7 +123,7 @@ $writer->writeBatch([['id' => 1, 'name' => 'a']]); // an array or any Traversa $writer->writeColumns(['id' => [2, 3], 'name' => ['b', null]]); $writer->close(); // footer, then the stream closed -$batches = new RustBatchReader($file, ['id', 'name'], 1_000, null, null); // batch size, offset, limit +$batches = new RustBatchReader($file, ['id', 'name'], 1_000, $plan); // batch size, rows to read $batches->schema(); // Flow\Arrow\RustArrowSchema $batches->next(); // Flow\Arrow\RustParquetBatch, null after the last ``` diff --git a/documentation/components/libs/parquet.md b/documentation/components/libs/parquet.md index 37b52280a7..49b89484c2 100644 --- a/documentation/components/libs/parquet.md +++ b/documentation/components/libs/parquet.md @@ -148,10 +148,10 @@ invalid file throws `InvalidArgumentException` `Given file is not valid Parquet There are several things we can read from parquet file: -- `ParquetFile::values(array $columns = [], ?int $limit = null, ?int $offset = null) : \Generator` -- `ParquetFile::columns(int $batchSize, array $columns = [], ?int $limit = null, ?int $offset = null) : \Generator` +- `ParquetFile::values(Query $query = new Query()) : \Generator` - rows, one by one +- `ParquetFile::columns(Query $query = new Query(), int $batchSize = 1024) : \Generator` - columns, in batches - `ParquetFile::metadata() : Metadata` -- `ParquetFile::schema() : Schema` - shortcut for `ParquetFile::metadata()->schema()` +- `ParquetFile::schema() : Schema` - the file schema, without building row-group metadata - `ParquetFile::close() : void` - closes the reader and its stream ### Reading the whole file: @@ -171,12 +171,12 @@ foreach ($file->values() as $row) { ```php use Flow\Parquet\Reader; +use function Flow\Parquet\DSL\parquet_query; -$reader = new Reader(); +$file = (new Reader())->read('path/to/file.parquet'); -$file = $reader->read('path/to/file.parquet'); -foreach ($file->values(["column_1", "column_2"]) as $row) { - // do something with $row +foreach ($file->values(parquet_query()->select('column_1', 'column_2')) as $row) { + // ['column_1' => ..., 'column_2' => ...] } ``` @@ -187,29 +187,88 @@ foreach ($file->values(["column_1", "column_2"]) as $row) { > To achieve the best results, we will need to play a bit with Writer options (covered later). ```php -use Flow\Parquet\Reader; +foreach ($file->values(parquet_query()->select('column_1')->offset(1000)->limit(100)) as $row) { + // rows 1000-1099 +} +``` -$reader = new Reader(); +### Reading columns in batches -$file = $reader->read('path/to/file.parquet'); -foreach ($file->values(["column_1", "column_2"], limit: 100, offset: 1000) as $row) { - // do something with $row +```php +foreach ($file->columns(parquet_query()->select('column_1', 'column_2'), batchSize: 1000) as $chunk) { + // ['column_1' => [...up to 1000 values], 'column_2' => [...]] } ``` -### Reading columns in batches +### Filtering reads ```php use Flow\Parquet\Reader; +use function Flow\Parquet\DSL\{parquet_column, parquet_query}; $file = (new Reader())->read('path/to/file.parquet'); -foreach ($file->columns(batchSize: 1000, columns: ["column_1", "column_2"]) as $chunk) { - // $chunk = ['column_1' => [...up to 1000 values], 'column_2' => [...]] +$query = parquet_query() + ->select('id', 'country') + ->where( + parquet_column('country')->equals('PL') + ->and(parquet_column('user.age')->lessThan(18)->negate()) + ->and(parquet_column('status')->in(['new', 'paid'])->or(parquet_column('note')->isNull())), + ) + ->offset(20) + ->limit(10); + +foreach ($file->values($query) as $row) { + // only matching rows: the 21st to the 30th match } ``` -`limit` and `offset` work as in `values()`. +The rows returned are exactly the matching ones; offset and limit count matching rows. Row groups and pages whose +statistics (column chunk statistics, the [page index](#page-index)) prove no row matches are never read - both engines +skip the same ones and read a kept page's bytes alone. A second `where()` narrows the first. + +| Method on `parquet_column($path)` | Filter | +|------------------------------------------------------------------------------|--------------------------| +| `equals`, `lessThan`, `lessThanOrEqual`, `greaterThan`, `greaterThanOrEqual` | `Comparison` | +| `notEquals` | `Not` of `equals` | +| `in(array)`, `notIn(array)` | `In`, `Not` of `In` | +| `isNull()`, `isNotNull()` | `IsNull`, its `Not` | +| `startsWith(string)` | `StartsWith` | + +- `->and()`, `->or()`, `->negate()` on any filter build `All`, `Any`, `Not`; `new All(...$filters)` and + `new Any(...$filters)` combine a list (namespace `Flow\Parquet\Filter`). +- `$path` names a leaf through structs (`'user.address.city'`); a column the file lacks reads as null. A leaf inside a + list or a map is refused (`InvalidArgumentException`): a row holds no single value there. +- Semantics are SQL's: a comparison on null is null, `NOT` null is null, only true keeps a row. Floats follow IEEE 754 + (`NaN` matches nothing, `-0.0` equals `0.0`), strings compare in byte order (`'10' < '9'`), `UINT_64` values + numerically, date-times by instant. A literal of another kind than the column's values (a string on an `INT64` + column) matches nothing, under `->negate()` too. +- Literals: scalars, `DateTimeInterface`, `Stringable` (uuids); `null` (use `isNull()`) and `NaN` are refused. +- Files written without `column_orders` (older flow-php versions) are never pruned, only filtered. The statistics a file + holds are set by the writer options in [Statistics](#statistics). + +#### Reading with a ReadPlan + +```php +use Flow\Parquet\Filter\Binding\BoundFilter; +use Flow\Parquet\Reader\ReadPlanner; +use function Flow\Parquet\DSL\parquet_column; + +$filter = parquet_column('id')->greaterThan(100); +$plan = (new ReadPlanner())->plan( + $file->reader()->footer(), + $file->indexes(), + BoundFilter::bind($filter, $file->schema(), $file->options()), + limit: 10, // rows kept by $plan->rowFilter, refused unless it is the whole filter +); + +foreach ($file->reader()->readColumns(['id'], 1024, $plan) as $batch) { + // only rows matching $plan->rowFilter +} +``` + +`ReadPlan::$rowFilter` holds the conjuncts the file compares exactly; rows come back matching them, a superset of the +query's matches when they are not the whole filter. ## Writing Parquet Files @@ -331,15 +390,24 @@ $writer->close(); ### Writer Options -- `DICTIONARY_PAGE_MIN_CARDINALITY_RATION` - default '0.4' - minimum ratio of unique values to total values for a column - to have dictionary encoding. -- `DICTIONARY_PAGE_SIZE` - default: `1Mb` - maximum size of dictionary page. +- `COLUMN_INDEX_TRUNCATE_LENGTH` - default: `64` - bytes a string/binary page's min/max in the column index are cut to. +- `DICTIONARY_PAGE_SIZE` - default: `1Mb` - maximum size of dictionary page; the pages of a column chunk after it is + reached are written `PLAIN`. - `GZIP_COMPRESSION_LEVEL` - default: `9` - compression level for GZIP compression (applied only when GZIP compression is enabled). +- `COLUMNS_BLOOM_FILTERS` - default: `null` - `BloomFilterOptions` by column flat path: those columns get a bloom filter + per row group ([Bloom filters](#bloom-filters)). +- `PAGE_HEADER_STATISTICS` - default: `false` - data page headers carry the page's statistics (at `StatisticsLevel::PAGE`). +- `PAGE_MAXIMUM_ROWS_COUNT` - default: `1000` - rows per data page: the PHP writer closes a page at exactly that many; + the arrow writer treats it as a target, checked between write batches, so its pages can hold more. Smaller pages let + the page index skip more of a filtered read. - `PAGE_SIZE_BYTES` - default: `128Kb` - maximum size of data page. - `ROW_GROUP_SIZE_BYTES` - default: `32Mb` - maximum size of row group. - `ROW_GROUP_SIZE_CHECK_INTERVAL` default: `1000` - number of rows to write before checking if row group size limit is reached. +- `STATISTICS_LEVEL` - default: `StatisticsLevel::PAGE` - `NONE`, `CHUNK` (column chunk statistics) or `PAGE` (also a + column index); the offset index is written at every level. +- `STATISTICS_TRUNCATE_LENGTH` - default: `64` - bytes a string/binary column chunk's min/max statistics are cut to. - `VALIDATE_DATA` - default: `true` - if set to `true` then writer will validate data against schema. - `WRITER_VERSION` - default `1` - tells writer which version of parquet format should be used. @@ -369,6 +437,89 @@ more frequently. Unfortunately, there is no one size fits all solution here. You will need to play a bit with those values to find the best one for your use case. +## Statistics + +Both engines write the column chunk statistics arrow-rs writes: `min_value`/`max_value` with exactness flags, +`nan_count` for floats and a column order per leaf. + +```php +set(Option::STATISTICS_TRUNCATE_LENGTH, 16))) + ->write(__DIR__ . '/orders.parquet', $schema, $rows); + +$metadata = (new Reader())->read(__DIR__ . '/orders.parquet')->metadata(); +$column = $metadata->schema()->getFlat('customer'); + +foreach ($metadata->rowGroups()->all() as $rowGroup) { + $statistics = $rowGroup->getColumnChunk($column)->statistics(); + + $statistics?->minValue($column); // physical value: bytes for strings, unscaled int for decimal(10, 2) + $statistics?->maxValue($column); + $statistics?->isMaxValueExact(); // false when the max was truncated and incremented + $statistics?->nullCount(); + $statistics?->nanCount(); // FLOAT, DOUBLE and FLOAT16 only +} + +$metadata->columnOrder($column); // ColumnOrder::TypeDefined; ColumnOrder::Absent in files written before it +``` + +| Physical type | Order of `min_value`/`max_value` | +|----------------------------------|----------------------------------------------------------------------------------------| +| BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY | unsigned bytes (`'10' < '9'`) | +| DECIMAL | signed (two's complement bytes, or the unscaled INT32/INT64) | +| INT32/INT64 `UINT_*` | unsigned | +| FLOAT, DOUBLE, FLOAT16 | IEEE 754 total order (`-0.0 < 0.0`); NaN counted, a bound only when every value is NaN | +| INT96, INTERVAL | none: no `min_value`/`max_value` is written | + +Strings and binaries longer than `STATISTICS_TRUNCATE_LENGTH` keep a prefix as the min and an incremented prefix as the +max. Decimals and FLOAT16 are never truncated. + +### Page index + +Both writers write a column index (each page's min, max, null and NaN counts) and an offset index (where each page +starts, and its first row) by default. Pages start on row boundaries, lists included. + +```php +set(Option::STATISTICS_LEVEL, StatisticsLevel::CHUNK))) + ->write(__DIR__ . '/orders.parquet', $schema, $rows); // no column index; NONE also drops chunk statistics + +$file = (new Reader())->read(__DIR__ . '/orders.parquet'); +$column = $file->schema()->getFlat('customer'); + +$file->indexes()->columnIndex(0, $column); // ?ColumnIndex of row group 0: nullPages, minValues, maxValues, ... +$file->indexes()->offsetIndex(0, $column); // ?OffsetIndex: pages[i]->offset, compressedPageSize, firstRowIndex +``` + +### Bloom filters + +A bloom filter per column chunk lets a read skip the row groups that cannot hold a value, where min/max cannot (an id +inside every group's range). Only `equals()` and `in()` use them. Both writers write them after each row group, for the +columns asked only. + +```php +set(Option::COLUMNS_BLOOM_FILTERS, [ + 'id' => new BloomFilterOptions(), // fpp 0.05, ndv 1 048 576 (the arrow-rs defaults) + 'user.email' => new BloomFilterOptions(0.01, 10_000), +])))->write(__DIR__ . '/users.parquet', $schema, $rows); + +(new Reader())->read(__DIR__ . '/users.parquet') + ->values(parquet_query()->select('id')->where(parquet_column('id')->in([42, 1337]))); +``` + ## Compressions > [!NOTE] diff --git a/documentation/contributing/rust.md b/documentation/contributing/rust.md index 79c6a7cf77..cdd6bfeac5 100644 --- a/documentation/contributing/rust.md +++ b/documentation/contributing/rust.md @@ -169,9 +169,10 @@ holds the same number: | flow_php | `FLOW_PHP_ABI` | `Flow\ETL\FlowPhpExtension::ABI` | | arrow | `FLOW_ARROW_ABI` | `Flow\Parquet\Engine\ArrowExtension::ABI` | -Bump both, in one change, whenever a class or interface the extension registers changes. Every `Adaptive*` pick calls -`available()`: an extension that is loaded with another ABI, or with none, is refused with a `RuntimeException` -naming the extension and its version. phpt `099` (flow_php) and `054` (arrow) pin that the two numbers are equal. +Bump both, in one change, whenever the contract changes: a class or interface the extension registers, or the +schema, options or values that cross it. Every `Adaptive*` pick calls `available()`: an extension that is loaded with +another ABI, or with none, is refused with a `RuntimeException` naming the extension and its version. phpt `099` +(flow_php) and `054` (arrow) pin that the two numbers are equal. The PHP interfaces an extension also registers are guarded by `interface_exists(X::class, false)`, so an extension that does not register one still gets the PHP declaration and fails at the ABI check, not with "Class not found". diff --git a/documentation/upgrading.md b/documentation/upgrading.md index 6a715ea207..ccbe728323 100644 --- a/documentation/upgrading.md +++ b/documentation/upgrading.md @@ -19,6 +19,117 @@ Upgrading from older versions: [0.35.x to 0.40.x](/documentation/upgrading-0.35- | `ListType>`, `MapType>` | `ListType`, `MapType` | | Mago `plugins = ["flow-php"]` infers `type_structure()` shapes | Built-in plugin no longer infers them - use `flow-php/mago-types-bridge` (`plugins = ["flow/types"]`) | +### 2) `flow-php/parquet` - statistics are written and read as arrow-rs does; one physical schema for one logical schema on both engines + +| Before | After | +|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| PHP writer string bounds by PHP `<` (`'9'` below `'10'`), whole strings, no exactness flags, no `nan_count`, no `column_orders`, deprecated `min`/`max` for every type | unsigned-byte order, truncated to `Option::STATISTICS_TRUNCATE_LENGTH` (64) bytes, exactness flags, `nan_count` for floats, `column_orders` per leaf, deprecated `min`/`max` only for signed sort orders | +| statistics of files written by older flow versions read as written | `ColumnOrder::Absent`: no bounds are trusted, so such files are never pruned; a statistic of the wrong width (older INT64 bounds are 16 bytes) reads as `null` | +| `StatisticsReader` read BYTE_ARRAY statistics with a length prefix, a FLBA decimal as a float, an INT64 decimal as the unscaled int | raw bytes for BYTE_ARRAY/FLBA (decimals as unscaled two's complement bytes, uuid as 16 bytes), the unscaled int for INT32/INT64 decimals | +| `FlatColumn::decimal()` of precision up to 18 written as FIXED_LEN_BYTE_ARRAY | INT32 up to precision 9, INT64 up to 18, FIXED_LEN_BYTE_ARRAY above; older files still read | +| the packer/unpacker encoded FLBA/BYTE_ARRAY decimals and UUIDs; `DecimalConverter` took INT-backed decimals only | `new DecimalConverter($physicalType, $precision, $scale, $byteLength)` and `UuidConverter` convert to the physical bytes; the packer/unpacker take them as-is | +| `FlatColumn::int32()` / `int64()` wrote converted `INT_32` / `INT_64` | plain `INT32` / `INT64`, as arrow-rs | +| a column with a logical type and no converted type written without one (DuckDB read a logical `UINT_64` as signed) | the converted type derived from the logical type, as arrow-rs; a `FlatColumn` whose converted and logical types disagree is refused, on read and write | +| the arrow writer named map entries `m.entries.key` | `m.key_value.key` (LogicalTypes.md): `COLUMNS_*` options keyed by `…key_value.key` address the same leaf on both engines | +| the arrow writer wrote `FlatColumn::enum()` as `STRING` | `ENUM` | +| `Option::ARROW_BATCH_SIZE`, read by nothing | removed | +| `Flow\Parquet\Dremel\Statistics\Comparator` | removed: `Flow\Parquet\Data\PhysicalOrder` | +| `new StatisticsCounter($column)` | `new StatisticsCounter($column, new StatisticsTruncation($length))` | +| `new UnsignedIntegerConverter($path, $bitWidth)` | `new UnsignedInt32Converter()`; UINT_64 is `UnsignedInt64Converter` | + +### 3) `flow-php/parquet`, `flow-php/etl-adapter-parquet` - `UINT_64` reads as a decimal string on both engines (reverses rows 84 and 85 of 0.44→0.45 for `UINT_64`) + +| Before | After | +|-----------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------| +| `UINT_64` read as `int`, refused above `PHP_INT_MAX` on both engines | read as the decimal string (`'18446744073709551615'`) on both engines; `UINT_8/16/32` stay `int` | +| `from_parquet()` typed a `UINT_64` column `integer` | `numeric_string` | +| writes accepted an int from 0 to `PHP_INT_MAX` into `UINT_64` | an int >= 0 or a canonical digit string up to `18446744073709551615` | +| `to_parquet()` refused a `numeric_string` column (`NumericStringType is not supported`) | written as a parquet `STRING` | + +### 4) `flow-php/parquet` - the `arrow` extension must be rebuilt (arrow-rs 60, ABI 2) + +| Before | After | +|---------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| +| arrow-ext built against arrow-rs 59, `FLOW_ARROW_ABI` 1 | arrow-rs 60, `FLOW_ARROW_ABI` / `ArrowExtension::ABI` 2 (covering every contract change below): an older build is refused by `ArrowExtension::available()` | + +### 5) `flow-php/parquet`, `flow-php/etl-adapter-parquet` - reads take a `Query` (or, below the file, a `ReadPlan`) instead of limit/offset, the PHP writer writes the page index, the reader exposes its `Footer`; the `arrow` and `flow_php` extensions must be rebuilt + +| Before | After | +|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `ParquetFile::values(array $columns, ?int $limit, ?int $offset)` | `values(Query $query)`: `$file->values(parquet_query()->select(...$columns)->offset($offset)->limit($limit))` | +| `ParquetFile::columns(int $batchSize, array $columns, ?int $limit, ?int $offset)` | `columns(Query $query, int $batchSize = 1024)` | +| a limit of `0` refused (`Limit must be greater than 0`); a negative offset or limit refused by `columns()` | `limit(0)` reads no row; a negative offset or limit is refused by `Query` | +| `ParquetFileReader::metadata()`, `schema()`, `rowsNumber()`, `totalByteSize()` | `ParquetFileReader::footer(): Footer` holds them (and `rowGroupChunks()`); a footer taken before `close()` stays readable | +| `ParquetFileReader::readColumns(array $columns, int $batchSize, ?int $limit, ?int $offset)` | `readColumns(array $columns, int $batchSize, ReadPlan $plan)`, returning the rows `$plan->selection` selects that match `$plan->rowFilter`, at most `$plan->limit`; build the plan with `ReadPlanner::plan()` or `ReadPlan::window($rows, $file->indexes(), $offset, $limit)` | +| `RustParquetFileReader::thrift()` | `RustParquetFileReader::footer()->thrift()` (`RustParquetFooter`) | +| `ParquetOpenSource::batches(Schema $schema, int $batchSize, ?int $offset, ?int $limit, Backend $backend)` | `batches(Schema $schema, int $batchSize, ReadPlan $plan, Backend $backend)` | +| `RustBatchReader` / `RustColumnsReader::__construct(..., int $batchSize, ?int $offset, ?int $limit)` | `__construct(..., int $batchSize, ReadPlan $plan)` | +| `ColumnReader::read(Column, Metadata, SourceStream, ?int $limit, ?int $offset)` | `readRowGroup(Column, RowGroup, int $group, SourceStream, RowSelection, Indexes, array $decoded = [])`, one row group at a time; `$decoded` pages, keyed by their first row, are served instead of read | +| `new ColumnReader(ColumnChunkReader $chunks, DremelAssembler $assembler)` | `new ColumnReader(new SelectedPages($chunks), $assembler)` | +| `ColumnChunkReader::read(ColumnChunk, FlatColumn, SourceStream)` yielded a list of pages | pages keyed by their first row in the row group; `?OffsetIndex $offsetIndex` with `list $pages` reads only those pages of it, `?Dictionary $dictionary` replaces the dictionary page, the generator returns the dictionary | +| `RowGroupContainer(string, RowGroup)` | `RowGroupContainer(string, RowGroup, list, list)` | +| `new PlainFlatColumnChunkBuilder($column, $options, $compression)` (and the RLE_DICTIONARY, DELTA_BINARY_PACKED builders, `ColumnChunkBuilderFactory::createBuilder()`) | a 4th argument, `StatisticsOptions::fromOptions($options)` | +| `ColumnChunk::dataPageOffset(): ?int` | `int`: a column chunk always records it | +| the PHP writer wrote no column index or offset index | both, by default (`Option::STATISTICS_LEVEL`, `StatisticsLevel::PAGE`; `CHUNK` drops the column index, `NONE` all statistics) | +| the PHP writer wrote page-header statistics on `DATA_PAGE_V2` | none, as arrow-rs: `Option::PAGE_HEADER_STATISTICS` (`true`) writes them, on v1 and v2 pages | +| `Option::PAGE_MAXIMUM_ROWS_COUNT` (default `1000`) was read by nothing | the PHP writer closes a page at exactly that many rows; the arrow writer treats it as a target (arrow-rs `data_page_row_count_limit`, checked between write batches), so its pages can hold more; below 1 is refused (`InvalidArgumentException`) | +| the arrow writer skipped a `COLUMNS_COMPRESSIONS` / `COLUMNS_ENCODINGS` entry whose column path is digits only (`'2024'`, an int array key) | applies it; a key that is not UTF-8 is refused (`InvalidArgumentException`) instead of crashing the process | +| `Option::ROW_GROUP_SIZE_BYTES` below 1 was taken by both writers | refused (`InvalidArgumentException`) by both writers | +| the arrow writer threw `Error` (a Rust panic) on `PLAIN_DICTIONARY` / `RLE_DICTIONARY` in `COLUMNS_ENCODINGS` | enables the column's dictionary: `RLE_DICTIONARY` data pages and a `PLAIN` dictionary page, as the PHP writer writes | +| the arrow writer skipped a `COLUMNS_COMPRESSIONS` / `COLUMNS_ENCODINGS` value that is not a string | refused (`InvalidArgumentException`) | +| files with an `RLE_DICTIONARY` column spanning more than one page, written by the PHP writer | were corrupt (later pages decoded to wrong values): rewrite them; one dictionary per chunk, `PLAIN` past `Option::DICTIONARY_PAGE_SIZE` | +| `Option::DICTIONARY_PAGE_MIN_CARDINALITY_RATION` | removed: it was never read; the dictionary falls back to `PLAIN` on `Option::DICTIONARY_PAGE_SIZE` only, as arrow-rs | +| `Options::set()` / `get()` took scalars and arrays | also a `UnitEnum`; `Options::getEnum(Option $option, string $class)` | +| no column index truncation option | `Option::COLUMN_INDEX_TRUNCATE_LENGTH` (64): BYTE_ARRAY/FIXED_LEN_BYTE_ARRAY page bounds in the column index | +| `DataPageHeaderV2::statistics(Options $options)` | `statistics()`, as `DataPageHeader::statistics()` | +| `ParquetFile::pageHeaders()` repeated every chunk of a row group under each column | each column's own chunk pages, once | +| `FLOW_PHP_ABI` / `FlowPhpExtension::ABI` 1 | 2: rebuild the `flow_php` extension (and `arrow`, item 4); an older build is refused by `available()` | + +### 6) `flow-php/etl`, `flow-php/types` - string comparisons in filters use byte order, as sort already does + +| Before | After | +|------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `ref('a')->lessThan(lit('10'))` on `'9'`: `true` (PHP `<` compares numeric strings as numbers) | `false`: `lessThan`, `lessThanEqual`, `greaterThan`, `greaterThanEqual`, `between`, `greatest` and `least` order two strings by bytes (`strcmp`), as `sortBy()` does | +| `greatest(lit('9'), lit('10'))`: `'10'` | `'9'` | +| a `numeric_string` column was comparable only with another `numeric_string` | comparable with a `string` literal too (`ref('id')->greaterThan(lit('1000'))`), and ordered numerically; still not comparable with an `integer` | +| `Between\Boundary::compareValues(mixed $left, mixed $right)` | removed: `(new ValueOrder(...$operandTypes))->compare($left, $right)`; `Boundary::compare($value, $lower, $upper, ValueOrder $order = new ValueOrder())` | + +### 7) `flow-php/etl` - a file extractor reads a `FileScan` and returns only rows matching its conjuncts; `Read` and `Executor\Pipeline` carry it + +| Before | After | +|--------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `FileExtractor::extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles())` | `FileExtractor::scan(FlowContext $context, FileScan $scan)`, returning only rows matching every one of `$scan->conjuncts`; `extract(FlowContext $context, ?int $limit = null)` is `Extractor`'s, `use Scanning;` makes it `scan()` with a limit | +| `$extractor->extract($context, pathFilter: $filter)` | `$extractor->scan($context, new FileScan(pathFilter: $filter))` | +| `Read::limit()`, `Read::pathFilter()`, `new Read($extractor, $limit, $pathFilter)` | `Read::scan()->limit`, `->pathFilter`, `->conjuncts`; `new Read($extractor, new FileScan($limit, $pathFilter))` | +| `Read::withPathFilter()` on any extractor | refused (`InvalidArgumentException`) on an extractor that is no `FileExtractor`, as `withFilter()` | +| `Executor\Pipeline::limit()`, `pathFilter()`, `new Pipeline(..., $limit, $pathFilter, $sources)` | `Pipeline::scan()`; `new Pipeline(..., $scan, $sources)` | +| a `filter()` over a file source stayed a `Filter` step | each deterministic part of it is pushed into the source (`FileScan::$conjuncts`) and leaves the step, an emptied step is removed; a `limit()` after it is pushed beside it, so the read stops after that many matching rows; `explain()` prints each part as `Filter: …` under `Read` | +| `new FileReadLoop($fileColumns, $schema)`, `read($sources, $batches, $batchSize, Backend $backend, $window)` | `new FileReadLoop($fileColumns, $schema, $scan->conjuncts)`, `read($sources, $batches, $batchSize, FlowContext $context, $window)` | +| `Optimizer::default()` ran `PushLimitIntoSource` before `PushFilterIntoSource` | after it | +| `Rewrite::of(Node $node)` | `of(Node $original, Node $rebuilt)`: the node as it was and rebuilt over its rewritten children | +| `new Flow\ETL\Plan\ReplaceLeaf($target, $replacement)` | `ReplaceNodes::one($target, $replacement)`, or `new ReplaceNodes($replacements, $removals)` for several in one rewrite (a removed node gives way to its child) | +| `new ScalarFunctionFilterTransformer($function, ?ScalarFunction $resolved)` | the second argument is `?Function\Evaluation\Filter`; `bind()` sets it, as before | +| `explain()`: `Select` printed `Columns: …`, other nodes `Defines columns: …` | `Keeps only: …` and `Changes columns: …` (a column defined, renamed or dropped) | +| `Redefined::$unknown` | `Redefined::isUnknown()`; `Redefined::isAllBut()` tells an `allBut()` apart | + +A custom file extractor: + +```php +// before +public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator +{ + // reads with $limit and $pathFilter +} + +// after +use Scanning; // extract() is scan() with only a limit + +public function scan(FlowContext $context, FileScan $scan): Generator +{ + // reads with $scan->limit and $scan->pathFilter, returns only rows matching every one of $scan->conjuncts +} +``` + --- ## Upgrading from 0.45.x to 0.46.x diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 29cbc06bf6..9a8fe2f9a8 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -58,6 +58,9 @@ src/lib/parquet/tests/Flow/Parquet/Tests/Integration + + src/lib/parquet-viewer/tests/Flow/ParquetViewer/Tests/Unit + src/lib/parquet-viewer/tests/Flow/ParquetViewer/Tests/Integration diff --git a/shell.nix b/shell.nix index 1118ac4b3e..2c61d28ced 100644 --- a/shell.nix +++ b/shell.nix @@ -114,6 +114,7 @@ pkgs.mkShell { pkgs.just pkgs.oxipng pkgs.jpegoptim + pkgs.thrift ] ++ pkgs.lib.optional with-blackfire pkgs.blackfire ++ pkgs.lib.optionals with-wasm [ diff --git a/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroExtractor.php b/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroExtractor.php index 82bd2a81ce..8581487bee 100644 --- a/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroExtractor.php +++ b/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroExtractor.php @@ -8,7 +8,10 @@ use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\Extractor; use Flow\ETL\Extractor\File\FileExtractor; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\File\PathFiltering; +use Flow\ETL\Extractor\File\Scanning; +use Flow\ETL\Extractor\Signal; use Flow\ETL\Extractor\Statistics; use Flow\ETL\FlowContext; use Flow\ETL\Rows; @@ -16,12 +19,12 @@ use Flow\Filesystem\Filesystem; use Flow\Filesystem\Local\NativeLocalFilesystem; use Flow\Filesystem\Path; -use Flow\Filesystem\Path\Filter; -use Flow\Filesystem\Path\Filter\OnlyFiles; use Generator; final class AvroExtractor implements Extractor, FileExtractor { + use Scanning; + private ?Schema $schema = null; use PathFiltering; @@ -35,7 +38,10 @@ public function __construct( ); } - public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator + /** + * @return Generator + */ + public function scan(FlowContext $context, FileScan $scan): Generator { yield Rows::empty(new Schema(), $context->backend()); } diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVExtractor.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVExtractor.php index 608d094f26..c65fff7748 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVExtractor.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVExtractor.php @@ -12,10 +12,12 @@ use Flow\ETL\Extractor\File\FileExtractor; use Flow\ETL\Extractor\File\FileReading; use Flow\ETL\Extractor\File\FileReadLoop; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\File\InferredColumns; use Flow\ETL\Extractor\File\ListedFiles; use Flow\ETL\Extractor\File\MetadataColumnsExtractor; use Flow\ETL\Extractor\File\ReadWindow; +use Flow\ETL\Extractor\File\Scanning; use Flow\ETL\Extractor\InfersSchema; use Flow\ETL\Extractor\RewindableExtractor; use Flow\ETL\Extractor\Signal; @@ -29,8 +31,6 @@ use Flow\Filesystem\Filesystem; use Flow\Filesystem\Local\NativeLocalFilesystem; use Flow\Filesystem\Path; -use Flow\Filesystem\Path\Filter; -use Flow\Filesystem\Path\Filter\OnlyFiles; use Flow\Types\Type\Native\String\StringTypeNarrower; use Generator; @@ -47,6 +47,7 @@ final class CSVExtractor implements { use Batches; use FileReading; + use Scanning; private SchemaInference $inference; @@ -97,13 +98,13 @@ public function isRepeatable(): bool /** * @return Generator */ - public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator + public function scan(FlowContext $context, FileScan $scan): Generator { $fileColumns = $this->fileColumns($this->filesystem, $this->path); $schema = $this->schema(); - $sources = iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $pathFilter), false); + $sources = iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $scan->pathFilter), false); - yield from (new FileReadLoop($fileColumns, $schema))->read( + yield from (new FileReadLoop($fileColumns, $schema, $scan->conjuncts))->read( $sources, new CSVFileBatches( new CSVFileReader($this->filesystem, $this->readOptions, $sources), @@ -117,8 +118,8 @@ public function extract(FlowContext $context, ?int $limit = null, Filter $pathFi : null, ), $this->batchSize(), - $context->backend(), - new ReadWindow(limit: $limit), + $context, + new ReadWindow(limit: $scan->limit), ); } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php index 501557b90d..0eb6bc9d0d 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php @@ -13,6 +13,7 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaDefinitionNotUniqueException; use Flow\ETL\Exception\SchemaMismatchException; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\Signal; use Flow\ETL\Row\Reference; use Flow\ETL\Rows; @@ -1005,7 +1006,7 @@ public function test_limit(): void $extractor = from_csv(path_real(__DIR__ . '/../Fixtures/orders_flow.csv')); $extractor->withBatchSize(1); - self::assertExtractedRowsCount(2, $extractor, flow_context(config()), limit: 2); + self::assertExtractedRowsCount(2, $extractor, flow_context(config()), scan: new FileScan(2)); } /** diff --git a/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalKeySetExtractorTest.php b/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalKeySetExtractorTest.php index 7ae0eed7ba..5c5b0633bb 100644 --- a/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalKeySetExtractorTest.php +++ b/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalKeySetExtractorTest.php @@ -15,6 +15,7 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\Exception\SchemaNotDerivableException; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowTestCase; use Generator; @@ -151,7 +152,7 @@ public function test_pushed_limit_issues_no_query_once_satisfied(): void ); $counter->reset(); - self::assertExtractedRowsCount(1500, $extractor, limit: 1500); + self::assertExtractedRowsCount(1500, $extractor, scan: new FileScan(1500)); // the NULL check, 1000, then a page narrowed to the 500 still wanted - and no query for row 1501 static::assertSame(3, $counter->count); } diff --git a/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalLimitOffsetExtractorTest.php b/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalLimitOffsetExtractorTest.php index 00746005eb..ac31f5a865 100644 --- a/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalLimitOffsetExtractorTest.php +++ b/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalLimitOffsetExtractorTest.php @@ -11,6 +11,7 @@ use Flow\ETL\Adapter\Doctrine\Tests\Double\NativeHandleStub; use Flow\ETL\Cardinality; use Flow\ETL\Exception\SchemaNotDerivableException; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Rows; use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowTestCase; @@ -213,7 +214,7 @@ public function test_pushed_limit_skips_the_count_query(): void ))->withBatchSize(2); $counter->reset(); - self::assertExtractedRowsCount(5, $extractor, limit: 5); + self::assertExtractedRowsCount(5, $extractor, scan: new FileScan(5)); // the three pages of the unlimited read, without the COUNT(*) in front of them static::assertSame(3, $counter->count); static::assertStringNotContainsString('COUNT(*)', implode(' ', $counter->queries)); diff --git a/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalQueryExtractorTest.php b/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalQueryExtractorTest.php index 23f43c874d..6f4d1a1624 100644 --- a/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalQueryExtractorTest.php +++ b/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalQueryExtractorTest.php @@ -12,6 +12,7 @@ use Flow\ETL\Adapter\Doctrine\Tests\Double\NativeHandleStub; use Flow\ETL\Cardinality; use Flow\ETL\Exception\SchemaNotDerivableException; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Rows; use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowTestCase; @@ -131,7 +132,7 @@ public function test_pushed_limit_stops_querying_further_parameter_sets(int $bat $counter->reset(); // at 100 the first set is one batch that overshoots the limit - trimming it is the limit operator's job - self::assertExtractedRowsCount($extractedRows, $extractor, limit: 5); + self::assertExtractedRowsCount($extractedRows, $extractor, scan: new FileScan(5)); static::assertSame(1, $counter->count); } diff --git a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/ExcelExtractor.php b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/ExcelExtractor.php index e079d336de..bae5e9de48 100644 --- a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/ExcelExtractor.php +++ b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/ExcelExtractor.php @@ -12,10 +12,12 @@ use Flow\ETL\Extractor\File\FileExtractor; use Flow\ETL\Extractor\File\FileReading; use Flow\ETL\Extractor\File\FileReadLoop; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\File\InferredColumns; use Flow\ETL\Extractor\File\ListedFiles; use Flow\ETL\Extractor\File\MetadataColumnsExtractor; use Flow\ETL\Extractor\File\ReadWindow; +use Flow\ETL\Extractor\File\Scanning; use Flow\ETL\Extractor\InfersSchema; use Flow\ETL\Extractor\RewindableExtractor; use Flow\ETL\Extractor\Signal; @@ -29,8 +31,6 @@ use Flow\Filesystem\Filesystem; use Flow\Filesystem\Local\NativeLocalFilesystem; use Flow\Filesystem\Path; -use Flow\Filesystem\Path\Filter; -use Flow\Filesystem\Path\Filter\OnlyFiles; use Generator; use Throwable; @@ -48,6 +48,7 @@ final class ExcelExtractor implements { use Batches; use FileReading; + use Scanning; private SchemaInference $inference; @@ -121,7 +122,7 @@ public function isRepeatable(): bool /** * @return Generator */ - public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator + public function scan(FlowContext $context, FileScan $scan): Generator { $fileColumns = $this->fileColumns($this->filesystem, $this->path); $schema = $this->schema(); @@ -141,12 +142,12 @@ public function extract(FlowContext $context, ?int $limit = null, Filter $pathFi $this->sampled = null; try { - yield from (new FileReadLoop($fileColumns, $schema))->read( - iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $pathFilter), false), + yield from (new FileReadLoop($fileColumns, $schema, $scan->conjuncts))->read( + iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $scan->pathFilter), false), $batches, $this->batchSize(), - $context->backend(), - new ReadWindow(limit: $limit), + $context, + new ReadWindow(limit: $scan->limit), ); } finally { $batches->close(); diff --git a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/ExcelExtractorTest.php b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/ExcelExtractorTest.php index b5190be478..57a8c1db70 100644 --- a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/ExcelExtractorTest.php +++ b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/ExcelExtractorTest.php @@ -14,6 +14,7 @@ use Flow\ETL\Exception\InferredSchemaException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaMismatchException; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\Signal; use Flow\ETL\Rows; use Flow\ETL\Tests\Context\ExtractedRows; @@ -759,7 +760,7 @@ public function test_limit_reached_on_the_first_file_tail_batch_skips_the_remain { $extractor = from_excel(ExcelFixtureContext::file('cross_stream/*/*.xlsx'))->withBatchSize(10); - static::assertCount(2, ExtractedRows::of($extractor, limit: 2)); + static::assertCount(2, ExtractedRows::of($extractor, scan: new FileScan(2))); } /** @@ -877,9 +878,9 @@ public function test_a_filtered_extract_infers_over_every_listed_file(): void $filtered = from_excel(ExcelFixtureContext::path('sniff/*')); $whole = from_excel(ExcelFixtureContext::path('sniff/*')); - iterator_to_array($filtered->extract(flow_context(), null, new KeepPaths([ExcelFixtureContext::path( - 'sniff/a', - )])), false); + iterator_to_array($filtered->scan(flow_context(), new FileScan(pathFilter: new KeepPaths([ + ExcelFixtureContext::path('sniff/a'), + ]))), false); $whole->schema(); static::assertEquals($whole->statistics()->rows, $filtered->statistics()->rows); diff --git a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonExtractor.php b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonExtractor.php index 913e63e3c1..ca88a7caac 100644 --- a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonExtractor.php +++ b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonExtractor.php @@ -12,9 +12,11 @@ use Flow\ETL\Extractor\File\FileExtractor; use Flow\ETL\Extractor\File\FileReading; use Flow\ETL\Extractor\File\FileReadLoop; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\File\ListedFiles; use Flow\ETL\Extractor\File\MetadataColumnsExtractor; use Flow\ETL\Extractor\File\ReadWindow; +use Flow\ETL\Extractor\File\Scanning; use Flow\ETL\Extractor\InfersSchema; use Flow\ETL\Extractor\RewindableExtractor; use Flow\ETL\Extractor\Signal; @@ -28,8 +30,6 @@ use Flow\Filesystem\Filesystem; use Flow\Filesystem\Local\NativeLocalFilesystem; use Flow\Filesystem\Path; -use Flow\Filesystem\Path\Filter; -use Flow\Filesystem\Path\Filter\OnlyFiles; use Flow\Types\Type\Logical\InstanceOfTypeNarrower; use Generator; @@ -46,6 +46,7 @@ final class JsonExtractor implements { use Batches; use FileReading; + use Scanning; private SchemaInference $inference; @@ -94,13 +95,13 @@ public function isRepeatable(): bool /** * @return Generator */ - public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator + public function scan(FlowContext $context, FileScan $scan): Generator { $fileColumns = $this->fileColumns($this->filesystem, $this->path); $schema = $this->schema(); - $sources = iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $pathFilter), false); + $sources = iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $scan->pathFilter), false); - yield from (new FileReadLoop($fileColumns, $schema))->read( + yield from (new FileReadLoop($fileColumns, $schema, $scan->conjuncts))->read( $sources, new JsonFileBatches( $this->filesystem, @@ -115,8 +116,8 @@ public function extract(FlowContext $context, ?int $limit = null, Filter $pathFi $this->pointer, ), $this->batchSize(), - $context->backend(), - new ReadWindow(limit: $limit), + $context, + new ReadWindow(limit: $scan->limit), ); } diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonExtractorTest.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonExtractorTest.php index 2475b04597..21e5a22b8c 100644 --- a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonExtractorTest.php +++ b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonExtractorTest.php @@ -12,6 +12,7 @@ use Flow\ETL\Config; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaMismatchException; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\Signal; use Flow\ETL\Schema\Definition\StringDefinition; use Flow\ETL\Tests\Context\ExtractedRows; @@ -197,7 +198,7 @@ public function test_limit(): void $extractor = from_json(path(__DIR__ . '/../../Fixtures/timezones.json')); $extractor->withBatchSize(1); - self::assertExtractedRowsCount(2, $extractor, flow_context(config()), limit: 2); + self::assertExtractedRowsCount(2, $extractor, flow_context(config()), scan: new FileScan(2)); } public function test_schema_appends_the_metadata_column(): void diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonLinesExtractorTest.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonLinesExtractorTest.php index 817c815bae..a7d0723814 100644 --- a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonLinesExtractorTest.php +++ b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonLinesExtractorTest.php @@ -11,6 +11,7 @@ use Flow\ETL\Config; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaMismatchException; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\Signal; use Flow\ETL\Schema\Definition\StringDefinition; use Flow\ETL\Tests\Context\ExtractedRows; @@ -189,7 +190,7 @@ public function test_limit(): void $extractor = from_json_lines(path(__DIR__ . '/../../Fixtures/timezones.jsonl')); $extractor->withBatchSize(1); - self::assertExtractedRowsCount(2, $extractor, flow_context(config()), limit: 2); + self::assertExtractedRowsCount(2, $extractor, flow_context(config()), scan: new FileScan(2)); } public function test_schema_appends_the_metadata_column(): void diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/AdaptiveParquetOpenSource.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/AdaptiveParquetOpenSource.php index 4692df24a8..f2571cdf2a 100644 --- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/AdaptiveParquetOpenSource.php +++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/AdaptiveParquetOpenSource.php @@ -9,6 +9,7 @@ use Flow\ETL\Schema; use Flow\Parquet\Engine\RustParquetFileReader; use Flow\Parquet\ParquetFile; +use Flow\Parquet\Reader\ReadPlan; use Iterator; final readonly class AdaptiveParquetOpenSource implements ParquetOpenSource @@ -25,9 +26,9 @@ public function __construct(ParquetFile $file, ?FlowPhpExtension $extension = nu : new PhpParquetOpenSource($file); } - public function batches(Schema $schema, int $batchSize, ?int $offset, ?int $limit, Backend $backend): Iterator + public function batches(Schema $schema, int $batchSize, ReadPlan $plan, Backend $backend): Iterator { - return $this->source->batches($schema, $batchSize, $offset, $limit, $backend); + return $this->source->batches($schema, $batchSize, $plan, $backend); } public function close(): void diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetExtractor.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetExtractor.php index 1f1f374412..e5706a4437 100644 --- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetExtractor.php +++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetExtractor.php @@ -12,8 +12,10 @@ use Flow\ETL\Extractor\File\FileExtractor; use Flow\ETL\Extractor\File\FileReading; use Flow\ETL\Extractor\File\FileReadLoop; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\File\MetadataColumnsExtractor; use Flow\ETL\Extractor\File\ReadWindow; +use Flow\ETL\Extractor\File\Scanning; use Flow\ETL\Extractor\File\SelfDescribingFile; use Flow\ETL\Extractor\File\SourceFile; use Flow\ETL\Extractor\RewindableExtractor; @@ -47,6 +49,7 @@ final class ParquetExtractor implements { use Batches; use FileReading; + use Scanning; private ByteOrder $byteOrder = ByteOrder::LITTLE_ENDIAN; @@ -111,9 +114,9 @@ public function isRepeatable(): bool /** * @return Generator */ - public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator + public function scan(FlowContext $context, FileScan $scan): Generator { - $sources = iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $pathFilter), false); + $sources = iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $scan->pathFilter), false); // every file is read under the first file's schema, or the union of all of them $target = $this->schema(); $fileColumns = $this->fileColumns($this->filesystem, $this->path); @@ -124,16 +127,18 @@ public function extract(FlowContext $context, ?int $limit = null, Filter $pathFi $this->unionByName ? null : new DerivedSchema($this->derivedSchema($this->schemaFiles(), $this->unionByName), $this->derivedFrom), + $fileColumns->withoutTail($target), + $scan->conjuncts, ); $this->firstFile = null; try { - yield from (new FileReadLoop($fileColumns, $target))->read( + yield from (new FileReadLoop($fileColumns, $target, $scan->conjuncts))->read( $sources, $batches, $this->batchSize(), - $context->backend(), - new ReadWindow($this->offset ?? 0, $limit), + $context, + new ReadWindow($this->offset ?? 0, $scan->limit), ); } finally { $batches->close(); diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetFileBatches.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetFileBatches.php index 204b82b3c4..54e707c23f 100644 --- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetFileBatches.php +++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetFileBatches.php @@ -4,6 +4,8 @@ namespace Flow\ETL\Adapter\Parquet; +use Flow\ETL\Adapter\Parquet\Pushdown\FilterTranslator; +use Flow\ETL\Adapter\Parquet\Pushdown\TranslatedConjunct; use Flow\ETL\Column\Backend; use Flow\ETL\Exception\InferredSchemaException; use Flow\ETL\Extractor\File\DerivedSchema; @@ -11,26 +13,41 @@ use Flow\ETL\Extractor\File\OffsetSkippingFileBatches; use Flow\ETL\Extractor\File\ReadWindow; use Flow\ETL\Extractor\File\SourceFile; +use Flow\ETL\Function\ScalarFunction; use Flow\ETL\Rows; use Flow\ETL\Schema; +use Flow\Parquet\Filter\All; +use Flow\Parquet\Filter\Binding\BoundFilter; +use Flow\Parquet\Reader\ReadPlanner; use Generator; final class ParquetFileBatches implements OffsetSkippingFileBatches { + /** + * @var list + */ + private readonly array $conjuncts; + /** * @param null|ParquetSourceFile $firstFile the file the schema pass kept open, read instead of opened again * @param null|DerivedSchema $derived the schema every file must match; null under unionByName + * @param list $conjuncts FileScan::$conjuncts */ public function __construct( private readonly ParquetSourceFileOpener $opener, private readonly FileColumns $fileColumns, private ?ParquetSourceFile $firstFile, private readonly ?DerivedSchema $derived, - ) {} + Schema $body, + array $conjuncts = [], + ) { + $this->conjuncts = (new FilterTranslator())->translate($conjuncts, $body); + } /** * Skips the whole file when the window's offset covers it; otherwise reads from the offset. A file of a narrower - * schema is matched to $body. + * schema is matched to $body. What the conjuncts a file answers exactly prove cannot match is skipped; FileReadLoop + * drops the rest that does not match them. * * @param int<1, max> $batchSize * @@ -57,7 +74,8 @@ public function batches( $opened = null; try { - $fileRows = $file->file->reader()->rowsNumber(); + $footer = $file->file->reader()->footer(); + $fileRows = $footer->rowsNumber(); if ($window->offset >= $fileRows) { return $fileRows; @@ -66,9 +84,31 @@ public function batches( $this->derived?->refuseDivergence($file->source()->uri(), $file->schema()); $fileBody = $this->fileColumns->withoutTail($file->schema()); + $kept = []; + + foreach ($this->conjuncts as $conjunct) { + if ($conjunct->readsAs($fileBody, $body)) { + $kept[] = $conjunct->filter; + } + } + $opened = $file->open(); - foreach ($opened->batches($fileBody, $batchSize, $window->offset, $window->limit, $backend) as $rows) { + // rows matching the conjuncts the file binds; FileReadLoop drops those the rest do not match + foreach ($opened->batches( + $fileBody, + $batchSize, + (new ReadPlanner())->plan( + $footer, + $file->file->indexes(), + $kept === [] + ? null + : BoundFilter::bind(new All(...$kept), $footer->schema(), $file->file->options()), + $window->offset, + $window->limit, + ), + $backend, + ) as $rows) { yield $fileBody->isSame($body) ? $rows : $rows->matchTo($body, $backend); } diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetOpenSource.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetOpenSource.php index c1cfc09013..bdf3e4314b 100644 --- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetOpenSource.php +++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetOpenSource.php @@ -7,6 +7,7 @@ use Flow\ETL\Column\Backend; use Flow\ETL\Rows; use Flow\ETL\Schema; +use Flow\Parquet\Reader\ReadPlan; use Iterator; use function interface_exists; @@ -20,11 +21,11 @@ interface ParquetOpenSource /** * Batches of at most $batchSize rows under $schema, every column adopted by $backend. * - * @param int<1, max> $batchSize + * @param int $batchSize refused below 1 * * @return Iterator */ - public function batches(Schema $schema, int $batchSize, ?int $offset, ?int $limit, Backend $backend): Iterator; + public function batches(Schema $schema, int $batchSize, ReadPlan $plan, Backend $backend): Iterator; public function close(): void; } diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetSourceFile.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetSourceFile.php index 6c9095029b..6ee7f9a20c 100644 --- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetSourceFile.php +++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetSourceFile.php @@ -74,8 +74,8 @@ public function source(): SourceFile public function statistics(): Statistics { return new Statistics( - rows: Cardinality::exact($this->file->reader()->rowsNumber()), - size: Cardinality::exact($this->file->reader()->totalByteSize()), + rows: Cardinality::exact($this->file->reader()->footer()->rowsNumber()), + size: Cardinality::exact($this->file->reader()->footer()->totalByteSize()), ); } } diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/PhpParquetOpenSource.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/PhpParquetOpenSource.php index 68eb9383da..dcda6eb973 100644 --- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/PhpParquetOpenSource.php +++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/PhpParquetOpenSource.php @@ -10,6 +10,7 @@ use Flow\ETL\Schema; use Flow\ETL\Schema\Definition; use Flow\Parquet\ParquetFile; +use Flow\Parquet\Reader\ReadPlan; use Generator; use function array_map; @@ -24,13 +25,13 @@ public function __construct( /** * @return Generator */ - public function batches(Schema $schema, int $batchSize, ?int $offset, ?int $limit, Backend $backend): Generator + public function batches(Schema $schema, int $batchSize, ReadPlan $plan, Backend $backend): Generator { $names = array_values(array_map(static fn(Definition $definition): string => $definition ->entry() ->name(), $schema->definitions())); - foreach ($this->file->columns($batchSize, $names, $limit, $offset) as $chunk) { + foreach ($this->file->reader()->readColumns($names, $batchSize, $plan) as $chunk) { $builder = new RowsBuilder($schema, $backend); foreach ($names as $name) { diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/Pushdown/FilterTranslator.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/Pushdown/FilterTranslator.php new file mode 100644 index 0000000000..d498cb6140 --- /dev/null +++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/Pushdown/FilterTranslator.php @@ -0,0 +1,355 @@ + $conjuncts + * + * @return list one per translatable conjunct + */ + public function translate(array $conjuncts, Schema $body): array + { + $translated = []; + + foreach ($conjuncts as $conjunct) { + $parquet = $this->filter($conjunct, $body); + + if ($parquet !== null) { + $translated[] = new TranslatedConjunct( + $parquet, + array_values((new ReferencedColumns())->in($conjunct)->names()), + ); + } + } + + return $translated; + } + + private function filter(ScalarFunction $function, Schema $body): ?Filter + { + $children = $function->children(); + + return match (true) { + $function instanceof All, $function instanceof Any => $this->combined($function, $body), + $function instanceof Not => $children[0] instanceof ScalarFunction + ? $this->filter($children[0], $body)?->negate() + : null, + $function instanceof Equals => $this->comparison($children[0], $children[1], Operator::Equal, $body, false), + $function instanceof NotEquals => $this->comparison( + $children[0], + $children[1], + Operator::Equal, + $body, + false, + )?->negate(), + $function instanceof Same => $this->comparison($children[0], $children[1], Operator::Equal, $body, true), + $function instanceof NotSame => $this->comparison( + $children[0], + $children[1], + Operator::Equal, + $body, + true, + )?->negate(), + $function instanceof LessThan => $this->comparison( + $children[0], + $children[1], + Operator::LessThan, + $body, + false, + ), + $function instanceof LessThanEqual => $this->comparison( + $children[0], + $children[1], + Operator::LessThanOrEqual, + $body, + false, + ), + $function instanceof GreaterThan => $this->comparison( + $children[0], + $children[1], + Operator::GreaterThan, + $body, + false, + ), + $function instanceof GreaterThanEqual => $this->comparison( + $children[0], + $children[1], + Operator::GreaterThanOrEqual, + $body, + false, + ), + $function instanceof IsNull => $this->isNull($children[0], $body), + $function instanceof IsNotNull => $this->isNull($children[0], $body)?->negate(), + $function instanceof IsIn => $this->in($children[0], $children[1], $body), + $function instanceof StartsWith => $this->startsWith($children[0], $children[1], $body), + $function instanceof Between => $this->between($function, $body), + default => null, + }; + } + + private function combined(All|Any $function, Schema $body): ?Filter + { + $filters = []; + + foreach ($function->children() as $child) { + $filter = $this->filter($child, $body); + + if ($filter === null) { + return null; + } + + $filters[] = $filter; + } + + return $function instanceof All ? new ParquetAll(...$filters) : new ParquetAny(...$filters); + } + + private function comparison( + FunctionTree $left, + FunctionTree $right, + Operator $operator, + Schema $body, + bool $strict, + ): ?Comparison { + if ($left instanceof Literal && $right instanceof UnresolvedReference) { + return $this->comparison( + $right, + $left, + match ($operator) { + Operator::LessThan => Operator::GreaterThan, + Operator::LessThanOrEqual => Operator::GreaterThanOrEqual, + Operator::GreaterThan => Operator::LessThan, + Operator::GreaterThanOrEqual => Operator::LessThanOrEqual, + Operator::Equal => Operator::Equal, + }, + $body, + $strict, + ); + } + + $definition = $this->definition($left, $body); + + if ($definition === null || !$right instanceof Literal) { + return null; + } + + $value = $this->literal( + type_bare($definition->type()), + $right->returns(), + $right->value(), + $operator !== Operator::Equal, + $strict, + ); + + return $value === null ? null : new Comparison(parquet_column($definition->entry()->name()), $operator, $value); + } + + private function isNull(FunctionTree $reference, Schema $body): ?Filter + { + $definition = $this->definition($reference, $body); + + return $definition === null ? null : parquet_column($definition->entry()->name())->isNull(); + } + + private function in(FunctionTree $haystack, FunctionTree $needle, Schema $body): ?Filter + { + $definition = $this->definition($needle, $body); + // @mago-expect analysis:mixed-assignment + $list = $haystack instanceof Literal ? $haystack->value() : null; + + if ($definition === null || !is_array($list)) { + return null; + } + + // NULL never matches: x IN (1, NULL) keeps what x IN (1) keeps, and NOT (x IN (1, NULL)) is never TRUE + $present = array_values(array_filter($list, static fn(mixed $element): bool => $element !== null)); + $detector = new TypeDetector(); + $values = array_map(fn(mixed $element): mixed => $this->literal( + type_bare($definition->type()), + $detector->detectType($element), + $element, + false, + false, + ), $present); + + return in_array(null, $values, true) ? null : parquet_column($definition->entry()->name())->in($values); + } + + private function startsWith(FunctionTree $haystack, FunctionTree $needle, Schema $body): ?Filter + { + $definition = $this->definition($haystack, $body); + // @mago-expect analysis:mixed-assignment + $prefix = $needle instanceof Literal ? $needle->value() : null; + + return $definition !== null && type_bare($definition->type()) instanceof StringType && is_string($prefix) + ? parquet_column($definition->entry()->name())->startsWith($prefix) + : null; + } + + private function between(Between $between, Schema $body): ?Filter + { + [$value, $lower, $upper, $boundary] = $between->children(); + + // @mago-expect analysis:mixed-assignment + $kind = $boundary instanceof Literal ? $boundary->value() : null; + + if (!$kind instanceof Boundary) { + return null; + } + + $lowerBound = $this->comparison( + $value, + $lower, + $kind === Boundary::INCLUSIVE || $kind === Boundary::LEFT_INCLUSIVE + ? Operator::GreaterThanOrEqual + : Operator::GreaterThan, + $body, + false, + ); + $upperBound = $this->comparison( + $value, + $upper, + $kind === Boundary::INCLUSIVE || $kind === Boundary::RIGHT_INCLUSIVE + ? Operator::LessThanOrEqual + : Operator::LessThan, + $body, + false, + ); + + return $lowerBound === null || $upperBound === null ? null : $lowerBound->and($upperBound); + } + + /** + * @return null|Definition the body column $function is a reference to, null when it is none + */ + private function definition(FunctionTree $function, Schema $body): ?Definition + { + return $function instanceof UnresolvedReference ? $body->findDefinition($function->to()) : null; + } + + /** + * @param Type $column + * @param Type $literal + * + * @return null|bool|DateTimeInterface|float|int|string the literal as the parquet filter takes it, null when + * the pair cannot be pushed + */ + private function literal(Type $column, Type $literal, mixed $value, bool $ordering, bool $strict): mixed + { + $literal = type_bare($literal); + + if (is_float($value) && is_nan($value)) { + return null; + } + + return match (true) { + $column instanceof JsonType => null, + $column instanceof NumericStringType => !$strict + && $literal instanceof StringType + && is_string($value) + && ctype_digit($value) + && ($value === '0' || $value[0] !== '0') + ? $value + : null, + !$strict && $column instanceof FloatType && $literal instanceof IntegerType => is_int($value) + && abs($value) <= LiteralDomain::EXACT_FLOAT_INTEGER + ? (float) $value + : null, + !$strict && $column instanceof IntegerType && $literal instanceof FloatType => is_float($value) + && floor($value) === $value + && abs($value) < LiteralDomain::EXACT_FLOAT_INTEGER + ? (int) $value + : null, + !$strict + && ( + $column instanceof DateTimeType + && $literal instanceof DateType + || $column instanceof DateType + && $literal instanceof DateTimeType + ) + => $value instanceof DateTimeInterface ? $value : null, + $column::class !== $literal::class, + !( + $ordering + ? (new ComparableValues())->orderedByPhysical($column) + : (new ComparableValues())->equalByPhysical($column) + ), + => null, + $value instanceof Uuid => $value->toString(), + is_bool($value), + is_int($value), + is_float($value), + is_string($value), + $value instanceof DateTimeInterface, + => $value, + default => null, + }; + } +} diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/Pushdown/TranslatedConjunct.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/Pushdown/TranslatedConjunct.php new file mode 100644 index 0000000000..025329e36c --- /dev/null +++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/Pushdown/TranslatedConjunct.php @@ -0,0 +1,44 @@ + $columns Flow column names the predicate reads + */ + public function __construct( + public Filter $filter, + public array $columns, + ) {} + + /** + * True when the file stores every column the predicate reads as $body types it, or not at all. A column the file + * stores as another type is cast to the body's, so its statistics are of the other type. + */ + public function readsAs(Schema $fileBody, Schema $body): bool + { + $comparator = new Comparator(); + + foreach ($this->columns as $column) { + $stored = $fileBody->findDefinition($column); + + if ( + $stored !== null + && !$comparator->equals(type_bare($stored->type()), type_bare($body->get($column)->type())) + ) { + return false; + } + } + + return true; + } +} diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/SchemaConverter.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/SchemaConverter.php index 8a31da9663..92e4b4903c 100644 --- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/SchemaConverter.php +++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/SchemaConverter.php @@ -11,6 +11,7 @@ use Flow\Parquet\ParquetFile\Schema\Column; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\ListElement; +use Flow\Parquet\ParquetFile\Schema\LogicalType\Integer; use Flow\Parquet\ParquetFile\Schema\NestedColumn; use Flow\Types\Type; use Flow\Types\Type\Logical\DateTimeType; @@ -20,6 +21,7 @@ use Flow\Types\Type\Logical\JsonType; use Flow\Types\Type\Logical\ListType; use Flow\Types\Type\Logical\MapType; +use Flow\Types\Type\Logical\NumericStringType; use Flow\Types\Type\Logical\OptionalType; use Flow\Types\Type\Logical\StructureElement; use Flow\Types\Type\Logical\StructureType; @@ -38,6 +40,7 @@ use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\date_schema; use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\definition_from_type; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\json_schema; @@ -56,6 +59,7 @@ use function Flow\Types\DSL\type_json; use function Flow\Types\DSL\type_list; use function Flow\Types\DSL\type_map; +use function Flow\Types\DSL\type_numeric_string; use function Flow\Types\DSL\type_optional; use function Flow\Types\DSL\type_string; use function Flow\Types\DSL\type_structure; @@ -118,6 +122,7 @@ private function flowToParquet(string $name, Type $type, bool $nullable): Column XMLElementType::class, XMLType::class, StringType::class, + NumericStringType::class, NullType::class, => FlatColumn::string($name, $repetition), BooleanType::class => FlatColumn::boolean($name, $repetition), @@ -171,6 +176,10 @@ private function parquetToFlowDefinition(Column $column): Definition $nullable = $column->repetition() === ParquetSchema\Repetition::OPTIONAL; + if (Integer::forColumn($column)?->is(64, false) === true) { + return definition_from_type($column->name(), type_numeric_string(), $nullable); + } + if ($logicalType === null) { return match ($column->type()) { ParquetSchema\PhysicalType::INT32 => match ($column->convertedType()) { @@ -247,6 +256,10 @@ public function parquetToFlowType(Column $column): Type $nullable = $column->repetition() === ParquetSchema\Repetition::OPTIONAL; + if (Integer::forColumn($column)?->is(64, false) === true) { + return $nullable ? type_optional(type_numeric_string()) : type_numeric_string(); + } + if ($logicalType === null) { $type = match ($column->type()) { ParquetSchema\PhysicalType::INT32 => match ($column->convertedType()) { diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Context/ParquetFilesContext.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Context/ParquetFilesContext.php index fc72becdf0..8ad13b9420 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Context/ParquetFilesContext.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Context/ParquetFilesContext.php @@ -5,6 +5,7 @@ namespace Flow\ETL\Adapter\Parquet\Tests\Context; use Flow\ETL\Adapter\Parquet\ParquetOpenSink; +use Flow\ETL\Adapter\Parquet\ParquetSourceFileOpener; use Flow\ETL\Adapter\Parquet\RustParquetOpenSink; use Flow\ETL\Adapter\Parquet\SchemaConverter; use Flow\ETL\Rows; @@ -20,14 +21,20 @@ use Flow\Parquet\Options; use Flow\Parquet\ParquetFile\Compressions; use Flow\Parquet\ParquetFile\Encodings; +use Flow\Parquet\ParquetFile\RowGroup\ColumnChunk; use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\Column; use Flow\Parquet\ParquetFile\Schema\FlatColumn; +use Flow\Parquet\ParquetFile\Schema\LogicalType; +use Flow\Parquet\ParquetFile\Schema\PhysicalType; use Flow\Parquet\Reader; +use Flow\Parquet\Reader\ReadPlan; +use Flow\Parquet\Reader\ReadPlanner; use Flow\Parquet\Writer; use function array_count_values; use function array_map; +use function array_sum; use function Flow\ETL\Adapter\Parquet\to_parquet; use function Flow\ETL\DSL\data_frame; use function Flow\ETL\DSL\from_rows; @@ -37,6 +44,13 @@ final class ParquetFilesContext { + public static function plan(Filesystem $filesystem, string $uri, int $offset = 0, ?int $limit = null): ReadPlan + { + $file = (new Reader())->readStream($filesystem->readFrom(path($uri))); + + return (new ReadPlanner())->plan($file->reader()->footer(), $file->indexes(), null, $offset, $limit); + } + /** * @return array */ @@ -149,4 +163,116 @@ public static function opensPerFile(CountingFilesystem $filesystem): array ->path() ->path(), $filesystem->openedStreams)); } + + /** + * Bytes read through every stream the filesystem opened. + */ + public static function bytesRead(CountingFilesystem $filesystem): int + { + return array_sum(array_map( + static fn(ClosingSpySourceStream $stream): int => $stream->readBytes, + $filesystem->openedStreams, + )); + } + + /** + * The chunks of the given columns in the given row groups, by file uri. + * + * @param array>> $groups + * + * @return list + */ + public static function chunks(Filesystem $filesystem, array $groups): array + { + $chunks = []; + + foreach ($groups as $uri => $columns) { + $file = Reader::php()->readStream($filesystem->readFrom(path($uri))); + $rowGroups = $file->metadata()->rowGroups()->all(); + + foreach ($columns as $column => $indexes) { + foreach ($indexes as $index) { + $chunks[] = $rowGroups[$index]->getColumnChunk($file->schema()->getFlat($column)); + } + } + } + + return $chunks; + } + + /** + * The bytes of the offset indexes of the given columns in the given row groups, by file uri. + * + * @param array>> $groups + */ + public static function offsetIndexBytes(Filesystem $filesystem, array $groups): int + { + return array_sum(array_map( + static fn(ColumnChunk $chunk): int => (int) $chunk->offsetIndex()?->length, + self::chunks($filesystem, $groups), + )); + } + + /** + * The bytes of the column and offset indexes of the given columns in the given row groups, by file uri. + * + * @param array>> $groups + */ + public static function pageIndexBytes(Filesystem $filesystem, array $groups): int + { + return array_sum(array_map( + static fn(ColumnChunk $chunk): int => ( + (int) $chunk->columnIndex()?->length + (int) $chunk->offsetIndex()?->length + ), + self::chunks($filesystem, $groups), + )); + } + + /** + * A PHP-engine opener of every column under the default options. + */ + public static function opener(Filesystem $filesystem): ParquetSourceFileOpener + { + return new ParquetSourceFileOpener( + $filesystem, + new PhpParquetEngine(), + Options::default(), + new SchemaConverter(), + [], + ); + } + + /** + * @param iterable $batches + */ + public static function rowCount(iterable $batches): int + { + $count = 0; + + foreach ($batches as $rows) { + $count += $rows->count(); + } + + return $count; + } + + /** + * A `UINT_64` column `big`, one row group per value. + * + * @param list $values + */ + public static function unsignedInt64(Filesystem $filesystem, string $uri, array $values): void + { + $options = Options::default()->set(Option::ROW_GROUP_SIZE_BYTES, 1)->set( + Option::ROW_GROUP_SIZE_CHECK_INTERVAL, + 1, + ); + $writer = new Writer(options: $options, engine: new PhpParquetEngine(options: $options)); + $writer->openForStream( + $filesystem->writeTo(path($uri)), + Schema::with(new FlatColumn('big', PhysicalType::INT64, logicalType: LogicalType::integer(64, false))), + ); + $writer->writeBatch(array_map(static fn(int|string $value): array => ['big' => $value], $values)); + $writer->close(); + } } diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Context/PushdownParityContext.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Context/PushdownParityContext.php new file mode 100644 index 0000000000..89c3f9cd1d --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Context/PushdownParityContext.php @@ -0,0 +1,109 @@ + [ + 'id' => [0, 1, 2], + 'name' => [0, 1, 2], + 'score' => [0, 1, 2], + 'd92' => [0, 1, 2], + 'd182' => [0, 1, 2], + 'day' => [0, 1, 2], + 'ts_us' => [0, 1, 2], + 'ts_ns' => [0, 1, 2], + 'uuid' => [0, 1, 2], + 'extra' => [0, 1, 2], + ], + 'memory://parity/b.parquet' => [ + 'id' => [0], + 'name' => [0], + 'score' => [0], + 'd92' => [0], + 'd182' => [0], + 'day' => [0], + 'ts_us' => [0], + 'ts_ns' => [0], + 'uuid' => [0], + ], + ]; + + public const string GLOB = 'memory://parity/*.parquet'; + + public static function write(Filesystem $filesystem): void + { + $options = Options::default()->set(Option::ROW_GROUP_SIZE_BYTES, 1)->set( + Option::ROW_GROUP_SIZE_CHECK_INTERVAL, + 10, + ); + $columns = [ + FlatColumn::int64('id'), + FlatColumn::string('name'), + FlatColumn::double('score'), + FlatColumn::decimal('d92', 9, 2), + FlatColumn::decimal('d182', 18, 2), + FlatColumn::date('day'), + FlatColumn::dateTime('ts_us'), + new FlatColumn( + 'ts_ns', + PhysicalType::INT64, + logicalType: new LogicalType( + LogicalType::TIMESTAMP, + timestamp: new Timestamp(true, TimeUnit::NANOSECONDS), + ), + ), + FlatColumn::uuid('uuid'), + ]; + $row = static fn(int $id): array => [ + 'id' => $id, + 'name' => (string) $id, + 'score' => $id < 10 ? NAN : $id * 1.5, + 'd92' => $id + 0.25, + 'd182' => $id + 0.25, + 'day' => (new DateTimeImmutable('2024-01-01'))->modify(sprintf('+%d days', $id)), + 'ts_us' => (new DateTimeImmutable('2024-01-01 00:00:00.123456'))->modify(sprintf('+%d hours', $id)), + 'ts_ns' => (new DateTimeImmutable('2024-01-01 00:00:00.123456'))->modify(sprintf('+%d hours', $id)), + 'uuid' => sprintf('00000000-0000-4000-8000-%012d', $id), + ]; + + foreach ([ + 'a' => [ + [...$columns, FlatColumn::string('extra')], + static fn(int $id): array => [...$row($id), 'extra' => 'x'], + ], + 'b' => [$columns, $row], + ] as $file => [$schema, $values]) { + $writer = new Writer(options: $options, engine: new PhpParquetEngine(options: $options)); + $writer->openForStream( + $filesystem->writeTo(path(sprintf('memory://parity/%s.parquet', $file))), + Schema::with(...$schema), + ); + $writer->writeBatch(array_map($values, $file === 'a' ? range(0, 29) : range(30, 39))); + $writer->close(); + } + } +} diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorFooterTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorFooterTest.php index 8a52068fe0..813011eb62 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorFooterTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorFooterTest.php @@ -6,6 +6,7 @@ use DateTimeImmutable; use Flow\ETL\Adapter\Parquet\Tests\Context\ParquetFilesContext; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Function\Between\Boundary; use Flow\ETL\Rows; use Flow\ETL\Tests\Double\CountingFilesystem; @@ -113,8 +114,11 @@ public function test_a_path_filter_excluding_the_first_file_closes_the_kept_one( $extractor = from_parquet(path(self::PAGINATION . '/*.parquet'), filesystem: $filesystem); $extractor->schema(); - $rows = iterator_to_array($extractor->extract(flow_context(), pathFilter: new KeepPaths([path(self::PAGINATION - . '/02_500.parquet')])), false); + $rows = iterator_to_array( + $extractor->scan(flow_context(), new FileScan(pathFilter: new KeepPaths([path(self::PAGINATION + . '/02_500.parquet')]))), + false, + ); static::assertSame(500, array_sum(array_map(static fn(Rows $batch): int => $batch->count(), $rows))); static::assertSame( diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorTest.php index b3735eaeba..d94f898d99 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorTest.php @@ -9,6 +9,7 @@ use Flow\ETL\Adapter\Parquet\Tests\Context\ParquetFilesContext; use Flow\ETL\Cardinality; use Flow\ETL\Exception\InferredSchemaException; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\Signal; use Flow\ETL\Plan\Stage; use Flow\ETL\Schema\Definition; @@ -21,6 +22,7 @@ use Flow\Parquet\Binary\ByteOrder; use Flow\Parquet\Engine\PhpParquetEngine; use Flow\Parquet\Options; +use Flow\Parquet\ParquetEngine; use Flow\Parquet\Reader; use Flow\Types\Value\Json; use Flow\Types\Value\Uuid; @@ -43,9 +45,11 @@ use function Flow\ETL\DSL\from_sequence_number; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\list_schema; +use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\map_schema; use function Flow\ETL\DSL\partition_by; use function Flow\ETL\DSL\partition_types; +use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\structure_schema; @@ -64,6 +68,37 @@ final class ParquetExtractorTest extends FlowTestCase { + /** + * @return Generator + */ + public static function engines(): Generator + { + yield 'default engine' => [null]; + yield 'php engine' => [new PhpParquetEngine()]; + } + + #[DataProvider('engines')] + public function test_a_body_predicate_skips_row_groups(?ParquetEngine $engine): void + { + $memory = memory_filesystem(); + ParquetFilesContext::rowGroups($memory, 'memory://groups.parquet', 3000, 1000); + $filtered = new CountingFilesystem($memory); + $unfiltered = new CountingFilesystem($memory); + $predicate = ref('id')->greaterThanEqual(lit(2500)); + + $rows = df() + ->read(from_parquet(path('memory://groups.parquet'), filesystem: $filtered, engine: $engine)) + ->filter($predicate) + ->fetch(); + $all = df() + ->read(from_parquet(path('memory://groups.parquet'), filesystem: $unfiltered, engine: $engine)) + ->fetch(); + + static::assertCount(500, $rows); + static::assertSame(df()->read(from_rows($all))->filter($predicate)->fetch()->toArray(), $rows->toArray()); + static::assertLessThan(ParquetFilesContext::bytesRead($unfiltered), ParquetFilesContext::bytesRead($filtered)); + } + public function test_limit(): void { $extractor = from_parquet(path(__DIR__ . '/Fixtures/orders_1k.parquet')); @@ -447,7 +482,10 @@ public function test_a_path_filter_narrows_the_read_but_not_the_schema(): void ); $schema = $extractor->schema(); - $batches = iterator_to_array($extractor->extract(flow_context(config()), pathFilter: new RejectingFilter())); + $batches = iterator_to_array($extractor->scan( + flow_context(config()), + new FileScan(pathFilter: new RejectingFilter()), + )); static::assertSame([], $batches); static::assertSame(1, $filesystem->readFromCalls); @@ -482,7 +520,7 @@ public function test_limit_reached_on_the_first_file_tail_batch_skips_the_remain { $extractor = from_parquet(path(__DIR__ . '/Fixtures/Pagination/*.parquet'))->withBatchSize(1500); - static::assertCount(1000, ExtractedRows::of($extractor, limit: 1000)); + static::assertCount(1000, ExtractedRows::of($extractor, scan: new FileScan(1000))); } public function test_the_second_file_is_asked_for_the_remainder_of_the_limit(): void @@ -501,7 +539,7 @@ public function test_limit_reached_on_a_full_batch_of_the_first_file_skips_the_r { $extractor = from_parquet(path(__DIR__ . '/Fixtures/Pagination/*.parquet'))->withBatchSize(500); - static::assertCount(1000, ExtractedRows::of($extractor, limit: 1000)); + static::assertCount(1000, ExtractedRows::of($extractor, scan: new FileScan(1000))); } public function test_is_repeatable(): void diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/PushdownParityTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/PushdownParityTest.php new file mode 100644 index 0000000000..f3d8a46374 --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/PushdownParityTest.php @@ -0,0 +1,160 @@ + + */ + public static function predicates(): Generator + { + $predicates = [ + 'string range' => [ref('name')->greaterThan(lit('3')), true], + 'NaN column' => [ref('score')->lessThanEqual(lit(20.0)), true], + 'decimal(9,2)' => [ref('d92')->greaterThan(lit(15.25)), true], + 'decimal(18,2)' => [ref('d182')->greaterThan(lit(15.25)), false], + 'date' => [ref('day')->greaterThanEqual(lit(new DateTimeImmutable('2024-01-21'))), true], + 'timestamp micros' => [ref('ts_us')->greaterThan(lit(new DateTimeImmutable('2024-01-01 20:30:00'))), true], + 'timestamp nanos' => [ref('ts_ns')->greaterThan(lit(new DateTimeImmutable('2024-01-01 20:30:00'))), true], + 'uuid' => [ref('uuid')->equals(lit(new Uuid('00000000-0000-4000-8000-000000000025'))), true], + 'a negated is in holding null, or an equality' => [ + not(ref('id')->isIn(lit([1, null])))->or(ref('id')->equals(lit(5))), + false, + ], + 'absent column equals' => [ref('extra')->equals(lit('x')), true], + 'absent column is null' => [ref('extra')->isNull(), true], + ]; + + foreach (['default engine' => null, 'php engine' => new PhpParquetEngine()] as $engineName => $engine) { + foreach ($predicates as $name => [$predicate, $prunes]) { + yield $engineName . ', ' . $name => [$engine, $predicate, $prunes]; + } + } + } + + /** + * @param bool $prunes true when the predicate skips a row group, false when it skips none + */ + #[DataProvider('predicates')] + public function test_a_pushed_predicate_returns_what_the_unpushed_one_returns( + ?ParquetEngine $engine, + ScalarFunction $predicate, + bool $prunes, + ): void { + $memory = memory_filesystem(); + PushdownParityContext::write($memory); + $pushedFilesystem = new CountingFilesystem($memory); + $plainFilesystem = new CountingFilesystem($memory); + + $pushed = df() + ->read( + from_parquet( + path(PushdownParityContext::GLOB), + engine: $engine, + filesystem: $pushedFilesystem, + )->unionByName(), + ) + ->filter($predicate) + ->fetch(); + $plain = df(config_builder()->optimizer(Optimizer::default()->without(PushFilterIntoSource::class))) + ->read( + from_parquet( + path(PushdownParityContext::GLOB), + engine: $engine, + filesystem: $plainFilesystem, + )->unionByName(), + ) + ->filter($predicate) + ->fetch(); + + static::assertNotSame([], $plain->reduceToArray('id')); + static::assertSame($plain->reduceToArray('id'), $pushed->reduceToArray('id')); + // a pushed read adds at most the page indexes it consults to the chunks it reads + static::assertLessThanOrEqual( + ParquetFilesContext::bytesRead($plainFilesystem) + + ParquetFilesContext::pageIndexBytes($memory, PushdownParityContext::CHUNKS) + + ParquetFilesContext::offsetIndexBytes($memory, PushdownParityContext::CHUNKS), + ParquetFilesContext::bytesRead($pushedFilesystem), + ); + + if ($prunes) { + static::assertLessThan( + ParquetFilesContext::bytesRead($plainFilesystem), + ParquetFilesContext::bytesRead($pushedFilesystem), + ); + } + } + + /** + * @return Generator + */ + public static function engines(): Generator + { + yield 'default engine' => [null]; + yield 'php engine' => [new PhpParquetEngine()]; + } + + #[DataProvider('engines')] + public function test_a_limit_beside_a_pushed_filter_reads_fewer_bytes_than_the_unlimited_read(?ParquetEngine $engine): void + { + $memory = memory_filesystem(); + PushdownParityContext::write($memory); + $limitedFilesystem = new CountingFilesystem($memory); + $unlimitedFilesystem = new CountingFilesystem($memory); + + $limited = df() + ->read( + from_parquet( + path(PushdownParityContext::GLOB), + engine: $engine, + filesystem: $limitedFilesystem, + )->unionByName(), + ) + ->filter(ref('id')->lessThan(lit(35))) + ->limit(10) + ->fetch(); + $unlimited = df() + ->read( + from_parquet( + path(PushdownParityContext::GLOB), + engine: $engine, + filesystem: $unlimitedFilesystem, + )->unionByName(), + ) + ->filter(ref('id')->lessThan(lit(35))) + ->fetch(); + + static::assertCount(10, $limited); + static::assertCount(35, $unlimited); + static::assertLessThan( + ParquetFilesContext::bytesRead($unlimitedFilesystem), + ParquetFilesContext::bytesRead($limitedFilesystem), + ); + } +} diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/UnsignedInt64ColumnTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/UnsignedInt64ColumnTest.php new file mode 100644 index 0000000000..f0339e2ccb --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/UnsignedInt64ColumnTest.php @@ -0,0 +1,44 @@ +openForStream( + $memory->writeTo($path), + Schema::with(new FlatColumn('id', PhysicalType::INT64, logicalType: LogicalType::integer(64, false))), + ); + $writer->writeBatch([['id' => 0], ['id' => '9223372036854775808'], ['id' => '18446744073709551615']]); + $writer->close(); + + $df = data_frame()->read(from_parquet($path, filesystem: $memory)); + $rows = $df->fetch(); + + static::assertTrue(type_equals(type_numeric_string(), $df->schema()->get('id')->type())); + static::assertTrue(type_equals(type_numeric_string(), $rows->schema()->get('id')->type())); + static::assertTrue(type_equals(type_numeric_string(), $rows->column('id')->type())); + static::assertSame(['0', '9223372036854775808', '18446744073709551615'], $rows->column('id')->values()); + } +} diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Mother/BodySchemaMother.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Mother/BodySchemaMother.php new file mode 100644 index 0000000000..d452587c64 --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Mother/BodySchemaMother.php @@ -0,0 +1,40 @@ +batches( schema(int_schema('id')), 10, - null, - null, + (new ReadPlanner())->plan($file->reader()->footer(), $file->indexes(), null), new PhpBackend(), )); $source->close(); @@ -48,8 +48,7 @@ public function test_with_flow_php_a_rust_engine_file_reads_through_rust(): void static::assertInstanceOf(RustIterator::class, $source->batches( schema(int_schema('id')), 10, - null, - null, + (new ReadPlanner())->plan($file->reader()->footer(), $file->indexes(), null), new PhpBackend(), )); $source->close(); @@ -69,8 +68,7 @@ public function test_without_flow_php_a_rust_engine_file_reads_through_php(): vo static::assertInstanceOf(Generator::class, $source->batches( schema(int_schema('id')), 10, - null, - null, + (new ReadPlanner())->plan($file->reader()->footer(), $file->indexes(), null), new PhpBackend(), )); $source->close(); diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/FlowToParquetSchemaTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/FlowToParquetSchemaTest.php index 8fce623443..50295d4b68 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/FlowToParquetSchemaTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/FlowToParquetSchemaTest.php @@ -13,6 +13,7 @@ use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\definition_from_type; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\integer_schema; use function Flow\ETL\DSL\json_schema; @@ -26,6 +27,7 @@ use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_list; use function Flow\Types\DSL\type_map; +use function Flow\Types\DSL\type_numeric_string; use function Flow\Types\DSL\type_string; use function Flow\Types\DSL\type_structure; @@ -122,4 +124,12 @@ public function test_converting_nested_structure_with_optional_elements_throws() 'a' => type_structure(['b' => type_string(), 'c' => structure_element('c', type_string(), optional: true)]), ])))); } + + public function test_a_numeric_string_column_is_written_as_a_string(): void + { + static::assertEquals( + ParquetSchema::with(FlatColumn::string('id')), + (new SchemaConverter())->toParquet(schema(definition_from_type('id', type_numeric_string(), true))), + ); + } } diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileBatchesTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileBatchesTest.php index 2d91a62f77..cc66637925 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileBatchesTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileBatchesTest.php @@ -23,6 +23,8 @@ use function array_map; use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\lit; +use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Filesystem\DSL\memory_filesystem; @@ -45,7 +47,13 @@ public function test_an_offset_that_covers_the_file_skips_it_and_returns_its_row new SchemaConverter(), [], ); - $file = (new ParquetFileBatches($opener, FileColumnsContext::discovering(names: []), null, null))->batches( + $file = (new ParquetFileBatches( + $opener, + FileColumnsContext::discovering(names: []), + null, + null, + schema(int_schema('id', nullable: true)), + ))->batches( new SourceFile(path('memory://a.parquet')), schema(int_schema('id', nullable: true)), 10, @@ -72,7 +80,13 @@ public function test_an_offset_inside_the_file_starts_the_read_there_and_is_cons new SchemaConverter(), [], ); - $file = (new ParquetFileBatches($opener, FileColumnsContext::discovering(names: []), null, null))->batches( + $file = (new ParquetFileBatches( + $opener, + FileColumnsContext::discovering(names: []), + null, + null, + schema(int_schema('id', nullable: true)), + ))->batches( new SourceFile(path('memory://a.parquet')), schema(int_schema('id', nullable: true)), 10, @@ -108,6 +122,7 @@ public function test_the_kept_first_file_is_read_instead_of_opened_again(): void FileColumnsContext::discovering(names: []), $kept, null, + schema(int_schema('id', nullable: true)), ))->batches( new SourceFile(path('memory://a.parquet')), schema(int_schema('id', nullable: true)), @@ -137,7 +152,13 @@ public function test_a_kept_first_file_of_another_path_is_closed_and_the_file_op $kept = $opener->open(new SourceFile(path('memory://a.parquet'))); $rows = iterator_to_array( - (new ParquetFileBatches($opener, FileColumnsContext::discovering(names: []), $kept, null))->batches( + (new ParquetFileBatches( + $opener, + FileColumnsContext::discovering(names: []), + $kept, + null, + schema(int_schema('id', nullable: true)), + ))->batches( new SourceFile(path('memory://b.parquet')), schema(int_schema('id', nullable: true)), 10, @@ -166,7 +187,13 @@ public function test_close_releases_a_kept_first_file_the_read_never_reached(): ); $kept = $opener->open(new SourceFile(path('memory://a.parquet'))); - (new ParquetFileBatches($opener, FileColumnsContext::discovering(names: []), $kept, null))->close(); + (new ParquetFileBatches( + $opener, + FileColumnsContext::discovering(names: []), + $kept, + null, + schema(int_schema('id', nullable: true)), + ))->close(); static::assertSame($filesystem->readFromCalls, $filesystem->closedStreams()); } @@ -191,6 +218,7 @@ public function test_a_file_that_diverges_from_the_derived_schema_is_refused_and FileColumnsContext::discovering(names: []), null, new DerivedSchema(schema(int_schema('id', nullable: true)), 'memory://a.parquet'), + schema(int_schema('id')), ))->batches( new SourceFile(path('memory://b.parquet')), schema(int_schema('id')), @@ -203,4 +231,132 @@ public function test_a_file_that_diverges_from_the_derived_schema_is_refused_and static::assertSame($filesystem->readFromCalls, $filesystem->closedStreams()); } } + + public function test_a_body_predicate_skips_the_row_groups_it_proves_cannot_match(): void + { + $filesystem = memory_filesystem(); + ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 9, 3); + $file = (new ParquetFileBatches( + ParquetFilesContext::opener($filesystem), + FileColumnsContext::discovering(names: []), + null, + null, + schema(int_schema('id', nullable: true), str_schema('name', nullable: true)), + [ref('id')->greaterThan(lit(5))], + ))->batches( + new SourceFile(path('memory://groups.parquet')), + schema(int_schema('id', nullable: true), str_schema('name', nullable: true)), + 10, + new PhpBackend(), + new ReadWindow(0), + ); + + static::assertSame(3, ParquetFilesContext::rowCount($file)); + } + + public function test_the_conjunct_on_a_column_the_file_stores_as_another_type_is_dropped(): void + { + $filesystem = memory_filesystem(); + ParquetFilesContext::unsignedInt64($filesystem, 'memory://uint64.parquet', ['9', '10', '100']); + // the body reads the UINT_64 leaf as a string, ordered by bytes: '10' and '100' are below '2', numerically none is + $file = (new ParquetFileBatches( + ParquetFilesContext::opener($filesystem), + FileColumnsContext::discovering(names: []), + null, + null, + schema(str_schema('big', nullable: true)), + [ref('big')->lessThan(lit('2'))], + ))->batches( + new SourceFile(path('memory://uint64.parquet')), + schema(str_schema('big', nullable: true)), + 10, + new PhpBackend(), + new ReadWindow(0), + ); + + static::assertSame(3, ParquetFilesContext::rowCount($file)); + } + + public function test_equality_on_a_column_absent_from_the_file_reads_no_row(): void + { + $filesystem = memory_filesystem(); + ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 9, 3); + $file = (new ParquetFileBatches( + ParquetFilesContext::opener($filesystem), + FileColumnsContext::discovering(names: []), + null, + null, + schema( + int_schema('id', nullable: true), + str_schema('name', nullable: true), + str_schema('missing', nullable: true), + ), + [ref('missing')->equals(lit('x'))], + ))->batches( + new SourceFile(path('memory://groups.parquet')), + schema( + int_schema('id', nullable: true), + str_schema('name', nullable: true), + str_schema('missing', nullable: true), + ), + 10, + new PhpBackend(), + new ReadWindow(0), + ); + + static::assertSame(0, ParquetFilesContext::rowCount($file)); + } + + public function test_is_null_on_a_column_absent_from_the_file_reads_every_row(): void + { + $filesystem = memory_filesystem(); + ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 9, 3); + $file = (new ParquetFileBatches( + ParquetFilesContext::opener($filesystem), + FileColumnsContext::discovering(names: []), + null, + null, + schema( + int_schema('id', nullable: true), + str_schema('name', nullable: true), + str_schema('missing', nullable: true), + ), + [ref('missing')->isNull()], + ))->batches( + new SourceFile(path('memory://groups.parquet')), + schema( + int_schema('id', nullable: true), + str_schema('name', nullable: true), + str_schema('missing', nullable: true), + ), + 10, + new PhpBackend(), + new ReadWindow(0), + ); + + static::assertSame(9, ParquetFilesContext::rowCount($file)); + } + + public function test_every_group_pruned_still_consumes_the_raw_offset(): void + { + $filesystem = memory_filesystem(); + ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 9, 3); + $file = (new ParquetFileBatches( + ParquetFilesContext::opener($filesystem), + FileColumnsContext::discovering(names: []), + null, + null, + schema(int_schema('id', nullable: true), str_schema('name', nullable: true)), + [ref('id')->greaterThan(lit(100))], + ))->batches( + new SourceFile(path('memory://groups.parquet')), + schema(int_schema('id', nullable: true), str_schema('name', nullable: true)), + 10, + new PhpBackend(), + new ReadWindow(2), + ); + + static::assertSame(0, ParquetFilesContext::rowCount($file)); + static::assertSame(2, $file->getReturn()); + } } diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSourceFileTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSourceFileTest.php index 2ffe2c9b15..f1ed6118bf 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSourceFileTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSourceFileTest.php @@ -10,6 +10,7 @@ use Flow\ETL\Tests\Double\CountingFilesystem; use Flow\ETL\Tests\FlowTestCase; use Flow\Filesystem\Local\NativeLocalFilesystem; +use Flow\Parquet\Reader\ReadPlanner; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\schema; @@ -32,7 +33,12 @@ public function test_open_reads_the_batches_of_the_file(): void $source = $file->open(); $counts = []; - foreach ($source->batches(schema(int_schema('id')), 400, 100, 850, new PhpBackend()) as $rows) { + foreach ($source->batches( + schema(int_schema('id')), + 400, + (new ReadPlanner())->plan($file->file->reader()->footer(), $file->file->indexes(), null, 100, 850), + new PhpBackend(), + ) as $rows) { $counts[] = $rows->count(); } diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetToFlowSchemaTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetToFlowSchemaTest.php index 51d51ae828..62a9647f2a 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetToFlowSchemaTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetToFlowSchemaTest.php @@ -7,11 +7,15 @@ use Flow\ETL\Adapter\Parquet\SchemaConverter; use Flow\ETL\Tests\FlowTestCase; use Flow\Parquet\ParquetFile\Schema; +use Flow\Parquet\ParquetFile\Schema\ConvertedType; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\ListElement; +use Flow\Parquet\ParquetFile\Schema\LogicalType; use Flow\Parquet\ParquetFile\Schema\MapKey; use Flow\Parquet\ParquetFile\Schema\MapValue; use Flow\Parquet\ParquetFile\Schema\NestedColumn; +use Flow\Parquet\ParquetFile\Schema\PhysicalType; +use Flow\Parquet\ParquetFile\Schema\Repetition; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\date_schema; @@ -27,9 +31,11 @@ use function Flow\ETL\DSL\time_schema; use function Flow\ETL\DSL\uuid_schema; use function Flow\Types\DSL\type_boolean; +use function Flow\Types\DSL\type_equals; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_list; use function Flow\Types\DSL\type_map; +use function Flow\Types\DSL\type_numeric_string; use function Flow\Types\DSL\type_optional; use function Flow\Types\DSL\type_string; use function Flow\Types\DSL\type_structure; @@ -123,4 +129,31 @@ public function test_converting_struct_to_flow_schema(): void $flowSchema, ); } + + public function test_uint64_columns_are_numeric_strings(): void + { + $flowSchema = (new SchemaConverter())->toFlow(Schema::with( + new FlatColumn('converted', PhysicalType::INT64, ConvertedType::UINT_64, repetition: Repetition::REQUIRED), + new FlatColumn('logical', PhysicalType::INT64, logicalType: LogicalType::integer(64, false)), + new FlatColumn('uint32', PhysicalType::INT32, ConvertedType::UINT_32), + )); + + static::assertTrue(type_equals(type_numeric_string(), $flowSchema->get('converted')->type())); + static::assertFalse($flowSchema->get('converted')->isNullable()); + static::assertTrue(type_equals(type_numeric_string(), $flowSchema->get('logical')->type())); + static::assertTrue($flowSchema->get('logical')->isNullable()); + static::assertTrue(type_equals(type_integer(), $flowSchema->get('uint32')->type())); + static::assertTrue(type_equals( + type_optional(type_numeric_string()), + (new SchemaConverter())->parquetToFlowType( + new FlatColumn('u', PhysicalType::INT64, logicalType: LogicalType::integer(64, false)), + ), + )); + static::assertTrue(type_equals( + type_numeric_string(), + (new SchemaConverter())->parquetToFlowType( + new FlatColumn('u', PhysicalType::INT64, ConvertedType::UINT_64, repetition: Repetition::REQUIRED), + ), + )); + } } diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/PhpParquetOpenSourceTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/PhpParquetOpenSourceTest.php index 61e9591630..209be0be23 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/PhpParquetOpenSourceTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/PhpParquetOpenSourceTest.php @@ -9,6 +9,7 @@ use Flow\ETL\Column\PhpBackend; use Flow\ETL\Tests\FlowTestCase; use Flow\Parquet\Engine\PhpParquetEngine; +use Flow\Parquet\Exception\InvalidArgumentException; use Flow\Parquet\Reader; use PHPUnit\Framework\Attributes\TestWith; @@ -18,6 +19,7 @@ use function Flow\ETL\DSL\str_schema; use function Flow\Filesystem\DSL\memory_filesystem; use function Flow\Filesystem\DSL\path; +use function iterator_to_array; use function range; final class PhpParquetOpenSourceTest extends FlowTestCase @@ -25,11 +27,11 @@ final class PhpParquetOpenSourceTest extends FlowTestCase /** * @param list $ids */ - #[TestWith([null, null, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]])] + #[TestWith([0, null, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]])] #[TestWith([3, 5, [3, 4, 5, 6, 7]])] #[TestWith([6, null, [6, 7, 8, 9]])] #[TestWith([0, 1, [0]])] - public function test_offset_and_limit_cross_row_groups(?int $offset, ?int $limit, array $ids): void + public function test_a_planned_window_crosses_row_groups(int $offset, ?int $limit, array $ids): void { $filesystem = memory_filesystem(); ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 10, 4); @@ -41,8 +43,7 @@ public function test_offset_and_limit_cross_row_groups(?int $offset, ?int $limit ))->readStream($filesystem->readFrom(path('memory://groups.parquet')))))->batches( schema(int_schema('id')), 3, - $offset, - $limit, + ParquetFilesContext::plan($filesystem, 'memory://groups.parquet', $offset, $limit), new PhpBackend(), ) as $rows) { static::assertLessThanOrEqual(3, $rows->count()); @@ -62,7 +63,12 @@ public function test_batches_hold_the_schema_columns(): void $names = []; - foreach ($source->batches(schema(str_schema('name')), 100, null, null, new PhpBackend()) as $rows) { + foreach ($source->batches( + schema(str_schema('name')), + 100, + ParquetFilesContext::plan($filesystem, 'memory://groups.parquet'), + new PhpBackend(), + ) as $rows) { static::assertEquals(schema(str_schema('name')), $rows->schema()); $names = [...$names, ...$rows->column('name')->values()]; } @@ -71,4 +77,40 @@ public function test_batches_hold_the_schema_columns(): void static::assertSame(array_map(static fn(int $id): string => 'n' . $id, range(0, 9)), $names); } + + public function test_a_column_the_file_lacks_is_refused(): void + { + $filesystem = memory_filesystem(); + ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 10, 4); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Parquet file has no column "missing"'); + + iterator_to_array((new PhpParquetOpenSource((new Reader( + engine: new PhpParquetEngine(), + ))->readStream($filesystem->readFrom(path('memory://groups.parquet')))))->batches( + schema(int_schema('missing')), + 10, + ParquetFilesContext::plan($filesystem, 'memory://groups.parquet'), + new PhpBackend(), + )); + } + + public function test_a_batch_size_below_one_is_refused(): void + { + $filesystem = memory_filesystem(); + ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 10, 4); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Parquet batch size must be greater than 0'); + + iterator_to_array((new PhpParquetOpenSource((new Reader( + engine: new PhpParquetEngine(), + ))->readStream($filesystem->readFrom(path('memory://groups.parquet')))))->batches( + schema(int_schema('id')), + 0, + ParquetFilesContext::plan($filesystem, 'memory://groups.parquet'), + new PhpBackend(), + )); + } } diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/Pushdown/FilterTranslatorTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/Pushdown/FilterTranslatorTest.php new file mode 100644 index 0000000000..b918586c4f --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/Pushdown/FilterTranslatorTest.php @@ -0,0 +1,195 @@ + [ref('id')->equals(lit(5)), parquet_column('id')->equals(5)]; + yield 'not equals' => [ref('id')->notEquals(lit(5)), parquet_column('id')->notEquals(5)]; + yield 'less than' => [ref('id')->lessThan(lit(5)), parquet_column('id')->lessThan(5)]; + yield 'less than or equal' => [ref('id')->lessThanEqual(lit(5)), parquet_column('id')->lessThanOrEqual(5)]; + yield 'greater than' => [ref('id')->greaterThan(lit(5)), parquet_column('id')->greaterThan(5)]; + yield 'greater than or equal' => [ + ref('id')->greaterThanEqual(lit(5)), + parquet_column('id')->greaterThanOrEqual(5), + ]; + yield 'a literal on the left is flipped' => [lit(5)->lessThan(ref('id')), parquet_column('id')->greaterThan(5)]; + yield 'a literal on the left of an equality' => [lit(5)->equals(ref('id')), parquet_column('id')->equals(5)]; + yield 'same on the same type' => [ref('id')->same(lit(5)), parquet_column('id')->equals(5)]; + yield 'not same on the same type' => [ref('id')->notSame(lit(5)), parquet_column('id')->notEquals(5)]; + yield 'is true' => [ref('flag')->isTrue(), parquet_column('flag')->equals(true)]; + yield 'is null' => [ref('id')->isNull(), parquet_column('id')->isNull()]; + yield 'is not null' => [ref('id')->isNotNull(), parquet_column('id')->isNotNull()]; + yield 'not' => [not(ref('id')->equals(lit(5))), parquet_column('id')->equals(5)->negate()]; + yield 'an or' => [ + ref('id')->equals(lit(5))->or(ref('name')->equals(lit('a'))), + parquet_column('id')->equals(5)->or(parquet_column('name')->equals('a')), + ]; + yield 'an and inside an or' => [ + ref('id')->equals(lit(5))->or(ref('id')->greaterThan(lit(1))->and(ref('name')->equals(lit('a')))), + parquet_column('id')->equals(5)->or( + parquet_column('id')->greaterThan(1)->and(parquet_column('name')->equals('a')), + ), + ]; + yield 'is in' => [ref('id')->isIn(lit([1, 2])), parquet_column('id')->in([1, 2])]; + yield 'is in drops null elements' => [ref('id')->isIn(lit([1, null, 2])), parquet_column('id')->in([1, 2])]; + yield 'a negated is in without null' => [ + not(ref('id')->isIn(lit([1, 2]))), + parquet_column('id')->in([1, 2])->negate(), + ]; + yield 'a negated is in holding null drops the null' => [ + not(ref('id')->isIn(lit([1, null]))), + parquet_column('id')->in([1])->negate(), + ]; + yield 'starts with' => [ref('name')->startsWith(lit('ab')), parquet_column('name')->startsWith('ab')]; + yield 'between left inclusive' => [ + ref('id')->between(lit(1), lit(5)), + parquet_column('id')->greaterThanOrEqual(1)->and(parquet_column('id')->lessThan(5)), + ]; + yield 'between inclusive' => [ + ref('id')->between(lit(1), lit(5), lit(Boundary::INCLUSIVE)), + parquet_column('id')->greaterThanOrEqual(1)->and(parquet_column('id')->lessThanOrEqual(5)), + ]; + yield 'between right inclusive' => [ + ref('id')->between(lit(1), lit(5), lit(Boundary::RIGHT_INCLUSIVE)), + parquet_column('id')->greaterThan(1)->and(parquet_column('id')->lessThanOrEqual(5)), + ]; + yield 'between exclusive' => [ + ref('id')->between(lit(1), lit(5), lit(Boundary::EXCLUSIVE)), + parquet_column('id')->greaterThan(1)->and(parquet_column('id')->lessThan(5)), + ]; + yield 'an int literal on a float column becomes a float' => [ + ref('price')->greaterThan(lit(10)), + parquet_column('price')->greaterThan(10.0), + ]; + yield 'an int literal on a float column at 2^53' => [ + ref('price')->equals(lit(9_007_199_254_740_992)), + parquet_column('price')->equals(9_007_199_254_740_992.0), + ]; + yield 'an integral float literal on an int column becomes an int' => [ + ref('id')->lessThan(lit(5.0)), + parquet_column('id')->lessThan(5), + ]; + yield 'a float literal on a float column' => [ + ref('price')->lessThan(lit(1.5)), + parquet_column('price')->lessThan(1.5), + ]; + yield 'strings' => [ref('name')->greaterThan(lit('b')), parquet_column('name')->greaterThan('b')]; + yield 'a canonical digit string on a numeric string column' => [ + ref('big')->greaterThan(lit('18446744073709551614')), + parquet_column('big')->greaterThan('18446744073709551614'), + ]; + yield 'zero on a numeric string column' => [ref('big')->equals(lit('0')), parquet_column('big')->equals('0')]; + yield 'a uuid literal becomes its string form' => [ + ref('uuid')->equals(lit(new Uuid('6c2f6e9e-4ec0-4ef7-9b2e-7f2d3f2f6b11'))), + parquet_column('uuid')->equals('6c2f6e9e-4ec0-4ef7-9b2e-7f2d3f2f6b11'), + ]; + yield 'a datetime literal passes through' => [ + ref('ts')->greaterThan(lit(new DateTimeImmutable('2024-01-01 12:00:00'))), + parquet_column('ts')->greaterThan(new DateTimeImmutable('2024-01-01 12:00:00')), + ]; + yield 'a date literal passes through' => [ + ref('day')->lessThan(lit(new DateTimeImmutable('2024-01-01'))), + parquet_column('day')->lessThan(new DateTimeImmutable('2024-01-01')), + ]; + yield 'a date literal on a datetime column' => [ + ref('ts')->greaterThanEqual(lit(new DateTimeImmutable('2024-01-01'))), + parquet_column('ts')->greaterThanOrEqual(new DateTimeImmutable('2024-01-01')), + ]; + yield 'a datetime literal on a date column' => [ + ref('day')->lessThan(lit(new DateTimeImmutable('2024-01-01 12:00:00'))), + parquet_column('day')->lessThan(new DateTimeImmutable('2024-01-01 12:00:00')), + ]; + yield 'json is null' => [ref('doc')->isNull(), parquet_column('doc')->isNull()]; + } + + #[DataProvider('translated')] + public function test_a_pushable_conjunct_is_translated(ScalarFunction $function, Filter $expected): void + { + $translated = (new FilterTranslator())->translate([$function], BodySchemaMother::everyLeafType()); + + static::assertCount(1, $translated); + static::assertEquals($expected, $translated[0]->filter); + } + + public static function refused(): Generator + { + yield 'a column outside the body' => [ref('year')->equals(lit(2024))]; + yield 'two references' => [ref('id')->equals(ref('price'))]; + yield 'two literals' => [lit(1)->equals(lit(1))]; + yield 'an unsupported function' => [ref('name')->endsWith(lit('a'))]; + yield 'an unsupported function inside a not' => [not(ref('name')->endsWith(lit('a')))]; + yield 'an unsupported function inside an or' => [ref('id')->equals(lit(5))->or(ref('name')->endsWith(lit( + 'a', + )))]; + yield 'is in over a reference haystack' => [ref('id')->isIn(ref('ids'))]; + yield 'is in with a refused element' => [ref('id')->isIn(lit([1, 'a']))]; + yield 'starts with on an int column' => [ref('id')->startsWith(lit('1'))]; + yield 'between with a refused bound' => [ref('id')->between(lit(1), lit('5'))]; + yield 'same across int and float' => [ref('id')->same(lit(5.0))]; + yield 'same on a numeric string column' => [ref('big')->same(lit('5'))]; + yield 'an int literal on a float column above 2^53' => [ref('price')->equals(lit(9_007_199_254_740_993))]; + yield 'a fractional float literal on an int column' => [ref('id')->lessThan(lit(5.5))]; + yield 'an integral float literal on an int column at 2^53' => [ref('id')->equals(lit(9_007_199_254_740_992.0))]; + yield 'a NaN literal' => [ref('price')->equals(lit(NAN))]; + yield 'an int literal on a string column' => [ref('name')->equals(lit(5))]; + yield 'a string literal on an int column' => [ref('id')->equals(lit('5'))]; + yield 'a numeric string column with a leading zero literal' => [ref('big')->equals(lit('05'))]; + yield 'a numeric string column with a non digit literal' => [ref('big')->equals(lit('-5'))]; + yield 'a numeric string column with an int literal' => [ref('big')->equals(lit(5))]; + yield 'ordering on a uuid column' => [ + ref('uuid')->greaterThan(lit(new Uuid('6c2f6e9e-4ec0-4ef7-9b2e-7f2d3f2f6b11'))), + ]; + yield 'equality on a json column' => [ref('doc')->equals(lit('{}'))]; + yield 'same across date and datetime' => [ref('day')->same(lit(new DateTimeImmutable('2024-01-01 12:00:00')))]; + } + + #[DataProvider('refused')] + public function test_a_conjunct_that_cannot_be_pushed_is_refused(ScalarFunction $function): void + { + static::assertSame([], (new FilterTranslator())->translate([$function], BodySchemaMother::everyLeafType())); + } + + public function test_every_conjunct_is_translated_on_its_own(): void + { + $translated = (new FilterTranslator())->translate([ + ref('id')->greaterThan(lit(1)), + ref('name')->endsWith(lit('a')), + ref('name')->equals(lit('b'))->or(ref('price')->lessThan(lit(2.0))), + ], BodySchemaMother::everyLeafType()); + + static::assertEquals( + [ + new TranslatedConjunct(parquet_column('id')->greaterThan(1), ['id']), + new TranslatedConjunct( + parquet_column('name')->equals('b')->or(parquet_column('price')->lessThan(2.0)), + ['name', 'price'], + ), + ], + $translated, + ); + } +} diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/Pushdown/TranslatedConjunctTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/Pushdown/TranslatedConjunctTest.php new file mode 100644 index 0000000000..f740e85ec8 --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/Pushdown/TranslatedConjunctTest.php @@ -0,0 +1,42 @@ +equals(1), ['id', 'name']))->readsAs( + schema(int_schema('id'), str_schema('name')), + schema(int_schema('id', nullable: true), str_schema('name', nullable: true)), + )); + } + + public function test_a_column_the_file_stores_as_another_type_is_not_read_as_is(): void + { + static::assertFalse((new TranslatedConjunct(parquet_column('big')->lessThan('2'), ['id', 'big']))->readsAs( + schema(int_schema('id'), definition_from_type('big', type_numeric_string())), + schema(int_schema('id'), str_schema('big')), + )); + } + + public function test_a_column_absent_from_the_file_is_read_as_is(): void + { + static::assertTrue((new TranslatedConjunct(parquet_column('missing')->isNull(), ['missing']))->readsAs( + schema(int_schema('id')), + schema(int_schema('id'), str_schema('missing', nullable: true)), + )); + } +} diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/RustParquetOpenSourceTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/RustParquetOpenSourceTest.php index dd5bfd380a..2293928b33 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/RustParquetOpenSourceTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/RustParquetOpenSourceTest.php @@ -31,11 +31,11 @@ final class RustParquetOpenSourceTest extends FlowTestCase /** * @param list $ids */ - #[TestWith([null, null, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]])] + #[TestWith([0, null, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]])] #[TestWith([3, 5, [3, 4, 5, 6, 7]])] #[TestWith([6, null, [6, 7, 8, 9]])] #[TestWith([0, 1, [0]])] - public function test_offset_and_limit_cross_row_groups(?int $offset, ?int $limit, array $ids): void + public function test_a_planned_window_crosses_row_groups(int $offset, ?int $limit, array $ids): void { $filesystem = memory_filesystem(); ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 10, 4); @@ -44,7 +44,12 @@ public function test_offset_and_limit_cross_row_groups(?int $offset, ?int $limit foreach ((new RustParquetOpenSource(new RustParquetFileReader($filesystem->readFrom(path( 'memory://groups.parquet', - )))))->batches(schema(int_schema('id')), 3, $offset, $limit, new AdaptiveBackend()) as $rows) { + )))))->batches( + schema(int_schema('id')), + 3, + ParquetFilesContext::plan($filesystem, 'memory://groups.parquet', $offset, $limit), + new AdaptiveBackend(), + ) as $rows) { static::assertLessThanOrEqual(3, $rows->count()); $read = [...$read, ...$rows->column('id')->values()]; } @@ -62,7 +67,12 @@ public function test_columns_are_adopted_by_the_backend(): void $names = []; - foreach ($source->batches(schema(str_schema('name')), 100, null, null, new PhpBackend()) as $rows) { + foreach ($source->batches( + schema(str_schema('name')), + 100, + ParquetFilesContext::plan($filesystem, 'memory://groups.parquet'), + new PhpBackend(), + ) as $rows) { static::assertEquals(schema(str_schema('name')), $rows->schema()); static::assertInstanceOf(ScalarColumn::class, $rows->column('name')); $names = [...$names, ...$rows->column('name')->values()]; @@ -80,13 +90,23 @@ public function test_a_second_batches_call_closes_the_first_reader(): void $source = new RustParquetOpenSource(new RustParquetFileReader($filesystem->readFrom(path( 'memory://groups.parquet', )))); - $first = $source->batches(schema(int_schema('id')), 3, null, null, new AdaptiveBackend()); + $first = $source->batches( + schema(int_schema('id')), + 3, + ParquetFilesContext::plan($filesystem, 'memory://groups.parquet'), + new AdaptiveBackend(), + ); $first->rewind(); static::assertSame([0, 1, 2], $first->current()->column('id')->values()); $second = []; - foreach ($source->batches(schema(int_schema('id')), 3, null, null, new AdaptiveBackend()) as $rows) { + foreach ($source->batches( + schema(int_schema('id')), + 3, + ParquetFilesContext::plan($filesystem, 'memory://groups.parquet'), + new AdaptiveBackend(), + ) as $rows) { $second = [...$second, ...$rows->column('id')->values()]; } diff --git a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlCursorExtractorTest.php b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlCursorExtractorTest.php index b1f9eb39c7..36c4df399e 100644 --- a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlCursorExtractorTest.php +++ b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlCursorExtractorTest.php @@ -11,6 +11,7 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaMismatchException; use Flow\ETL\Exception\SchemaNotDerivableException; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowTestCase; use Flow\PostgreSql\AST\Transformers\ExplainConfig; @@ -322,7 +323,7 @@ public function test_pushed_limit_issues_no_query_once_satisfied(): void ); $extractor = from_pgsql_cursor($client, 'SELECT id FROM t'); - self::assertExtractedRowsCount(1500, $extractor, limit: 1500); + self::assertExtractedRowsCount(1500, $extractor, scan: new FileScan(1500)); // FETCH 1000, then FETCH 500 - a full narrowed fetch, which only the limit itself can stop static::assertSame(2, $client->callsTo('cursor')); } diff --git a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlKeySetExtractorTest.php b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlKeySetExtractorTest.php index 598840c54b..511bcf23e1 100644 --- a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlKeySetExtractorTest.php +++ b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlKeySetExtractorTest.php @@ -14,6 +14,7 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\Exception\SchemaNotDerivableException; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowTestCase; use Flow\PostgreSql\AST\Transformers\ExplainConfig; @@ -387,7 +388,7 @@ public function test_pushed_limit_issues_no_query_once_satisfied(): void pgsql_pagination_key_set(pgsql_pagination_key_asc('id')), ); - self::assertExtractedRowsCount(1500, $extractor, limit: 1500); + self::assertExtractedRowsCount(1500, $extractor, scan: new FileScan(1500)); // 1000, then a page narrowed to the 500 still wanted - and no third query for row 1501 static::assertSame(2, $client->callsTo('cursor')); } diff --git a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLimitOffsetExtractorTest.php b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLimitOffsetExtractorTest.php index 17b3e1d473..e38cf78f8b 100644 --- a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLimitOffsetExtractorTest.php +++ b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLimitOffsetExtractorTest.php @@ -10,6 +10,7 @@ use Flow\ETL\Cardinality; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaNotDerivableException; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Rows; use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowTestCase; @@ -303,7 +304,7 @@ public function test_pushed_limit_skips_the_count_query(): void ->willReturnCursors(new StubCursor([['id' => '1'], ['id' => '2']]), new StubCursor([['id' => '3']])); $extractor = from_pgsql_limit_offset($client, 'SELECT id FROM t ORDER BY id')->withBatchSize(2); - self::assertExtractedRowsCount(3, $extractor, limit: 3); + self::assertExtractedRowsCount(3, $extractor, scan: new FileScan(3)); // the count query is the only caller of fetchScalarInt() static::assertSame(0, $client->callsTo('fetchScalarInt')); } diff --git a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextExtractor.php b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextExtractor.php index 6bbb6fbcba..c404495d37 100644 --- a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextExtractor.php +++ b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextExtractor.php @@ -11,9 +11,11 @@ use Flow\ETL\Extractor\File\FileExtractor; use Flow\ETL\Extractor\File\FileReading; use Flow\ETL\Extractor\File\FileReadLoop; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\File\ListedFiles; use Flow\ETL\Extractor\File\MetadataColumnsExtractor; use Flow\ETL\Extractor\File\ReadWindow; +use Flow\ETL\Extractor\File\Scanning; use Flow\ETL\Extractor\RewindableExtractor; use Flow\ETL\Extractor\Signal; use Flow\ETL\Extractor\Statistics; @@ -23,8 +25,6 @@ use Flow\Filesystem\Filesystem; use Flow\Filesystem\Local\NativeLocalFilesystem; use Flow\Filesystem\Path; -use Flow\Filesystem\Path\Filter; -use Flow\Filesystem\Path\Filter\OnlyFiles; use Generator; use function Flow\ETL\DSL\schema; @@ -40,6 +40,8 @@ final class TextExtractor implements MetadataColumnsExtractor, RewindableExtractor { + use Scanning; + private ?Schema $schema = null; use Batches; @@ -74,19 +76,20 @@ public function isRepeatable(): bool /** * @return Generator */ - public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator + public function scan(FlowContext $context, FileScan $scan): Generator { $fileColumns = $this->fileColumns($this->filesystem, $this->path); yield from (new FileReadLoop( $fileColumns, $fileColumns->declare($this->schema ?? schema(str_schema('text'))), + $scan->conjuncts, ))->read( - iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $pathFilter), false), + iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $scan->pathFilter), false), new TextFileBatches($this->filesystem), $this->batchSize(), - $context->backend(), - new ReadWindow(limit: $limit), + $context, + new ReadWindow(limit: $scan->limit), ); } diff --git a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTest.php b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTest.php index 66513a4c61..c8c7f26f4a 100644 --- a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTest.php +++ b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTest.php @@ -8,6 +8,7 @@ use Flow\ETL\Adapter\Text\TextExtractor; use Flow\ETL\Cardinality; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\Signal; use Flow\ETL\Tests\Context\ExtractedRows; use Flow\ETL\Tests\Context\MemoryFiles; @@ -56,7 +57,7 @@ public function test_limit(): void $extractor = from_text(path_real(__DIR__ . '/../Fixtures/orders_flow.csv')); $extractor->withBatchSize(1); - self::assertExtractedRowsCount(2, $extractor, flow_context(config()), limit: 2); + self::assertExtractedRowsCount(2, $extractor, flow_context(config()), scan: new FileScan(2)); } public function test_partition_columns_are_not_leaking_between_streams(): void @@ -116,7 +117,7 @@ public function test_limit_reached_on_the_first_file_tail_batch_skips_the_remain { $extractor = from_text(__DIR__ . '/../Fixtures/cross_stream/*/data.txt')->withBatchSize(10); - static::assertCount(1, ExtractedRows::of($extractor, limit: 1)); + static::assertCount(1, ExtractedRows::of($extractor, scan: new FileScan(1))); } public function test_metadata_columns_extend_a_declared_schema(): void diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLParserExtractor.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLParserExtractor.php index 08ac7ea654..36eb7d0254 100644 --- a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLParserExtractor.php +++ b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLParserExtractor.php @@ -11,9 +11,11 @@ use Flow\ETL\Extractor\File\FileExtractor; use Flow\ETL\Extractor\File\FileReading; use Flow\ETL\Extractor\File\FileReadLoop; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\File\ListedFiles; use Flow\ETL\Extractor\File\MetadataColumnsExtractor; use Flow\ETL\Extractor\File\ReadWindow; +use Flow\ETL\Extractor\File\Scanning; use Flow\ETL\Extractor\RewindableExtractor; use Flow\ETL\Extractor\Signal; use Flow\ETL\Extractor\Statistics; @@ -23,8 +25,6 @@ use Flow\Filesystem\Filesystem; use Flow\Filesystem\Local\NativeLocalFilesystem; use Flow\Filesystem\Path; -use Flow\Filesystem\Path\Filter; -use Flow\Filesystem\Path\Filter\OnlyFiles; use Generator; use function Flow\ETL\DSL\schema; @@ -42,6 +42,7 @@ final class XMLParserExtractor implements { use Batches; use FileReading; + use Scanning; /** * @var int<1, max> @@ -96,19 +97,20 @@ public function isRepeatable(): bool /** * @return Generator */ - public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator + public function scan(FlowContext $context, FileScan $scan): Generator { $fileColumns = $this->fileColumns($this->filesystem, $this->path); yield from (new FileReadLoop( $fileColumns, $fileColumns->declare($this->schema ?? schema(xml_schema('node'))), + $scan->conjuncts, ))->read( - iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $pathFilter), false), + iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $scan->pathFilter), false), new XMLFileBatches($this->filesystem, $this->xmlNodePath, $this->bufferSize), $this->batchSize(), - $context->backend(), - new ReadWindow(limit: $limit), + $context, + new ReadWindow(limit: $scan->limit), ); } diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLParserExtractorTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLParserExtractorTest.php index f61c618cf1..70dbfd67ba 100644 --- a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLParserExtractorTest.php +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLParserExtractorTest.php @@ -11,6 +11,7 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\Exception\SchemaMismatchException; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\Signal; use Flow\ETL\Tests\Context\ExtractedRows; use Flow\ETL\Tests\Context\MemoryFiles; @@ -386,7 +387,7 @@ public function test_limit_reached_on_the_first_file_tail_batch_skips_the_remain { $extractor = from_xml(__DIR__ . '/../Fixtures/cross_stream/*/file.xml', 'root/item')->withBatchSize(10); - static::assertCount(1, ExtractedRows::of($extractor, limit: 1)); + static::assertCount(1, ExtractedRows::of($extractor, scan: new FileScan(1))); } public function test_is_repeatable(): void diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/PhysicalFor.php b/src/core/etl/src/Flow/ETL/Column/Physical/PhysicalFor.php index ccd58a6b90..600f942940 100644 --- a/src/core/etl/src/Flow/ETL/Column/Physical/PhysicalFor.php +++ b/src/core/etl/src/Flow/ETL/Column/Physical/PhysicalFor.php @@ -120,7 +120,6 @@ public function supports(Type $type): bool $base instanceof EmptyArrayType, $base instanceof PositiveIntegerType, $base instanceof NonEmptyStringType, - $base instanceof NumericStringType, $base instanceof ClassStringType, => false, default => $this->supportsChild($base), diff --git a/src/core/etl/src/Flow/ETL/DSL/functions.php b/src/core/etl/src/Flow/ETL/DSL/functions.php index 6212068b75..18326aeee3 100644 --- a/src/core/etl/src/Flow/ETL/DSL/functions.php +++ b/src/core/etl/src/Flow/ETL/DSL/functions.php @@ -226,6 +226,7 @@ use Flow\Types\Type\Logical\JsonType; use Flow\Types\Type\Logical\ListType; use Flow\Types\Type\Logical\MapType; +use Flow\Types\Type\Logical\NumericStringType; use Flow\Types\Type\Logical\OptionalType; use Flow\Types\Type\Logical\StructureType; use Flow\Types\Type\Logical\TimeType; @@ -1620,6 +1621,7 @@ function definition_from_type( $type instanceof IntegerType => new IntegerDefinition($ref, $nullable, $metadata), $type instanceof FloatType => new FloatDefinition($ref, $nullable, $metadata), $type instanceof StringType => new StringDefinition($ref, $nullable, $metadata), + $type instanceof NumericStringType => new StringDefinition($ref, $nullable, $metadata, $type), $type instanceof DateType => new DateDefinition($ref, $nullable, $metadata), $type instanceof DateTimeType => new DateTimeDefinition($ref, $type, $nullable, $metadata), $type instanceof TimeType => new TimeDefinition($ref, $nullable, $metadata), diff --git a/src/core/etl/src/Flow/ETL/Executor.php b/src/core/etl/src/Flow/ETL/Executor.php index 60b6e75af9..2a2988812b 100644 --- a/src/core/etl/src/Flow/ETL/Executor.php +++ b/src/core/etl/src/Flow/ETL/Executor.php @@ -10,7 +10,6 @@ use Flow\ETL\Executor\PhysicalPlan; use Flow\ETL\Executor\Pipeline; use Flow\ETL\Extractor\File\FileExtractor; -use Flow\Filesystem\Path\Filter\OnlyFiles; use Generator; use Throwable; use WeakMap; @@ -142,19 +141,16 @@ public function executePipeline(Pipeline $pipeline): Generator 'pipeline #%d', $leaf->id, )); + $scan = $leaf->scan(); $generator = (new AdoptedBatches($leaf->context()->backend()))->of( $source instanceof FileExtractor - ? $source->extract($leaf->context(), $leaf->limit(), $leaf->pathFilter()) - : $source->extract($leaf->context(), $leaf->limit()), + ? $source->scan($leaf->context(), $scan) + : $source->extract($leaf->context(), $scan->limit), ); $sources = $leaf->sources(); if ($sources !== null) { - $generator = $sources->count( - $source, - $generator, - $leaf->limit() !== null || !$leaf->pathFilter() instanceof OnlyFiles, - ); + $generator = $sources->count($source, $generator, $scan->narrows()); } foreach ($chain as $stage) { diff --git a/src/core/etl/src/Flow/ETL/Executor/Pipeline.php b/src/core/etl/src/Flow/ETL/Executor/Pipeline.php index 534927d41b..f5b5bce309 100644 --- a/src/core/etl/src/Flow/ETL/Executor/Pipeline.php +++ b/src/core/etl/src/Flow/ETL/Executor/Pipeline.php @@ -4,9 +4,8 @@ namespace Flow\ETL\Executor; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\FlowContext; -use Flow\Filesystem\Path\Filter; -use Flow\Filesystem\Path\Filter\OnlyFiles; final readonly class Pipeline { @@ -18,10 +17,8 @@ * @param FlowContext $context the frame this pipeline was planned in * @param null|self $input the pipeline whose output is this one's rows; null when this pipeline * reads the extractor in $segments - * @param null|int<1, max> $limit handed to the source extractor when this pipeline reads it; meaningless - * behind an input edge - * @param Filter $pathFilter handed to a FileExtractor source when this pipeline reads it; meaningless - * behind an input edge + * @param FileScan $scan handed to a FileExtractor source (only its limit to any other source) when this + * pipeline reads it; meaningless behind an input edge * @param null|SourceRows $sources counts the rows of the source when this pipeline reads it and the run is * analyzed with source statistics; null otherwise */ @@ -30,8 +27,7 @@ public function __construct( private Segments $segments, private FlowContext $context, private ?self $input = null, - private ?int $limit = null, - private Filter $pathFilter = new OnlyFiles(), + private FileScan $scan = new FileScan(), private ?SourceRows $sources = null, ) {} @@ -45,17 +41,9 @@ public function context(): FlowContext return $this->context; } - /** - * @return null|int<1, max> - */ - public function limit(): ?int - { - return $this->limit; - } - - public function pathFilter(): Filter + public function scan(): FileScan { - return $this->pathFilter; + return $this->scan; } public function sources(): ?SourceRows diff --git a/src/core/etl/src/Flow/ETL/Executor/SourceRead.php b/src/core/etl/src/Flow/ETL/Executor/SourceRead.php index 602a908493..ea2572bb3a 100644 --- a/src/core/etl/src/Flow/ETL/Executor/SourceRead.php +++ b/src/core/etl/src/Flow/ETL/Executor/SourceRead.php @@ -34,7 +34,7 @@ public function rows(): int } /** - * Every read ran to its end and none had a limit or a partition filter pushed into it. + * Every read ran to its end and none had a limit, a partition filter or a predicate pushed into it. */ public function isComplete(): bool { diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/FileBatches.php b/src/core/etl/src/Flow/ETL/Extractor/File/FileBatches.php index 6e33736842..fb793345c3 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/File/FileBatches.php +++ b/src/core/etl/src/Flow/ETL/Extractor/File/FileBatches.php @@ -18,7 +18,9 @@ interface FileBatches * The return value is the rows of the window's offset this file consumed. A format that cannot skip rows is given * offset 0 and a limit that counts the offset still to skip, and returns 0: FileReadLoop slices the offset from its * rows. An OffsetSkippingFileBatches is given the offset and returns the rows it skipped. The limit is a hint a - * format may exceed - FileReadLoop trims what goes past it. + * format may exceed - FileReadLoop trims what goes past it - and null when the read carries conjuncts, whose rows + * FileReadLoop counts. FileReadLoop applies the conjuncts to every row returned; a format given them when built may + * skip what they prove cannot match. * * A file's own resources are released by the generator (finally). Resources that span files - a kept first file, * an inference sample - belong to the per-read object; the extractor that built it calls its close() in a finally diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/FileExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/File/FileExtractor.php index f8b5369cc0..f874f6b49c 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/File/FileExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/File/FileExtractor.php @@ -10,8 +10,6 @@ use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\Filesystem\Path; -use Flow\Filesystem\Path\Filter; -use Flow\Filesystem\Path\Filter\OnlyFiles; use Generator; /** @@ -21,13 +19,13 @@ interface FileExtractor extends Extractor { /** - * @param null|int<1, max> $limit see Extractor::extract() - * @param Filter $pathFilter what this read lists. The Filter step stays in the plan, so a source that lists - * more, or ignores it, is still correct. + * Reads what the plan pushed into this read and returns only rows matching every one of $scan->conjuncts. The + * limit and the path filter are hints: the plan still enforces the limit, and every path filter is also one of + * the conjuncts, so a source that reads or lists more is still correct. extract() is scan() with only a limit. * * @return Generator */ - public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator; + public function scan(FlowContext $context, FileScan $scan): Generator; public function source(): Path; diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/FileMetadataRow.php b/src/core/etl/src/Flow/ETL/Extractor/File/FileMetadataRow.php new file mode 100644 index 0000000000..d5dbc02d79 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Extractor/File/FileMetadataRow.php @@ -0,0 +1,27 @@ +path->extension(); + + return [ + 'path' => $status->path->path(), + 'protocol' => $status->path->protocol(), + 'file_name' => $status->path->filename(), + 'base_name' => $status->path->basename(), + 'is_file' => $status->isFile(), + 'is_dir' => $status->isDirectory(), + // Path::extension() answers false for an extensionless file; the column is one + // type, so the absence is spelled null rather than a boolean in a string column. + 'extension' => $extension === false ? null : $extension, + ]; + } +} diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/FileReadLoop.php b/src/core/etl/src/Flow/ETL/Extractor/File/FileReadLoop.php index 3375c2b4d1..6080039d8c 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/File/FileReadLoop.php +++ b/src/core/etl/src/Flow/ETL/Extractor/File/FileReadLoop.php @@ -4,8 +4,10 @@ namespace Flow\ETL\Extractor\File; -use Flow\ETL\Column\Backend; use Flow\ETL\Extractor\Signal; +use Flow\ETL\FlowContext; +use Flow\ETL\Function\Evaluation\Filter; +use Flow\ETL\Function\ScalarFunction; use Flow\ETL\Rows; use Flow\ETL\Schema; use Generator; @@ -14,9 +16,13 @@ final readonly class FileReadLoop { + /** + * @param list $conjuncts FileScan::$conjuncts + */ public function __construct( private FileColumns $fileColumns, private Schema $schema, + private array $conjuncts = [], ) {} /** @@ -29,17 +35,20 @@ public function read( array $sources, FileBatches $batches, int $batchSize, - Backend $backend, + FlowContext $context, ReadWindow $window, ): Generator { $body = $this->fileColumns->withoutTail($this->schema); + $backend = $context->backend(); + $filter = Filter::bind($this->schema, ...$this->conjuncts); $offset = $window->offset; $skips = $batches instanceof OffsetSkippingFileBatches; $yielded = 0; foreach ($sources as $source) { $constants = $this->fileColumns->forFile($source, $this->schema); - $wanted = $window->limit === null ? null : $window->limit - $yielded; + // a limit counts the rows the filter keeps, which a format would count before it + $wanted = $window->limit === null || $this->conjuncts !== [] ? null : $window->limit - $yielded; $file = $batches->batches( $source, $body, @@ -63,11 +72,16 @@ public function read( $batch = $batch->slice($skipped, $batch->count() - $skipped); } - if ($window->limit !== null && $batch->count() > ($window->limit - $yielded)) { - $batch = $batch->slice(0, $window->limit - $yielded); + $rows = $filter->filter($constants->fillRows($batch, $this->schema, $backend), $context); + + if ($rows->isEmpty()) { + continue; + } + + if ($window->limit !== null && $rows->count() > ($window->limit - $yielded)) { + $rows = $rows->slice(0, $window->limit - $yielded); } - $rows = $constants->fillRows($batch, $this->schema, $backend); $yielded += $rows->count(); $signal = yield $rows; diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/FileScan.php b/src/core/etl/src/Flow/ETL/Extractor/File/FileScan.php new file mode 100644 index 0000000000..2d43e85028 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Extractor/File/FileScan.php @@ -0,0 +1,77 @@ + + */ + public ?int $limit; + + /** + * @param null|int $limit the rows the plan needs, see Extractor::extract() + * @param Filter $pathFilter what this read lists + * @param list $conjuncts predicates over the source's columns, in push order; the source returns + * only rows matching every one, and $limit counts those rows + * + * @throws InvalidArgumentException when $limit is not greater than 0 + */ + public function __construct( + ?int $limit = null, + public Filter $pathFilter = new OnlyFiles(), + public array $conjuncts = [], + ) { + if ($limit !== null && $limit <= 0) { + throw new InvalidArgumentException('Limit must be greater than 0'); + } + + $this->limit = $limit; + } + + public function narrows(): bool + { + return $this->limit !== null || !$this->pathFilter instanceof OnlyFiles || $this->conjuncts !== []; + } + + public function withFilter(ScalarFunction $filter): self + { + return new self($this->limit, $this->pathFilter, [...$this->conjuncts, ...(new Conjuncts())->of($filter)]); + } + + /** + * Narrowing only: a second push may lower the limit, never raise it. + * + * @throws InvalidArgumentException when $limit is not greater than 0 + */ + public function withLimit(int $limit): self + { + return new self( + $this->limit === null ? $limit : min($this->limit, $limit), + $this->pathFilter, + $this->conjuncts, + ); + } + + public function withPathFilter(Filter $filter): self + { + return new self( + $this->limit, + $this->pathFilter instanceof Filters + ? $this->pathFilter->add($filter) + : new Filters($this->pathFilter, $filter), + $this->conjuncts, + ); + } +} diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/ListedFileRow.php b/src/core/etl/src/Flow/ETL/Extractor/File/ListedFileRow.php new file mode 100644 index 0000000000..ada03cbc48 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Extractor/File/ListedFileRow.php @@ -0,0 +1,15 @@ + the row of one listed file, its partition columns left out + */ + public function row(FileStatus $status): array; +} diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/ListingScan.php b/src/core/etl/src/Flow/ETL/Extractor/File/ListingScan.php new file mode 100644 index 0000000000..0dae904f41 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Extractor/File/ListingScan.php @@ -0,0 +1,79 @@ + $batchSize + * + * @return Generator + */ + public function scan(FlowContext $context, FileScan $scan, Schema $schema, int $batchSize): Generator + { + $backend = $context->backend(); + $filter = Filter::bind($schema, ...$scan->conjuncts); + $builder = new RowsBuilder($schema, $backend); + $yielded = 0; + $listing = (new FileListing($this->filesystem))->list($this->path, $scan->pathFilter); + + while ($listing->valid()) { + $fileStatus = type_instance_of(FileStatus::class)->assert($listing->current()); + $listing->next(); + + $builder->append([ + ...$this->row->row($fileStatus), + ...$this->fileColumns->forFile(new SourceFile($fileStatus->path, $fileStatus->size), $schema)->values(), + ]); + + if ($builder->count() < $batchSize && $listing->valid()) { + continue; + } + + $rows = $filter->filter($builder->finish(), $context); + $builder = new RowsBuilder($schema, $backend); + + if ($rows->isEmpty()) { + continue; + } + + if ($scan->limit !== null && $rows->count() > ($scan->limit - $yielded)) { + $rows = $rows->slice(0, $scan->limit - $yielded); + } + + $yielded += $rows->count(); + $signal = yield $rows; + + if ($signal === Signal::STOP) { + return; + } + + if ($scan->limit !== null && $yielded >= $scan->limit) { + return; + } + } + } +} diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/PathPartitionsRow.php b/src/core/etl/src/Flow/ETL/Extractor/File/PathPartitionsRow.php new file mode 100644 index 0000000000..b9fdf9267f --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Extractor/File/PathPartitionsRow.php @@ -0,0 +1,25 @@ + $status->path->uri(), + 'partitions' => array_merge(...array_values(array_map(static fn(Partition $p) => [ + $p->name => $p->value, + ], $status->path->partitions()->toArray()))), + ]; + } +} diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/Scanning.php b/src/core/etl/src/Flow/ETL/Extractor/File/Scanning.php new file mode 100644 index 0000000000..9d6ec43bfe --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Extractor/File/Scanning.php @@ -0,0 +1,24 @@ + + */ + public function extract(FlowContext $context, ?int $limit = null): Generator + { + return $this->scan($context, new FileScan(limit: $limit)); + } +} diff --git a/src/core/etl/src/Flow/ETL/Extractor/FilesExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/FilesExtractor.php index 41295a8a14..3924b29ebb 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/FilesExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/FilesExtractor.php @@ -8,17 +8,19 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Extractor; use Flow\ETL\Extractor\File\FileExtractor; +use Flow\ETL\Extractor\File\FileMetadataRow; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\File\ListedFiles; use Flow\ETL\Extractor\File\ListingPartitions; -use Flow\ETL\Extractor\File\SourceFile; +use Flow\ETL\Extractor\File\ListingScan; +use Flow\ETL\Extractor\File\Scanning; use Flow\ETL\FlowContext; -use Flow\ETL\Rows\RowsBuilder; +use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\Filesystem\FileListing; use Flow\Filesystem\Filesystem; use Flow\Filesystem\Local\NativeLocalFilesystem; use Flow\Filesystem\Path; -use Flow\Filesystem\Path\Filter; use Flow\Filesystem\Path\Filter\OnlyFiles; use Generator; @@ -34,6 +36,7 @@ final class FilesExtractor implements BatchableExtractor, Extractor, FileExtract private ?Statistics $statistics = null; use Batches; + use Scanning; use ListingPartitions; private readonly Filesystem $filesystem; @@ -61,56 +64,16 @@ public function isRepeatable(): bool } /** - * @return Generator + * @return Generator */ - public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator + public function scan(FlowContext $context, FileScan $scan): Generator { - $batchSize = $this->batchSize(); - $backend = $context->backend(); - $fileColumns = $this->fileColumns($this->filesystem, $this->path); - $schema = $this->schema(); - $builder = new RowsBuilder($schema, $backend); - $yielded = 0; - - foreach ((new FileListing($this->filesystem))->list($this->path, $pathFilter) as $fileStatus) { - $extension = $fileStatus->path->extension(); - - $builder->append([ - 'path' => $fileStatus->path->path(), - 'protocol' => $fileStatus->path->protocol(), - 'file_name' => $fileStatus->path->filename(), - 'base_name' => $fileStatus->path->basename(), - 'is_file' => $fileStatus->isFile(), - 'is_dir' => $fileStatus->isDirectory(), - // Path::extension() answers false for an extensionless file; the column is one - // type, so the absence is spelled null rather than a boolean in a string column. - 'extension' => $extension === false ? null : $extension, - ...$fileColumns->forFile(new SourceFile($fileStatus->path, $fileStatus->size), $schema)->values(), - ]); - - if ($builder->count() < $batchSize) { - continue; - } - - $rows = $builder->finish(); - $yielded += $rows->count(); - - $signal = yield $rows; - - if ($signal === Signal::STOP) { - return; - } - - $builder = new RowsBuilder($schema, $backend); - - if ($limit !== null && $yielded >= $limit) { - return; - } - } - - if ($builder->count() > 0) { - yield $builder->finish(); - } + return (new ListingScan( + $this->filesystem, + $this->path, + $this->fileColumns($this->filesystem, $this->path), + new FileMetadataRow(), + ))->scan($context, $scan, $this->schema(), $this->batchSize()); } public function partitionSchema(): Schema diff --git a/src/core/etl/src/Flow/ETL/Extractor/PathPartitionsExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/PathPartitionsExtractor.php index d9a9fd0220..9713a9c95a 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/PathPartitionsExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/PathPartitionsExtractor.php @@ -8,23 +8,21 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Extractor; use Flow\ETL\Extractor\File\FileExtractor; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\File\ListingPartitions; -use Flow\ETL\Extractor\File\SourceFile; +use Flow\ETL\Extractor\File\ListingScan; +use Flow\ETL\Extractor\File\PathPartitionsRow; +use Flow\ETL\Extractor\File\Scanning; use Flow\ETL\FlowContext; -use Flow\ETL\Rows\RowsBuilder; +use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\Filesystem\FileListing; use Flow\Filesystem\Filesystem; use Flow\Filesystem\Local\NativeLocalFilesystem; -use Flow\Filesystem\Partition; use Flow\Filesystem\Path; -use Flow\Filesystem\Path\Filter; use Flow\Filesystem\Path\Filter\OnlyFiles; use Generator; -use function array_map; -use function array_merge; -use function array_values; use function Flow\ETL\DSL\map_schema; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -40,6 +38,7 @@ final class PathPartitionsExtractor implements BatchableExtractor, Extractor, Fi private ?Statistics $statistics = null; use Batches; + use Scanning; use ListingPartitions; private readonly Filesystem $filesystem; @@ -67,49 +66,16 @@ public function isRepeatable(): bool } /** - * @return Generator + * @return Generator */ - public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator + public function scan(FlowContext $context, FileScan $scan): Generator { - $batchSize = $this->batchSize(); - $backend = $context->backend(); - $fileColumns = $this->fileColumns($this->filesystem, $this->path); - $schema = $this->schema(); - $builder = new RowsBuilder($schema, $backend); - $yielded = 0; - - foreach ((new FileListing($this->filesystem))->list($this->path, $pathFilter) as $fileStatus) { - $builder->append([ - 'path' => $fileStatus->path->uri(), - 'partitions' => array_merge(...array_values(array_map(static fn(Partition $p) => [ - $p->name => $p->value, - ], $fileStatus->path->partitions()->toArray()))), - ...$fileColumns->forFile(new SourceFile($fileStatus->path, $fileStatus->size), $schema)->values(), - ]); - - if ($builder->count() < $batchSize) { - continue; - } - - $rows = $builder->finish(); - $yielded += $rows->count(); - - $signal = yield $rows; - - if ($signal === Signal::STOP) { - return; - } - - $builder = new RowsBuilder($schema, $backend); - - if ($limit !== null && $yielded >= $limit) { - return; - } - } - - if ($builder->count() > 0) { - yield $builder->finish(); - } + return (new ListingScan( + $this->filesystem, + $this->path, + $this->fileColumns($this->filesystem, $this->path), + new PathPartitionsRow(), + ))->scan($context, $scan, $this->schema(), $this->batchSize()); } public function partitionSchema(): Schema diff --git a/src/core/etl/src/Flow/ETL/FlowPhpExtension.php b/src/core/etl/src/Flow/ETL/FlowPhpExtension.php index 20844015ee..0569a67795 100644 --- a/src/core/etl/src/Flow/ETL/FlowPhpExtension.php +++ b/src/core/etl/src/Flow/ETL/FlowPhpExtension.php @@ -18,7 +18,7 @@ * The contract version this package expects from flow_php, equal to the extension's FLOW_PHP_ABI. Both sides change * together whenever a class or interface the extension registers changes. */ - public const int ABI = 1; + public const int ABI = 2; public function __construct( public bool $loaded, diff --git a/src/core/etl/src/Flow/ETL/Function/Between.php b/src/core/etl/src/Flow/ETL/Function/Between.php index 89f9dd2e83..7edc5f315a 100644 --- a/src/core/etl/src/Flow/ETL/Function/Between.php +++ b/src/core/etl/src/Flow/ETL/Function/Between.php @@ -111,6 +111,7 @@ public function eval(Rows $rows, FlowContext $context): Column [$values, $lowerBounds, $upperBounds] = array_map(static fn(Column $column): array => $sameType ? $comparable->ordering($column) : $column->values(), $columns); + $order = new ValueOrder(...array_map(static fn(Column $column): Type => $column->type(), $columns)); $results = []; $i = 0; @@ -120,7 +121,7 @@ public function eval(Rows $rows, FlowContext $context): Column throw new InvalidArgumentException('Between function requires valid boundary'); } - $results[] = $boundary->compare($values[$i], $lowerBounds[$i], $upperBounds[$i]); + $results[] = $boundary->compare($values[$i], $lowerBounds[$i], $upperBounds[$i], $order); } } catch (Exception $e) { throw EvaluationException::at($i, $e); diff --git a/src/core/etl/src/Flow/ETL/Function/Between/Boundary.php b/src/core/etl/src/Flow/ETL/Function/Between/Boundary.php index 20e538645d..c7b181d281 100644 --- a/src/core/etl/src/Flow/ETL/Function/Between/Boundary.php +++ b/src/core/etl/src/Flow/ETL/Function/Between/Boundary.php @@ -4,15 +4,7 @@ namespace Flow\ETL\Function\Between; -use DateInterval; -use DateTimeImmutable; -use DateTimeInterface; -use Flow\ETL\Exception\InvalidArgumentException; - -use function is_array; -use function is_bool; -use function is_numeric; -use function is_string; +use Flow\ETL\Function\ValueOrder; enum Boundary { @@ -21,8 +13,12 @@ enum Boundary case LEFT_INCLUSIVE; case RIGHT_INCLUSIVE; - public function compare(mixed $value, mixed $lowerBound, mixed $upperbound): ?bool - { + public function compare( + mixed $value, + mixed $lowerBound, + mixed $upperbound, + ValueOrder $order = new ValueOrder(), + ): ?bool { if ($value === null) { return null; } @@ -31,8 +27,8 @@ public function compare(mixed $value, mixed $lowerBound, mixed $upperbound): ?bo if ($lowerBound !== null) { $aboveLower = match ($this) { - self::INCLUSIVE, self::LEFT_INCLUSIVE => $this->compareValues($value, $lowerBound) >= 0, - self::EXCLUSIVE, self::RIGHT_INCLUSIVE => $this->compareValues($value, $lowerBound) > 0, + self::INCLUSIVE, self::LEFT_INCLUSIVE => $order->compare($lowerBound, $value) <= 0, + self::EXCLUSIVE, self::RIGHT_INCLUSIVE => $order->compare($lowerBound, $value) < 0, }; } @@ -40,8 +36,8 @@ public function compare(mixed $value, mixed $lowerBound, mixed $upperbound): ?bo if ($upperbound !== null) { $belowUpper = match ($this) { - self::INCLUSIVE, self::RIGHT_INCLUSIVE => $this->compareValues($value, $upperbound) <= 0, - self::EXCLUSIVE, self::LEFT_INCLUSIVE => $this->compareValues($value, $upperbound) < 0, + self::INCLUSIVE, self::RIGHT_INCLUSIVE => $order->compare($value, $upperbound) <= 0, + self::EXCLUSIVE, self::LEFT_INCLUSIVE => $order->compare($value, $upperbound) < 0, }; } @@ -56,37 +52,4 @@ public function compare(mixed $value, mixed $lowerBound, mixed $upperbound): ?bo return true; } - - public function compareValues(mixed $left, mixed $right): int - { - if (is_numeric($left) && is_numeric($right)) { - return (float) $left <=> (float) $right; - } - - if (is_string($left) && is_string($right)) { - return $left <=> $right; - } - - if ($left instanceof DateTimeInterface && $right instanceof DateTimeInterface) { - return $left <=> $right; - } - - if ($left instanceof DateInterval && $right instanceof DateInterval) { - $reference = new DateTimeImmutable('@0'); - - return $reference->add($left) <=> $reference->add($right); - } - - if (is_bool($left) && is_bool($right)) { - return $left <=> $right; - } - - if (is_array($left) && is_array($right)) { - return $left <=> $right; - } - - throw new InvalidArgumentException( - 'Boundary::compare requires value and bounds to be of a comparable type (numeric, string, DateTimeInterface, DateInterval, bool, or array).', - ); - } } diff --git a/src/core/etl/src/Flow/ETL/Function/Conjuncts.php b/src/core/etl/src/Flow/ETL/Function/Conjuncts.php new file mode 100644 index 0000000000..e6a5e5dcbf --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Function/Conjuncts.php @@ -0,0 +1,28 @@ + + */ + public function of(ScalarFunction $function): array + { + if (!$function instanceof All) { + return [$function]; + } + + $conjuncts = []; + + foreach ($function->children() as $child) { + $conjuncts[] = $this->of($child); + } + + return array_merge(...$conjuncts); + } +} diff --git a/src/core/etl/src/Flow/ETL/Function/Evaluation/Filter.php b/src/core/etl/src/Flow/ETL/Function/Evaluation/Filter.php new file mode 100644 index 0000000000..ee2da8bdad --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Function/Evaluation/Filter.php @@ -0,0 +1,103 @@ + $predicates resolved against the bound schema + */ + private function __construct( + private array $predicates, + ) {} + + /** + * @throws InvalidArgumentException + */ + public static function bind(Schema $schema, ScalarFunction ...$predicates): self + { + $resolver = new ReferenceResolver(); + $resolved = []; + + foreach ($predicates as $predicate) { + $function = $resolver->resolve($predicate, $schema); + $resolver->assertResolved($function, $schema); + (new ExpandingFunctions())->refuse($function, 'filter'); + + // type_bare() keeps the gate blind to nullability - a null-propagating predicate declares + // ?boolean, and a NULL row is dropped like a false one. + if (!type_equals(type_bare($function->returns()), type_boolean())) { + throw new InvalidArgumentException(sprintf( + 'filter() requires a predicate returning boolean, "%s" returns "%s". ' + . 'Use an explicit comparison, e.g. ->notEquals(lit(0)).', + $function::class, + $function->returns()->toString(), + )); + } + + $resolved[] = $function; + } + + return new self($resolved); + } + + /** + * The rows every predicate is TRUE for: each is evaluated only on the rows the ones before it kept. + */ + public function filter(Rows $rows, FlowContext $context): Rows + { + if ($this->predicates === []) { + return $rows; + } + + $kept = $rows->isEmpty() ? [] : range(0, $rows->count() - 1); + + foreach ($this->predicates as $predicate) { + if ($kept === []) { + break; + } + + $truths = array_keys( + (new Selection($kept)) + ->evaluate($predicate, $rows, $context) + ->physicals(), + true, + true, + ); + + if (count($kept) === $rows->count()) { + $kept = $truths; + + continue; + } + + $next = []; + + foreach ($truths as $row) { + $next[] = $kept[$row]; + } + + $kept = $next; + } + + return count($kept) === $rows->count() ? $rows : $rows->gather($kept); + } +} diff --git a/src/core/etl/src/Flow/ETL/Function/GreaterThan.php b/src/core/etl/src/Flow/ETL/Function/GreaterThan.php index d87f2fd5c2..066e2cdb41 100644 --- a/src/core/etl/src/Flow/ETL/Function/GreaterThan.php +++ b/src/core/etl/src/Flow/ETL/Function/GreaterThan.php @@ -4,8 +4,6 @@ namespace Flow\ETL\Function; -use DateInterval; -use DateTimeImmutable; use Exception; use Flow\ETL\Column\Column; use Flow\ETL\Column\ComparableValues; @@ -85,6 +83,7 @@ public function eval(Rows $rows, FlowContext $context): Column [$lefts, $rights] = type_bare($leftColumn->type())::class === type_bare($rightColumn->type())::class ? [$comparable->ordering($leftColumn), $comparable->ordering($rightColumn)] : [$leftColumn->values(), $rightColumn->values()]; + $order = new ValueOrder($leftColumn->type(), $rightColumn->type()); $results = []; $i = 0; @@ -100,17 +99,7 @@ public function eval(Rows $rows, FlowContext $context): Column continue; } - // The dispatch picks a comparison strategy from the values, never a column type - bind has - // already proved the pair comparable in returns(). - if ($left instanceof DateInterval && $right instanceof DateInterval) { - $reference = new DateTimeImmutable('@0'); - $results[] = $reference->add($left) > $reference->add($right); - - continue; - } - - // @mago-expect analysis:mixed-operand(2) - $results[] = $left > $right; + $results[] = $order->compare($right, $left) < 0; } } catch (Exception $e) { throw EvaluationException::at($i, $e); diff --git a/src/core/etl/src/Flow/ETL/Function/GreaterThanEqual.php b/src/core/etl/src/Flow/ETL/Function/GreaterThanEqual.php index 41d7f760df..827371c05c 100644 --- a/src/core/etl/src/Flow/ETL/Function/GreaterThanEqual.php +++ b/src/core/etl/src/Flow/ETL/Function/GreaterThanEqual.php @@ -4,8 +4,6 @@ namespace Flow\ETL\Function; -use DateInterval; -use DateTimeImmutable; use Exception; use Flow\ETL\Column\Column; use Flow\ETL\Column\ComparableValues; @@ -85,6 +83,7 @@ public function eval(Rows $rows, FlowContext $context): Column [$lefts, $rights] = type_bare($leftColumn->type())::class === type_bare($rightColumn->type())::class ? [$comparable->ordering($leftColumn), $comparable->ordering($rightColumn)] : [$leftColumn->values(), $rightColumn->values()]; + $order = new ValueOrder($leftColumn->type(), $rightColumn->type()); $results = []; $i = 0; @@ -100,17 +99,7 @@ public function eval(Rows $rows, FlowContext $context): Column continue; } - // The dispatch picks a comparison strategy from the values, never a column type - bind has - // already proved the pair comparable in returns(). - if ($left instanceof DateInterval && $right instanceof DateInterval) { - $reference = new DateTimeImmutable('@0'); - $results[] = $reference->add($left) >= $reference->add($right); - - continue; - } - - // @mago-expect analysis:mixed-operand(2) - $results[] = $left >= $right; + $results[] = $order->compare($right, $left) <= 0; } } catch (Exception $e) { throw EvaluationException::at($i, $e); diff --git a/src/core/etl/src/Flow/ETL/Function/Greatest.php b/src/core/etl/src/Flow/ETL/Function/Greatest.php index ac6f45fa1e..abae972f82 100644 --- a/src/core/etl/src/Flow/ETL/Function/Greatest.php +++ b/src/core/etl/src/Flow/ETL/Function/Greatest.php @@ -16,7 +16,6 @@ use function array_map; use function array_values; use function Flow\ETL\DSL\lit; -use function max; final class Greatest implements ScalarFunction { @@ -73,23 +72,33 @@ public function returns(): Type public function eval(Rows $rows, FlowContext $context): Column { - $arguments = array_map(static fn(ScalarFunction $value): array => (new Parameter($value))->values( + $columns = array_map(static fn(ScalarFunction $value): Column => (new Parameter($value))->column( $rows, $context, ), $this->values); + $order = new ValueOrder(...array_map(static fn(Column $column): Type => $column->type(), $columns)); + $arguments = array_map(static fn(Column $column): array => $column->values(), $columns); $results = []; for ($i = 0, $count = $rows->count(); $i < $count; $i++) { // NULL iff every argument is NULL - nulls are skipped, never compared - $values = []; + $greatest = null; foreach ($arguments as $argument) { - if ($argument[$i] !== null) { - $values[] = $argument[$i]; + // @mago-ignore analysis:mixed-assignment + $value = $argument[$i]; + + if ($value === null) { + continue; + } + + if ($greatest === null || $order->compare($greatest, $value) < 0) { + // @mago-ignore analysis:mixed-assignment + $greatest = $value; } } - $results[] = $values === [] ? null : max($values); + $results[] = $greatest; } return (new ResultColumn($context->backend()))->of($this, $results); diff --git a/src/core/etl/src/Flow/ETL/Function/Least.php b/src/core/etl/src/Flow/ETL/Function/Least.php index c89fd37a4e..ecabdee12f 100644 --- a/src/core/etl/src/Flow/ETL/Function/Least.php +++ b/src/core/etl/src/Flow/ETL/Function/Least.php @@ -16,7 +16,6 @@ use function array_map; use function array_values; use function Flow\ETL\DSL\lit; -use function min; final class Least implements ScalarFunction { @@ -73,23 +72,33 @@ public function returns(): Type public function eval(Rows $rows, FlowContext $context): Column { - $arguments = array_map(static fn(ScalarFunction $value): array => (new Parameter($value))->values( + $columns = array_map(static fn(ScalarFunction $value): Column => (new Parameter($value))->column( $rows, $context, ), $this->values); + $order = new ValueOrder(...array_map(static fn(Column $column): Type => $column->type(), $columns)); + $arguments = array_map(static fn(Column $column): array => $column->values(), $columns); $results = []; for ($i = 0, $count = $rows->count(); $i < $count; $i++) { // NULL iff every argument is NULL - nulls are skipped, never compared - $values = []; + $least = null; foreach ($arguments as $argument) { - if ($argument[$i] !== null) { - $values[] = $argument[$i]; + // @mago-ignore analysis:mixed-assignment + $value = $argument[$i]; + + if ($value === null) { + continue; + } + + if ($least === null || $order->compare($value, $least) < 0) { + // @mago-ignore analysis:mixed-assignment + $least = $value; } } - $results[] = $values === [] ? null : min($values); + $results[] = $least; } return (new ResultColumn($context->backend()))->of($this, $results); diff --git a/src/core/etl/src/Flow/ETL/Function/LessThan.php b/src/core/etl/src/Flow/ETL/Function/LessThan.php index c644f4c1c6..57ab86434b 100644 --- a/src/core/etl/src/Flow/ETL/Function/LessThan.php +++ b/src/core/etl/src/Flow/ETL/Function/LessThan.php @@ -4,8 +4,6 @@ namespace Flow\ETL\Function; -use DateInterval; -use DateTimeImmutable; use Exception; use Flow\ETL\Column\Column; use Flow\ETL\Column\ComparableValues; @@ -85,6 +83,7 @@ public function eval(Rows $rows, FlowContext $context): Column [$lefts, $rights] = type_bare($leftColumn->type())::class === type_bare($rightColumn->type())::class ? [$comparable->ordering($leftColumn), $comparable->ordering($rightColumn)] : [$leftColumn->values(), $rightColumn->values()]; + $order = new ValueOrder($leftColumn->type(), $rightColumn->type()); $results = []; $i = 0; @@ -100,17 +99,7 @@ public function eval(Rows $rows, FlowContext $context): Column continue; } - // The dispatch picks a comparison strategy from the values, never a column type - bind has - // already proved the pair comparable in returns(). - if ($left instanceof DateInterval && $right instanceof DateInterval) { - $reference = new DateTimeImmutable('@0'); - $results[] = $reference->add($left) < $reference->add($right); - - continue; - } - - // @mago-expect analysis:mixed-operand(2) - $results[] = $left < $right; + $results[] = $order->compare($left, $right) < 0; } } catch (Exception $e) { throw EvaluationException::at($i, $e); diff --git a/src/core/etl/src/Flow/ETL/Function/LessThanEqual.php b/src/core/etl/src/Flow/ETL/Function/LessThanEqual.php index c94d5de930..589b6e392d 100644 --- a/src/core/etl/src/Flow/ETL/Function/LessThanEqual.php +++ b/src/core/etl/src/Flow/ETL/Function/LessThanEqual.php @@ -4,8 +4,6 @@ namespace Flow\ETL\Function; -use DateInterval; -use DateTimeImmutable; use Exception; use Flow\ETL\Column\Column; use Flow\ETL\Column\ComparableValues; @@ -85,6 +83,7 @@ public function eval(Rows $rows, FlowContext $context): Column [$lefts, $rights] = type_bare($leftColumn->type())::class === type_bare($rightColumn->type())::class ? [$comparable->ordering($leftColumn), $comparable->ordering($rightColumn)] : [$leftColumn->values(), $rightColumn->values()]; + $order = new ValueOrder($leftColumn->type(), $rightColumn->type()); $results = []; $i = 0; @@ -100,17 +99,7 @@ public function eval(Rows $rows, FlowContext $context): Column continue; } - // The dispatch picks a comparison strategy from the values, never a column type - bind has - // already proved the pair comparable in returns(). - if ($left instanceof DateInterval && $right instanceof DateInterval) { - $reference = new DateTimeImmutable('@0'); - $results[] = $reference->add($left) <= $reference->add($right); - - continue; - } - - // @mago-expect analysis:mixed-operand(2) - $results[] = $left <= $right; + $results[] = $order->compare($left, $right) <= 0; } } catch (Exception $e) { throw EvaluationException::at($i, $e); diff --git a/src/core/etl/src/Flow/ETL/Function/ValueOrder.php b/src/core/etl/src/Flow/ETL/Function/ValueOrder.php new file mode 100644 index 0000000000..d3eacc304a --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Function/ValueOrder.php @@ -0,0 +1,54 @@ +, so `$a < $b` is `compare($a, $b) < 0` and `$a > $b` is `compare($b, $a) < 0`, + * NaN included. + */ +final readonly class ValueOrder +{ + private bool $numericStrings; + + /** + * @param Type ...$operands + */ + public function __construct(Type ...$operands) + { + $numericStrings = false; + + foreach ($operands as $operand) { + $numericStrings = $numericStrings || type_bare($operand) instanceof NumericStringType; + } + + $this->numericStrings = $numericStrings; + } + + public function compare(mixed $left, mixed $right): int + { + if (!$this->numericStrings && is_string($left) && is_string($right)) { + return strcmp($left, $right) <=> 0; + } + + if ($left instanceof DateInterval && $right instanceof DateInterval) { + $reference = new DateTimeImmutable('@0'); + + return $reference->add($left) <=> $reference->add($right); + } + + // @mago-expect analysis:mixed-operand(2) + return $left <=> $right; + } +} diff --git a/src/core/etl/src/Flow/ETL/Optimizer.php b/src/core/etl/src/Flow/ETL/Optimizer.php index 909def5325..bbb8c6103f 100644 --- a/src/core/etl/src/Flow/ETL/Optimizer.php +++ b/src/core/etl/src/Flow/ETL/Optimizer.php @@ -37,8 +37,8 @@ public static function default(): self return new self( new CombineLimits(), new CombineSortAndLimit(), - new PushLimitIntoSource(), new PushFilterIntoSource(), + new PushLimitIntoSource(), new CountFromStatistics(), new PushProjectionIntoExpand(), ); diff --git a/src/core/etl/src/Flow/ETL/Optimizer/FilterWalk.php b/src/core/etl/src/Flow/ETL/Optimizer/FilterWalk.php index 098733d12f..016d69e024 100644 --- a/src/core/etl/src/Flow/ETL/Optimizer/FilterWalk.php +++ b/src/core/etl/src/Flow/ETL/Optimizer/FilterWalk.php @@ -35,9 +35,9 @@ public function reachedByEveryConsumer(Node\Filter $filter, Node ...$consumerInp /** * $predicate as $leaf sees it, or null when it may not be evaluated there instead of above $filter. - * Another Filter is TRANSPARENT here: it only removes rows, and the pushed Filter node stays, so pruning a - * file drops rows that filter would have dropped anyway. Limit/Offset/Until/Distinct/Discard are not: they - * make WHICH rows arrive depend on what the source read. + * Another Filter is TRANSPARENT here: it only removes rows, so the order two filters drop rows in does not change + * which rows survive both. Limit/Offset/Until/Distinct/Discard are not: they make WHICH rows arrive depend on + * what the source read. * * A node that redefines a column the predicate reads blocks it, unless the column is a plain alias of a * column below (Rename, a WithColumn of a bare reference): the predicate is rewritten to read that column. diff --git a/src/core/etl/src/Flow/ETL/Optimizer/JoinSides.php b/src/core/etl/src/Flow/ETL/Optimizer/JoinSides.php index 2eea368fbb..26ac44d21e 100644 --- a/src/core/etl/src/Flow/ETL/Optimizer/JoinSides.php +++ b/src/core/etl/src/Flow/ETL/Optimizer/JoinSides.php @@ -21,15 +21,15 @@ public function __construct( private FlowContext $context, ) {} - public function of(Node $node): Node + public function of(Node $original, Node $rebuilt): Node { - if (!$node instanceof JoinsFrame) { - return $node; + if (!$rebuilt instanceof JoinsFrame) { + return $rebuilt; } - return $node->withChildren([ - $node->children()[0], - $this->optimizer->optimize(new LogicalPlan($node->right()), $this->context)->root, + return $rebuilt->withChildren([ + $rebuilt->children()[0], + $this->optimizer->optimize(new LogicalPlan($rebuilt->right()), $this->context)->root, ]); } } diff --git a/src/core/etl/src/Flow/ETL/Optimizer/Rule/CombineLimits.php b/src/core/etl/src/Flow/ETL/Optimizer/Rule/CombineLimits.php index 6837db8868..a9d893143e 100644 --- a/src/core/etl/src/Flow/ETL/Optimizer/Rule/CombineLimits.php +++ b/src/core/etl/src/Flow/ETL/Optimizer/Rule/CombineLimits.php @@ -23,14 +23,16 @@ public function apply(LogicalPlan $plan, FlowContext $context): LogicalPlan return $plan->transformUp($this); } - public function of(Node $node): Node + public function of(Node $original, Node $rebuilt): Node { - if (!$node instanceof Limit) { - return $node; + if (!$rebuilt instanceof Limit) { + return $rebuilt; } - $child = $node->children()[0]; + $child = $rebuilt->children()[0]; - return $child instanceof Limit ? new Limit($child->children()[0], min($node->limit, $child->limit)) : $node; + return $child instanceof Limit + ? new Limit($child->children()[0], min($rebuilt->limit, $child->limit)) + : $rebuilt; } } diff --git a/src/core/etl/src/Flow/ETL/Optimizer/Rule/CountFromStatistics.php b/src/core/etl/src/Flow/ETL/Optimizer/Rule/CountFromStatistics.php index 6ab5bce034..ce90b859d6 100644 --- a/src/core/etl/src/Flow/ETL/Optimizer/Rule/CountFromStatistics.php +++ b/src/core/etl/src/Flow/ETL/Optimizer/Rule/CountFromStatistics.php @@ -11,7 +11,6 @@ use Flow\ETL\Plan\Node\Read; use Flow\ETL\Plan\Node\Result; use Flow\ETL\Processor\CountingProcessor; -use Flow\Filesystem\Path\Filter\OnlyFiles; use function Flow\ETL\DSL\from_rows; @@ -22,7 +21,7 @@ public function apply(LogicalPlan $plan, FlowContext $context): LogicalPlan $count = $plan->root instanceof Result ? $plan->root->children()[0] : null; $read = $count instanceof Count ? $count->children()[0] : null; - if (!$read instanceof Read || $read->limit() !== null || !$read->pathFilter() instanceof OnlyFiles) { + if (!$read instanceof Read || $read->scan()->narrows()) { return $plan; } diff --git a/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushFilterIntoSource.php b/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushFilterIntoSource.php index 3d597af875..a696fdd9ca 100644 --- a/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushFilterIntoSource.php +++ b/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushFilterIntoSource.php @@ -8,19 +8,24 @@ use Flow\ETL\Filesystem\ScalarFunctionFilter; use Flow\ETL\FlowContext; use Flow\ETL\Function\All; +use Flow\ETL\Function\Conjuncts; use Flow\ETL\Function\ReferencedColumns; use Flow\ETL\Optimizer\FilterWalk; use Flow\ETL\Optimizer\Rule; use Flow\ETL\Plan\LogicalPlan; use Flow\ETL\Plan\Node; -use Flow\ETL\Plan\ReplaceLeaf; +use Flow\ETL\Plan\ReplaceNodes; +use SplObjectStorage; use function array_diff; +use function array_reverse; +use function count; /** - * Evaluate a partition-only predicate once per FILE instead of once per row: the file never opens. The Filter node - * stays, so a source that lists more than the pushed filter admits still returns the right rows. The parts of an AND - * that read only partition columns are pushed one by one; an OR is pushed only as a whole. + * Moves every deterministic part of an AND into the file source, which returns only the rows matching it; a + * partition-only part is also pushed as a path filter, so a file it rejects is never listed. A pushed part leaves its + * Filter node, and a node left with no part is removed. An OR is pushed only as a whole. Nodes are pushed leaf first; + * a part kept in a node stops the push of every part after it and of every node above it. */ final readonly class PushFilterIntoSource implements Rule { @@ -34,43 +39,70 @@ public function apply(LogicalPlan $plan, FlowContext $context): LogicalPlan } $partitions = $extractor->partitionSchema(); - - if ($partitions->count() === 0) { - return $plan; - } - $consumerInputs = $plan->consumerInputs(); $walk = new FilterWalk(); $columns = new ReferencedColumns(); + $conjuncts = new Conjuncts(); $names = $partitions->references()->names(); $read = $leaf; + /** @var SplObjectStorage $replacements */ + $replacements = new SplObjectStorage(); + $removals = []; + $filters = []; for ($node = $plan->root; $node->children() !== []; $node = $node->children()[0]) { - if (!$node instanceof Node\Filter || !$walk->reachedByEveryConsumer($node, ...$consumerInputs)) { - continue; + if ($node instanceof Node\Filter && $walk->reachedByEveryConsumer($node, ...$consumerInputs)) { + $filters[] = $node; } + } - foreach ($node->function instanceof All ? $node->function->children() : [$node->function] as $conjunct) { - if (!$conjunct->deterministic()) { - continue; - } + // leaf first, so the source evaluates a lower filter before the one above it relies on what it removed + foreach (array_reverse($filters) as $filter) { + $parts = $conjuncts->of($filter->function); + $kept = []; - $atLeaf = $walk->predicateAtLeaf($node, $leaf, $conjunct); + foreach ($parts as $conjunct) { + // a conjunct after a kept one would be evaluated on rows the kept one removes + $atLeaf = $kept === [] && $conjunct->deterministic() + ? $walk->predicateAtLeaf($filter, $leaf, $conjunct) + : null; if ($atLeaf === null) { + $kept[] = $conjunct; + continue; } - $refs = $columns->in($atLeaf); + $referenced = $columns->in($atLeaf)->names(); - if ($refs->all() === [] || array_diff($refs->names(), $names) !== []) { - continue; + if ($referenced !== [] && array_diff($referenced, $names) === []) { + $read = $read->withPathFilter(new ScalarFunctionFilter($atLeaf, $partitions, $context)); } - $read = $read->withPathFilter(new ScalarFunctionFilter($atLeaf, $partitions, $context)); + $read = $read->withFilter($atLeaf); + } + + if ($kept === []) { + $removals[] = $filter; + } elseif (count($kept) !== count($parts)) { + $replacements[$filter] = new Node\Filter( + $filter->children()[0], + count($kept) === 1 ? $kept[0] : new All(...$kept), + ); } + + // a filter above a kept conjunct would be evaluated on rows that conjunct removes + if ($kept !== []) { + break; + } + } + + if ($read !== $leaf) { + $replacements[$leaf] = $read; } - return $read === $leaf ? $plan : $plan->transformUp(new ReplaceLeaf($leaf, $read)); + return $replacements->count() === 0 && $removals === [] + ? $plan + : $plan->transformUp(new ReplaceNodes($replacements, $removals)); } } diff --git a/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushLimitIntoSource.php b/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushLimitIntoSource.php index 36beb317b9..70edda1588 100644 --- a/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushLimitIntoSource.php +++ b/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushLimitIntoSource.php @@ -8,7 +8,7 @@ use Flow\ETL\Optimizer\LimitWalk; use Flow\ETL\Optimizer\Rule; use Flow\ETL\Plan\LogicalPlan; -use Flow\ETL\Plan\ReplaceLeaf; +use Flow\ETL\Plan\ReplaceNodes; use function max; @@ -34,6 +34,6 @@ public function apply(LogicalPlan $plan, FlowContext $context): LogicalPlan $limits[] = $limit; } - return $plan->transformUp(new ReplaceLeaf($leaf, $leaf->withLimit(max($limits)))); + return $plan->transformUp(ReplaceNodes::one($leaf, $leaf->withLimit(max($limits)))); } } diff --git a/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushProjectionIntoExpand.php b/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushProjectionIntoExpand.php index 4a32d1fd81..cf91544685 100644 --- a/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushProjectionIntoExpand.php +++ b/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushProjectionIntoExpand.php @@ -9,13 +9,10 @@ use Flow\ETL\Optimizer\Rule; use Flow\ETL\Plan\LogicalPlan; use Flow\ETL\Plan\Node; -use Flow\ETL\Plan\ReplaceLeaf; +use Flow\ETL\Plan\ReplaceNodes; use Flow\ETL\Plan\RequiredColumns; use SplObjectStorage; -use function arsort; -use function spl_object_id; - /** * Columns no consumer reads are not copied into the rows an expand emits (Spark: ColumnPruning setting * Generate.unrequiredChildIndex). @@ -40,33 +37,15 @@ public function apply(LogicalPlan $plan, FlowContext $context): LogicalPlan } } - $depths = []; - $targets = []; + /** @var SplObjectStorage $replacements */ + $replacements = new SplObjectStorage(); foreach ($demands as $node) { - if ($demands[$node]->isAll()) { - continue; - } - - $id = spl_object_id($node); - $targets[$id] = $node; - $depths[$id] = 0; - - for ($below = $node; $below->children() !== []; $below = $below->children()[0]) { - $depths[$id]++; + if (!$demands[$node]->isAll()) { + $replacements[$node] = $node->withCarries($demands[$node]); } } - // TransformUp hands Rewrite::of() the node rebuilt over its new children, so a node whose descendant was - // replaced first would no longer be the target; the one farthest from the leaf goes first - arsort($depths); - - foreach ($depths as $id => $_) { - $plan = $plan->transformUp( - new ReplaceLeaf($targets[$id], $targets[$id]->withCarries($demands[$targets[$id]])), - ); - } - - return $plan; + return $replacements->count() === 0 ? $plan : $plan->transformUp(new ReplaceNodes($replacements)); } } diff --git a/src/core/etl/src/Flow/ETL/Optimizer/TopNRewrite.php b/src/core/etl/src/Flow/ETL/Optimizer/TopNRewrite.php index 2beb7e7720..af11e63585 100644 --- a/src/core/etl/src/Flow/ETL/Optimizer/TopNRewrite.php +++ b/src/core/etl/src/Flow/ETL/Optimizer/TopNRewrite.php @@ -15,19 +15,19 @@ */ final readonly class TopNRewrite implements Rewrite { - public function of(Node $node): Node + public function of(Node $original, Node $rebuilt): Node { - if (!$node instanceof Limit) { - return $node; + if (!$rebuilt instanceof Limit) { + return $rebuilt; } - $sort = $node->children()[0]; + $sort = $rebuilt->children()[0]; if (!$sort instanceof Sort) { - return $node; + return $rebuilt; } // a TopN past its memory limit falls back to the sort's own spilling path, so any limit is safe to rewrite - return new TopN($sort->children()[0], $sort->refs, $node->limit, $sort->algorithm); + return new TopN($sort->children()[0], $sort->refs, $rebuilt->limit, $sort->algorithm); } } diff --git a/src/core/etl/src/Flow/ETL/Plan/Explain/Details.php b/src/core/etl/src/Flow/ETL/Plan/Explain/Details.php index a6815e6530..a28edc17f3 100644 --- a/src/core/etl/src/Flow/ETL/Plan/Explain/Details.php +++ b/src/core/etl/src/Flow/ETL/Plan/Explain/Details.php @@ -5,6 +5,7 @@ namespace Flow\ETL\Plan\Explain; use Flow\ETL\Extractor\File\FileExtractor; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Join\Comparison\Any; use Flow\ETL\Plan\Materialization; use Flow\ETL\Plan\Node; @@ -80,7 +81,7 @@ public function labelled(Node $node): array $node instanceof Offset => ['Skip: ' . $node->offset], $node instanceof TopN => ['Top: ' . $node->limit], $node instanceof Rename => [sprintf('Rename: %s → %s', $node->from, $node->to)], - $node instanceof Select => ['Columns: ' . $this->columns($node->entries)], + $node instanceof Select => ['Keeps only: ' . $this->columns($node->entries)], $node instanceof Drop => ['Drops: ' . $this->columns($node->entries)], $node instanceof Distinct => [ 'Distinct on: ' . ($node->entries === [] ? 'every column' : $this->columns($node->entries)), @@ -206,15 +207,29 @@ public function read(Read $read): array $lines[] = 'Source: ' . $extractor->source()->uri(); } - $limit = $read->limit(); + return [...$lines, ...$this->scan($read->scan())]; + } + + /** + * What the plan pushed into a read, each conjunct the source applies on its own line. + * + * @return list + */ + public function scan(FileScan $scan): array + { + $lines = []; - if ($limit !== null) { - $lines[] = 'Limit: ' . $limit; + if ($scan->limit !== null) { + $lines[] = 'Limit: ' . $scan->limit; } // every file source reads only files unless a filter was pushed into it - if (!$read->pathFilter() instanceof OnlyFiles) { - $lines[] = 'Files: ' . $this->name($read->pathFilter()); + if (!$scan->pathFilter instanceof OnlyFiles) { + $lines[] = 'Files: ' . $this->name($scan->pathFilter); + } + + foreach ($scan->conjuncts as $conjunct) { + $lines[] = 'Filter: ' . $this->name($conjunct); } return $lines; @@ -234,10 +249,10 @@ public function notes(Node $node): array $notes[] = 'Buffers all rows before passing them on'; } - if ($redefines->unknown) { - $notes[] = 'Defines columns known only at run time'; - } elseif ($redefines->names !== []) { - $notes[] = 'Defines columns: ' . implode(', ', $redefines->names); + if ($redefines->isUnknown()) { + $notes[] = 'Changes columns known only at run time'; + } elseif (!$redefines->isAllBut() && $redefines->names !== []) { + $notes[] = 'Changes columns: ' . implode(', ', $redefines->names); } return $notes; @@ -255,8 +270,10 @@ public function declarations(Node $node): string $node->materialization()->name, ]; - if ($redefines->unknown) { + if ($redefines->isUnknown()) { $declarations[] = 'redefines unknown'; + } elseif ($redefines->isAllBut()) { + $declarations[] = 'redefines all but ' . implode(', ', $redefines->names); } elseif ($redefines->names !== []) { $declarations[] = 'redefines ' . implode(', ', $redefines->names); } diff --git a/src/core/etl/src/Flow/ETL/Plan/Explain/PhysicalOutline.php b/src/core/etl/src/Flow/ETL/Plan/Explain/PhysicalOutline.php index cea5a9af82..3c893d8e6e 100644 --- a/src/core/etl/src/Flow/ETL/Plan/Explain/PhysicalOutline.php +++ b/src/core/etl/src/Flow/ETL/Plan/Explain/PhysicalOutline.php @@ -9,8 +9,8 @@ use Flow\ETL\Executor\Pipeline; use Flow\ETL\Processor\HashJoinProcessor; use Flow\ETL\Transformer\CrossJoinRowsTransformer; -use Flow\Filesystem\Path\Filter\OnlyFiles; +use function array_map; use function implode; final readonly class PhysicalOutline @@ -94,18 +94,7 @@ public function joined(PhysicalPlan $right): Entry */ public function source(Pipeline $pipeline): array { - $lines = []; - $limit = $pipeline->limit(); - - if ($limit !== null) { - $lines[] = ' Limit: ' . $limit; - } - - if (!$pipeline->pathFilter() instanceof OnlyFiles) { - $lines[] = ' Files: ' . $this->details->name($pipeline->pathFilter()); - } - - return $lines; + return array_map(static fn(string $line): string => ' ' . $line, $this->details->scan($pipeline->scan())); } /** diff --git a/src/core/etl/src/Flow/ETL/Plan/Node.php b/src/core/etl/src/Flow/ETL/Plan/Node.php index 9b2e4d0129..e9d0a3c87d 100644 --- a/src/core/etl/src/Flow/ETL/Plan/Node.php +++ b/src/core/etl/src/Flow/ETL/Plan/Node.php @@ -54,7 +54,7 @@ public function transparency(): Transparency; public function materialization(): Materialization; /** - * Columns this node introduces or renames on its output. A predicate that references one of them means a + * Columns this node introduces, renames or removes on its output. A predicate that references one of them means a * different column below this node, so it cannot be pushed past it. Answered from what the node holds, * never from a class list. */ diff --git a/src/core/etl/src/Flow/ETL/Plan/Node/Drop.php b/src/core/etl/src/Flow/ETL/Plan/Node/Drop.php index 0b3360c587..e35cac6df4 100644 --- a/src/core/etl/src/Flow/ETL/Plan/Node/Drop.php +++ b/src/core/etl/src/Flow/ETL/Plan/Node/Drop.php @@ -10,6 +10,7 @@ use Flow\ETL\Plan\RowCount; use Flow\ETL\Plan\Transparency; use Flow\ETL\Row\Reference; +use Flow\ETL\Row\References; final readonly class Drop implements Node { @@ -51,6 +52,6 @@ public function materialization(): Materialization public function redefines(): Redefined { - return Redefined::none(); + return Redefined::names(...References::init(...$this->entries)->names()); } } diff --git a/src/core/etl/src/Flow/ETL/Plan/Node/Read.php b/src/core/etl/src/Flow/ETL/Plan/Node/Read.php index f94adb446e..5a3de95d3c 100644 --- a/src/core/etl/src/Flow/ETL/Plan/Node/Read.php +++ b/src/core/etl/src/Flow/ETL/Plan/Node/Read.php @@ -7,6 +7,9 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\Extractor; +use Flow\ETL\Extractor\File\FileExtractor; +use Flow\ETL\Extractor\File\FileScan; +use Flow\ETL\Function\ScalarFunction; use Flow\ETL\Plan\Materialization; use Flow\ETL\Plan\Node; use Flow\ETL\Plan\Redefined; @@ -14,22 +17,17 @@ use Flow\ETL\Plan\Transparency; use Flow\ETL\Schema; use Flow\Filesystem\Path\Filter; -use Flow\Filesystem\Path\Filter\Filters; -use Flow\Filesystem\Path\Filter\OnlyFiles; - -use function min; final readonly class Read implements Node { /** * @param Extractor $extractor the instance the user handed to read(); the planner never mutates or copies it - * @param null|int<1, max> $limit the pushed limit, handed to Extractor::extract() when the plan runs - * @param Filter $pathFilter the pushed partition filter, handed to FileExtractor::extract() when the plan runs + * @param FileScan $scan what the plan pushed into the read, handed to FileExtractor::scan() when the plan + * runs; a non-file extractor receives only its limit */ public function __construct( private Extractor $extractor, - private ?int $limit = null, - private Filter $pathFilter = new OnlyFiles(), + private FileScan $scan = new FileScan(), ) {} public function extractor(): Extractor @@ -37,28 +35,17 @@ public function extractor(): Extractor return $this->extractor; } - /** - * @return null|int<1, max> - */ - public function limit(): ?int + public function scan(): FileScan { - return $this->limit; + return $this->scan; } - public function pathFilter(): Filter - { - return $this->pathFilter; - } - - public function withPathFilter(Filter $filter): self + /** + * @throws InvalidArgumentException when the extractor is no FileExtractor, which has nowhere to apply it + */ + public function withFilter(ScalarFunction $filter): self { - return new self( - $this->extractor, - $this->limit, - $this->pathFilter instanceof Filters - ? $this->pathFilter->add($filter) - : new Filters($this->pathFilter, $filter), - ); + return new self($this->fileExtractor(), $this->scan->withFilter($filter)); } /** @@ -68,15 +55,15 @@ public function withPathFilter(Filter $filter): self */ public function withLimit(int $limit): self { - if ($limit <= 0) { - throw new InvalidArgumentException('Limit must be greater than 0'); - } - - return new self( - $this->extractor, - $this->limit === null ? $limit : min($this->limit, $limit), - $this->pathFilter, - ); + return new self($this->extractor, $this->scan->withLimit($limit)); + } + + /** + * @throws InvalidArgumentException when the extractor is no FileExtractor, which has nowhere to apply it + */ + public function withPathFilter(Filter $filter): self + { + return new self($this->fileExtractor(), $this->scan->withPathFilter($filter)); } /** @@ -119,4 +106,13 @@ public function redefines(): Redefined { return Redefined::none(); } + + private function fileExtractor(): FileExtractor + { + return $this->extractor instanceof FileExtractor + ? $this->extractor + : throw new InvalidArgumentException( + 'Only a FileExtractor reads a pushed filter, ' . $this->extractor::class . ' does not', + ); + } } diff --git a/src/core/etl/src/Flow/ETL/Plan/Node/Rename.php b/src/core/etl/src/Flow/ETL/Plan/Node/Rename.php index c3f29e3ee8..c7806345eb 100644 --- a/src/core/etl/src/Flow/ETL/Plan/Node/Rename.php +++ b/src/core/etl/src/Flow/ETL/Plan/Node/Rename.php @@ -48,6 +48,6 @@ public function materialization(): Materialization public function redefines(): Redefined { - return Redefined::alias($this->to, $this->from); + return Redefined::rename($this->from, $this->to); } } diff --git a/src/core/etl/src/Flow/ETL/Plan/Node/Select.php b/src/core/etl/src/Flow/ETL/Plan/Node/Select.php index 0221ceca07..c651bf7a17 100644 --- a/src/core/etl/src/Flow/ETL/Plan/Node/Select.php +++ b/src/core/etl/src/Flow/ETL/Plan/Node/Select.php @@ -10,6 +10,7 @@ use Flow\ETL\Plan\RowCount; use Flow\ETL\Plan\Transparency; use Flow\ETL\Row\Reference; +use Flow\ETL\Row\References; final readonly class Select implements Node { @@ -51,6 +52,6 @@ public function materialization(): Materialization public function redefines(): Redefined { - return Redefined::none(); + return Redefined::allBut(...References::init(...$this->entries)->names()); } } diff --git a/src/core/etl/src/Flow/ETL/Plan/Redefined.php b/src/core/etl/src/Flow/ETL/Plan/Redefined.php index b64220a352..4eafeeacb7 100644 --- a/src/core/etl/src/Flow/ETL/Plan/Redefined.php +++ b/src/core/etl/src/Flow/ETL/Plan/Redefined.php @@ -15,19 +15,27 @@ * @param array $aliases a redefined name => the column below the node it only renames */ private function __construct( - public bool $unknown, + private Redefinition $redefinition, public array $names, private array $aliases, ) {} public static function none(): self { - return new self(false, [], []); + return new self(Redefinition::names, [], []); } public static function names(string ...$names): self { - return new self(false, array_values($names), []); + return new self(Redefinition::names, array_values($names), []); + } + + /** + * The node keeps only $kept (Select): every other column is gone above it. + */ + public static function allBut(string ...$kept): self + { + return new self(Redefinition::allBut, array_values($kept), []); } /** @@ -36,7 +44,15 @@ public static function names(string ...$names): self */ public static function alias(string $name, string $below): self { - return new self(false, [$name], [$name => $below]); + return new self(Redefinition::names, [$name], [$name => $below]); + } + + /** + * $to holds exactly the values of $from, which is gone above the node. + */ + public static function rename(string $from, string $to): self + { + return new self(Redefinition::names, [$to, $from], [$to => $from]); } /** @@ -44,12 +60,26 @@ public static function alias(string $name, string $below): self */ public static function unknown(): self { - return new self(true, [], []); + return new self(Redefinition::unknown, [], []); } public function defines(string $name): bool { - return $this->unknown || in_array($name, $this->names, true); + return match ($this->redefinition) { + Redefinition::unknown => true, + Redefinition::allBut => !in_array($name, $this->names, true), + Redefinition::names => in_array($name, $this->names, true), + }; + } + + public function isAllBut(): bool + { + return $this->redefinition === Redefinition::allBut; + } + + public function isUnknown(): bool + { + return $this->redefinition === Redefinition::unknown; } /** diff --git a/src/core/etl/src/Flow/ETL/Plan/Redefinition.php b/src/core/etl/src/Flow/ETL/Plan/Redefinition.php new file mode 100644 index 0000000000..51dedd4f57 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Plan/Redefinition.php @@ -0,0 +1,12 @@ +target ? $this->replacement : $node; - } -} diff --git a/src/core/etl/src/Flow/ETL/Plan/ReplaceNodes.php b/src/core/etl/src/Flow/ETL/Plan/ReplaceNodes.php new file mode 100644 index 0000000000..2074e2e685 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Plan/ReplaceNodes.php @@ -0,0 +1,59 @@ + $replacements target => replacement, which takes the target's rewritten + * children + * @param list $removals nodes of one child, each giving way to its rewritten child + */ + public function __construct( + private SplObjectStorage $replacements, + private array $removals = [], + ) { + foreach ($removals as $removal) { + if ($replacements->offsetExists($removal)) { + throw new InvalidArgumentException( + 'A node is either replaced or removed, ' . $removal::class . ' is both', + ); + } + } + } + + public static function one(Node $target, Node $replacement): self + { + /** @var SplObjectStorage $replacements */ + $replacements = new SplObjectStorage(); + $replacements[$target] = $replacement; + + return new self($replacements); + } + + public function of(Node $original, Node $rebuilt): Node + { + if (in_array($original, $this->removals, true)) { + return count($rebuilt->children()) === 1 + ? $rebuilt->children()[0] + : throw new InvalidArgumentException( + 'Only a node of one child gives way to it, ' . $original::class . ' has ' + . count($rebuilt->children()), + ); + } + + return $this->replacements->offsetExists($original) + ? $this->replacements[$original]->withChildren($rebuilt->children()) + : $rebuilt; + } +} diff --git a/src/core/etl/src/Flow/ETL/Plan/Rewrite.php b/src/core/etl/src/Flow/ETL/Plan/Rewrite.php index 5b60893224..b466330d12 100644 --- a/src/core/etl/src/Flow/ETL/Plan/Rewrite.php +++ b/src/core/etl/src/Flow/ETL/Plan/Rewrite.php @@ -7,10 +7,11 @@ interface Rewrite { /** - * The replacement for this node, or the node itself when nothing changes. + * The replacement for $original, given as $rebuilt over its rewritten children; $rebuilt itself when nothing + * changes. * * Apply it to a plan through LogicalPlan::transformUp(): its memo rewrites a node every consumer shares * exactly once, so the shared prefix stays shared. It never reaches into a join's right side. */ - public function of(Node $node): Node; + public function of(Node $original, Node $rebuilt): Node; } diff --git a/src/core/etl/src/Flow/ETL/Plan/TransformUp.php b/src/core/etl/src/Flow/ETL/Plan/TransformUp.php index 2a674b80c7..f3d2db87ca 100644 --- a/src/core/etl/src/Flow/ETL/Plan/TransformUp.php +++ b/src/core/etl/src/Flow/ETL/Plan/TransformUp.php @@ -41,6 +41,6 @@ public function of(Node $node, Rewrite $rewrite): Node $children[$i] = $this->of($children[$i], $rewrite); } - return $this->memo[$node] = $rewrite->of($node->withChildren($children)); + return $this->memo[$node] = $rewrite->of($node, $node->withChildren($children)); } } diff --git a/src/core/etl/src/Flow/ETL/Planner/PipelineSplit.php b/src/core/etl/src/Flow/ETL/Planner/PipelineSplit.php index fb5645b886..fcd00eb8d8 100644 --- a/src/core/etl/src/Flow/ETL/Planner/PipelineSplit.php +++ b/src/core/etl/src/Flow/ETL/Planner/PipelineSplit.php @@ -11,11 +11,11 @@ use Flow\ETL\Executor\Raw; use Flow\ETL\Executor\Segments; use Flow\ETL\Executor\SourceRows; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\FlowContext; use Flow\ETL\Plan\LogicalPlan; use Flow\ETL\Plan\Materialization; use Flow\ETL\Plan\Node; -use Flow\Filesystem\Path\Filter\OnlyFiles; use SplObjectStorage; use function array_reverse; @@ -59,8 +59,7 @@ public function of( $read = $logical->source(); $input = null; $segments = new Segments($read->extractor()); - $limit = $read->limit(); - $pathFilter = $read->pathFilter(); + $scan = $read->scan(); $attachment = new SinkAttachment($planned, $context); $remembered = $attachment->attach($logical->sinksOnSpine(), $onSpine); @@ -76,15 +75,14 @@ public function of( } if ($node->materialization() === Materialization::blocking && $node !== $top) { - $input = new Pipeline($id++, $segments, $context, $input, $limit, $pathFilter, $sources); + $input = new Pipeline($id++, $segments, $context, $input, $scan, $sources); $segments = new Segments(); - $limit = null; - $pathFilter = new OnlyFiles(); + $scan = new FileScan(); $sources = null; } } - $pipeline = new Pipeline($id, $segments, $context, $input, $limit, $pathFilter, $sources); + $pipeline = new Pipeline($id, $segments, $context, $input, $scan, $sources); $refusal = $planned->refusal(); return $refusal === null diff --git a/src/core/etl/src/Flow/ETL/Schema/Definition/StringDefinition.php b/src/core/etl/src/Flow/ETL/Schema/Definition/StringDefinition.php index e17bb917cd..8a2e04dce3 100644 --- a/src/core/etl/src/Flow/ETL/Schema/Definition/StringDefinition.php +++ b/src/core/etl/src/Flow/ETL/Schema/Definition/StringDefinition.php @@ -29,14 +29,18 @@ */ private Type $type; + /** + * @param null|Type $type + */ public function __construct( string|Reference $ref, private bool $nullable = false, ?Metadata $metadata = null, + ?Type $type = null, ) { $this->ref = UnresolvedReference::init($ref); $this->metadata = $metadata ?? Metadata::empty(); - $this->type = type_string(); + $this->type = $type ?? type_string(); } /** @@ -44,7 +48,7 @@ public function __construct( */ public function addMetadata(string $key, int|string|bool|float|array $value): static { - return new self($this->ref, $this->nullable, $this->metadata->add($key, $value)); + return new self($this->ref, $this->nullable, $this->metadata->add($key, $value), $this->type); } public function entry(): Reference @@ -85,7 +89,7 @@ public function isSame(Definition $definition): bool public function makeNullable(bool $nullable = true): static { - return new self($this->ref, $nullable, $this->metadata); + return new self($this->ref, $nullable, $this->metadata, $this->type); } public function matches(mixed $value): bool @@ -111,6 +115,7 @@ public function merge(Definition $definition): Definition $this->ref, $this->nullable || $definition->isNullable(), $this->metadata->merge($definition->metadata()), + type_equals($this->type, $definition->type()) ? $this->type : type_string(), ); } @@ -134,12 +139,12 @@ public function normalize(): array public function rename(string $newName): static { - return new self($newName, $this->nullable, $this->metadata); + return new self($newName, $this->nullable, $this->metadata, $this->type); } public function setMetadata(Metadata $metadata): static { - return new self($this->ref, $this->nullable, $metadata); + return new self($this->ref, $this->nullable, $metadata, $this->type); } public function type(): Type diff --git a/src/core/etl/src/Flow/ETL/Schema/Formatter/PHPFormatter/TypeFormatter.php b/src/core/etl/src/Flow/ETL/Schema/Formatter/PHPFormatter/TypeFormatter.php index 3be4378cc4..2026001445 100644 --- a/src/core/etl/src/Flow/ETL/Schema/Formatter/PHPFormatter/TypeFormatter.php +++ b/src/core/etl/src/Flow/ETL/Schema/Formatter/PHPFormatter/TypeFormatter.php @@ -12,6 +12,7 @@ use Flow\Types\Type\Logical\JsonType; use Flow\Types\Type\Logical\ListType; use Flow\Types\Type\Logical\MapType; +use Flow\Types\Type\Logical\NumericStringType; use Flow\Types\Type\Logical\OptionalType; use Flow\Types\Type\Logical\StructureType; use Flow\Types\Type\Logical\TimeType; @@ -109,6 +110,7 @@ private function formatSimpleType(Type $type, bool $nullable): string $reflection = match ($type::class) { ArrayType::class => new ReflectionFunction('\\Flow\\Types\\DSL\\type_array'), StringType::class => new ReflectionFunction('\\Flow\\Types\\DSL\\type_string'), + NumericStringType::class => new ReflectionFunction('\\Flow\\Types\\DSL\\type_numeric_string'), IntegerType::class => new ReflectionFunction('\\Flow\\Types\\DSL\\type_integer'), BooleanType::class => new ReflectionFunction('\\Flow\\Types\\DSL\\type_boolean'), FloatType::class => new ReflectionFunction('\\Flow\\Types\\DSL\\type_float'), diff --git a/src/core/etl/src/Flow/ETL/Schema/Formatter/PHPSchemaFormatter.php b/src/core/etl/src/Flow/ETL/Schema/Formatter/PHPSchemaFormatter.php index ee33dae1eb..2ba94537fd 100644 --- a/src/core/etl/src/Flow/ETL/Schema/Formatter/PHPSchemaFormatter.php +++ b/src/core/etl/src/Flow/ETL/Schema/Formatter/PHPSchemaFormatter.php @@ -19,6 +19,7 @@ use Flow\Types\Type\Logical\JsonType; use Flow\Types\Type\Logical\ListType; use Flow\Types\Type\Logical\MapType; +use Flow\Types\Type\Logical\NumericStringType; use Flow\Types\Type\Logical\StructureType; use Flow\Types\Type\Logical\TimeType; use Flow\Types\Type\Logical\TimeZoneType; @@ -152,6 +153,7 @@ private function formatSchema(Schema $schema, int $level = 1): string DateTimeType::class => $this->dateTimeType($definition), FloatType::class => $this->floatType($definition), EnumType::class => $this->enumType($definition), + NumericStringType::class => $this->numericStringType($definition), ListType::class => $this->listType($definition), MapType::class => $this->mapType($definition), StructureType::class => $this->structureType($definition), @@ -201,6 +203,23 @@ private function mapType(Definition $definition): string ); } + /** + * @param Definition $definition + */ + private function numericStringType(Definition $definition): string + { + $reflection = new ReflectionFunction("\Flow\ETL\DSL\\definition_from_type"); + + return sprintf( + '\%s("%s", type: %s, nullable: %s, metadata: %s)', + $reflection->getName(), + $definition->entry()->name(), + $this->typeFormatter->format($definition->type()), + $definition->isNullable() ? 'true' : 'false', + $this->formatMetadata($definition->metadata()), + ); + } + /** * @param Definition $definition */ diff --git a/src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionFilterTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionFilterTransformer.php index c7a9b6ef23..36772860c5 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionFilterTransformer.php +++ b/src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionFilterTransformer.php @@ -8,33 +8,29 @@ use Flow\ETL\Config\Telemetry\TelemetryAttributes; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Function\ExpandingFunctions; -use Flow\ETL\Function\ReferenceResolver; +use Flow\ETL\Function\Evaluation\Filter; use Flow\ETL\Function\ScalarFunction; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\ETL\Transformer; use Throwable; -use function array_keys; -use function Flow\Types\DSL\type_bare; -use function Flow\Types\DSL\type_boolean; -use function Flow\Types\DSL\type_equals; -use function sprintf; - final readonly class ScalarFunctionFilterTransformer implements Transformer { /** - * @param null|ScalarFunction $resolved the predicate resolved against the bound schema; only bind() sets it + * @param null|Filter $filter the predicate bound to the input schema; only bind() sets it */ public function __construct( public ScalarFunction $function, - private ?ScalarFunction $resolved = null, + private ?Filter $filter = null, ) {} + /** + * @throws InvalidArgumentException + */ public function bind(Schema $input): BoundStep { - return new BoundStep(new self($this->function, $this->resolve($input)), $input); + return new BoundStep(new self($this->function, Filter::bind($input, $this->function)), $input); } public function transform(Rows $rows, FlowContext $context): Rows @@ -42,8 +38,7 @@ public function transform(Rows $rows, FlowContext $context): Rows $context->telemetry()->transformationStarted($this); try { - $function = $this->resolved ?? $this->resolve($rows->schema()); - $result = $rows->gather(array_keys($function->eval($rows, $context)->physicals(), true, true)); + $result = ($this->filter ?? Filter::bind($rows->schema(), $this->function))->filter($rows, $context); $context->telemetry()->transformationCompleted($this, [ TelemetryAttributes::ATTR_TRANSFORMATION_INPUT_ROWS => $rows->count(), @@ -57,28 +52,4 @@ public function transform(Rows $rows, FlowContext $context): Rows throw $e; } } - - /** - * @throws InvalidArgumentException - */ - private function resolve(Schema $input): ScalarFunction - { - $resolver = new ReferenceResolver(); - $resolved = $resolver->resolve($this->function, $input); - $resolver->assertResolved($resolved, $input); - (new ExpandingFunctions())->refuse($resolved, 'filter'); - - // type_bare() keeps the gate blind to nullability - a null-propagating predicate declares - // ?boolean, and (bool) null === false is how a NULL row is dropped. - if (!type_equals(type_bare($resolved->returns()), type_boolean())) { - throw new InvalidArgumentException(sprintf( - 'filter() requires a predicate returning boolean, "%s" returns "%s". ' - . 'Use an explicit comparison, e.g. ->notEquals(lit(0)).', - $resolved::class, - $resolved->returns()->toString(), - )); - } - - return $resolved; - } } diff --git a/src/core/etl/src/Flow/Floe/FloeExtractor.php b/src/core/etl/src/Flow/Floe/FloeExtractor.php index 1aa43bd420..d65e2f3ffb 100644 --- a/src/core/etl/src/Flow/Floe/FloeExtractor.php +++ b/src/core/etl/src/Flow/Floe/FloeExtractor.php @@ -13,8 +13,10 @@ use Flow\ETL\Extractor\File\FileExtractor; use Flow\ETL\Extractor\File\FileReading; use Flow\ETL\Extractor\File\FileReadLoop; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\File\MetadataColumnsExtractor; use Flow\ETL\Extractor\File\ReadWindow; +use Flow\ETL\Extractor\File\Scanning; use Flow\ETL\Extractor\RewindableExtractor; use Flow\ETL\Extractor\Signal; use Flow\ETL\Extractor\Statistics; @@ -24,8 +26,6 @@ use Flow\Filesystem\Filesystem; use Flow\Filesystem\Local\NativeLocalFilesystem; use Flow\Filesystem\Path; -use Flow\Filesystem\Path\Filter; -use Flow\Filesystem\Path\Filter\OnlyFiles; use Flow\Floe\Codec\NoopCodec; use Generator; @@ -42,6 +42,7 @@ final class FloeExtractor implements { use Batches; use FileReading; + use Scanning; private ?int $offset = null; @@ -78,12 +79,12 @@ public function isRepeatable(): bool /** * @return Generator */ - public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator + public function scan(FlowContext $context, FileScan $scan): Generator { $fileColumns = $this->fileColumns($this->filesystem, $this->path); - yield from (new FileReadLoop($fileColumns, $this->schema()))->read( - iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $pathFilter), false), + yield from (new FileReadLoop($fileColumns, $this->schema(), $scan->conjuncts))->read( + iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $scan->pathFilter), false), new FloeFileBatches( $this->filesystem, $this->codec, @@ -94,8 +95,8 @@ public function extract(FlowContext $context, ?int $limit = null, Filter $pathFi : new DerivedSchema($this->derivedSchema($this->files(), $this->unionByName), $this->derivedFrom), ), $this->batchSize(), - $context->backend(), - new ReadWindow($this->offset ?? 0, $limit), + $context, + new ReadWindow($this->offset ?? 0, $scan->limit), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/ExtractedRows.php b/src/core/etl/tests/Flow/ETL/Tests/Context/ExtractedRows.php index 836b22c590..2a02c79b1e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/ExtractedRows.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/ExtractedRows.php @@ -5,11 +5,10 @@ namespace Flow\ETL\Tests\Context; use Flow\ETL\Extractor; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\FlowContext; use Flow\ETL\Rows; use Flow\ETL\Tests\FlowTestCase; -use Flow\Filesystem\Path\Filter; -use Flow\Filesystem\Path\Filter\OnlyFiles; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\rows; @@ -17,19 +16,12 @@ final class ExtractedRows { - /** - * @param null|int<1, max> $limit - */ - public static function of( - Extractor $extractor, - ?FlowContext $context = null, - ?int $limit = null, - Filter $pathFilter = new OnlyFiles(), - ): Rows { + public static function of(Extractor $extractor, ?FlowContext $context = null, FileScan $scan = new FileScan()): Rows + { $context ??= flow_context(); $extracted = null; - foreach (FlowTestCase::extracted($extractor, $context, $limit, $pathFilter) as $batch) { + foreach (FlowTestCase::extracted($extractor, $context, $scan) as $batch) { $extracted = $extracted === null ? $batch : $extracted->concat($context->backend(), $batch); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/RecordingFileExtractor.php b/src/core/etl/tests/Flow/ETL/Tests/Double/RecordingFileExtractor.php index 5e9cbe6011..f66fd572be 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/RecordingFileExtractor.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/RecordingFileExtractor.php @@ -5,33 +5,31 @@ namespace Flow\ETL\Tests\Double; use Flow\ETL\Extractor\File\FileExtractor; +use Flow\ETL\Extractor\File\FileScan; +use Flow\ETL\Extractor\File\Scanning; use Flow\ETL\Extractor\Statistics; use Flow\ETL\FlowContext; +use Flow\ETL\Function\Evaluation\Filter; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\Filesystem\Path; -use Flow\Filesystem\Path\Filter; -use Flow\Filesystem\Path\Filter\OnlyFiles; use Generator; use function array_values; use function Flow\Filesystem\DSL\path; /** - * A file source that records the limit and path filter of every read, so a test can prove what the planner - * pushed reached the read - and that the instance itself was left alone. + * A file source that records the scan of every read, so a test can prove what the planner pushed reached the + * read - and that the instance itself was left alone. It returns only the rows matching the scan's filter. */ final class RecordingFileExtractor implements FileExtractor { - /** - * @var list - */ - public array $limits = []; + use Scanning; /** - * @var list + * @var list */ - public array $pathFilters = []; + public array $scans = []; /** * @var list @@ -48,13 +46,13 @@ public function __construct( $this->partitions = new Schema(); } - public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator + public function scan(FlowContext $context, FileScan $scan): Generator { - $this->limits[] = $limit; - $this->pathFilters[] = $pathFilter; + $this->scans[] = $scan; + $filter = Filter::bind($this->schema, ...$scan->conjuncts); foreach ($this->batches as $rows) { - yield $rows; + yield $filter->filter($rows, $context); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/RenameSelectRewrite.php b/src/core/etl/tests/Flow/ETL/Tests/Double/RenameSelectRewrite.php index 223e428f12..5bc42d6fbe 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/RenameSelectRewrite.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/RenameSelectRewrite.php @@ -11,8 +11,8 @@ final readonly class RenameSelectRewrite implements Rewrite { - public function of(Node $node): Node + public function of(Node $original, Node $rebuilt): Node { - return $node instanceof Select ? NodeMother::select($node->children()[0], 'name') : $node; + return $rebuilt instanceof Select ? NodeMother::select($rebuilt->children()[0], 'name') : $rebuilt; } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/SpineCopyingRule.php b/src/core/etl/tests/Flow/ETL/Tests/Double/SpineCopyingRule.php index 6a243df2cd..614e6f6803 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/SpineCopyingRule.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/SpineCopyingRule.php @@ -24,8 +24,8 @@ public function apply(LogicalPlan $plan, FlowContext $context): LogicalPlan return new LogicalPlan((new TransformUp())->of($plan->root, $this)); } - public function of(Node $node): Node + public function of(Node $original, Node $rebuilt): Node { - return $node instanceof Read ? new Read($node->extractor(), $node->limit(), $node->pathFilter()) : $node; + return $rebuilt instanceof Read ? new Read($rebuilt->extractor(), $rebuilt->scan()) : $rebuilt; } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php b/src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php index a62327205e..5702d6f613 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php +++ b/src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php @@ -8,12 +8,11 @@ use Flow\ETL\Extractor; use Flow\ETL\Extractor\BatchableExtractor; use Flow\ETL\Extractor\File\FileExtractor; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\RewindableExtractor; use Flow\ETL\Extractor\Signal; use Flow\ETL\FlowContext; use Flow\ETL\Rows; -use Flow\Filesystem\Path\Filter; -use Flow\Filesystem\Path\Filter\OnlyFiles; use Generator; use PHPUnit\Framework\TestCase; use RuntimeException; @@ -110,20 +109,18 @@ final public static function assertExtractorHonoursMaximum( /** * @param array $expectedArray - * @param null|int<1, max> $limit */ final public static function assertExtractedRowsAsArrayEquals( array $expectedArray, Extractor $extractor, ?FlowContext $flowContext = null, string $message = '', - ?int $limit = null, - Filter $pathFilter = new OnlyFiles(), + FileScan $scan = new FileScan(), ): void { $flowContext ??= flow_context(); $extractedRows = rows(schema()); - foreach (self::extracted($extractor, $flowContext, $limit, $pathFilter) as $nextRows) { + foreach (self::extracted($extractor, $flowContext, $scan) as $nextRows) { $extractedRows = $extractedRows->isEmpty() ? $nextRows : $extractedRows->concat(new PhpBackend(), $nextRows); @@ -132,42 +129,34 @@ final public static function assertExtractedRowsAsArrayEquals( static::assertEquals($expectedArray, $extractedRows->toArray(), $message); } - /** - * @param null|int<1, max> $limit - */ final public static function assertExtractedRowsCount( int $expectedCount, Extractor $extractor, ?FlowContext $flowContext = null, string $message = '', - ?int $limit = null, - Filter $pathFilter = new OnlyFiles(), + FileScan $scan = new FileScan(), ): void { $flowContext ??= flow_context(); $totalRows = 0; - foreach (self::extracted($extractor, $flowContext, $limit, $pathFilter) as $rows) { + foreach (self::extracted($extractor, $flowContext, $scan) as $rows) { $totalRows += $rows->count(); } static::assertSame($expectedCount, $totalRows, $message); } - /** - * @param null|int<1, max> $limit - */ final public static function assertExtractedRowsEquals( Rows $expectedRows, Extractor $extractor, ?FlowContext $flowContext = null, string $message = '', - ?int $limit = null, - Filter $pathFilter = new OnlyFiles(), + FileScan $scan = new FileScan(), ): void { $flowContext ??= flow_context(); $extractedRows = rows(schema()); - foreach (self::extracted($extractor, $flowContext, $limit, $pathFilter) as $nextRows) { + foreach (self::extracted($extractor, $flowContext, $scan) as $nextRows) { $extractedRows = $extractedRows->isEmpty() ? $nextRows : $extractedRows->concat(new PhpBackend(), $nextRows); @@ -177,19 +166,13 @@ final public static function assertExtractedRowsEquals( } /** - * @param null|int<1, max> $limit - * * @return Generator */ - final public static function extracted( - Extractor $extractor, - FlowContext $context, - ?int $limit, - Filter $pathFilter, - ): Generator { + final public static function extracted(Extractor $extractor, FlowContext $context, FileScan $scan): Generator + { return $extractor instanceof FileExtractor - ? $extractor->extract($context, $limit, $pathFilter) - : $extractor->extract($context, $limit); + ? $extractor->scan($context, $scan) + : $extractor->extract($context, $scan->limit); } public function repositoryRoot(): string diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/AnalyzeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/AnalyzeTest.php index 9b149c6ff1..3cfd0fedfc 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/AnalyzeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/AnalyzeTest.php @@ -18,12 +18,17 @@ use Flow\ETL\Tests\Double\InlineLoader; use Flow\ETL\Tests\FlowIntegrationTestCase; use Flow\ETL\Tests\Mother\MarketRowsMother; +use Flow\Parquet\Option; +use Flow\Parquet\Options; +use function Flow\ETL\Adapter\Parquet\from_parquet; +use function Flow\ETL\Adapter\Parquet\to_parquet; use function Flow\ETL\Adapter\Text\from_text; use function Flow\ETL\DSL\analyze; use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_array; +use function Flow\ETL\DSL\from_sequence_number; use function Flow\ETL\DSL\infer_schema; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\join_on; @@ -31,6 +36,8 @@ use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; +use function Flow\Filesystem\DSL\memory_filesystem; +use function Flow\Filesystem\DSL\path; final class AnalyzeTest extends FlowIntegrationTestCase { @@ -216,6 +223,35 @@ public function test_a_source_read_under_a_pushed_limit_reports_no_rows_error(): static::assertNull($sources[0]->rowsError()); } + public function test_a_source_read_under_a_pushed_filter_reports_no_rows_error(): void + { + $memory = memory_filesystem(); + df() + ->read(from_sequence_number('id', 1, 3000)) + ->write(to_parquet( + path('memory://analyze/ids.parquet'), + Options::default()->set(Option::ROW_GROUP_SIZE_BYTES, 1)->set( + Option::ROW_GROUP_SIZE_CHECK_INTERVAL, + 1000, + ), + filesystem: $memory, + )) + ->run(); + + $report = df() + ->read(from_parquet(path('memory://analyze/ids.parquet'), filesystem: $memory)) + ->filter(ref('id')->greaterThan(lit(2500))) + ->run(analyze: analyze()->withSourceStatistics()); + + static::assertNotNull($report); + $sources = $report->sources(); + static::assertNotNull($sources); + static::assertCount(1, $sources); + static::assertFalse($sources[0]->complete); + static::assertLessThan(3000, $sources[0]->rows); + static::assertNull($sources[0]->rowsError()); + } + public function test_run_explicit_analyze_overrides_config(): void { $config = config_builder()->analyze(analyze())->build(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/FilterTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/FilterTest.php index 48082a3f72..4006314426 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/FilterTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/FilterTest.php @@ -6,8 +6,11 @@ use Flow\ETL\Tests\FlowIntegrationTestCase; +use function Flow\ETL\Adapter\CSV\from_csv; +use function Flow\ETL\Adapter\CSV\to_csv; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_array; +use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; final class FilterTest extends FlowIntegrationTestCase @@ -58,4 +61,22 @@ public function test_single_filter(): void $rows->toArray(), ); } + + public function test_a_filter_over_a_file_source_guards_the_filter_above_it(): void + { + $file = $this->cacheDir->path() . '/guarded-filters.csv'; + + df() + ->read(from_array([['a' => 1, 'b' => 0], ['a' => 2, 'b' => 2]])) + ->write(to_csv($file)) + ->run(); + + $rows = df() + ->read(from_csv($file)) + ->filter(ref('b')->notEquals(lit(0))) + ->filter(ref('a')->divide(ref('b'))->greaterThan(lit(0))) + ->fetch(); + + static::assertSame([['a' => 2, 'b' => 2]], $rows->toArray()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/LimitTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/LimitTest.php index 9e9e1003c1..96f93dc591 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/LimitTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/LimitTest.php @@ -448,6 +448,7 @@ public function test_a_limit_over_a_read_frame_reaches_that_frames_source(bool $ $rows = df()->read($nested)->offset(1)->limit(2)->fetch(); static::assertSame([['id' => 2], ['id' => 3]], $rows->toArray()); - static::assertSame([3], $source->limits); + static::assertCount(1, $source->scans); + static::assertSame(3, $source->scans[0]->limit); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinglePlanTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinglePlanTest.php index e2f12340e5..794800a9eb 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinglePlanTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinglePlanTest.php @@ -5,6 +5,7 @@ namespace Flow\ETL\Tests\Integration\DataFrame; use Flow\ETL\ErrorHandler\IgnoreError; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Memory\ArrayMemory; use Flow\ETL\Optimizer; use Flow\ETL\Tests\Double\RecordingFileExtractor; @@ -13,7 +14,6 @@ use Flow\ETL\Tests\Mother\PartitionedSourceMother; use Flow\ETL\Transformation\AddRowIndex\StartFrom; use Flow\ETL\Transformer\AddRowIndexTransformer; -use Flow\Filesystem\Path\Filter; use RuntimeException; use function array_filter; @@ -97,7 +97,8 @@ public function test_a_joined_frame_is_optimized_with_the_outer_frames_optimizer ->join($right, join_on(['id' => 'id'], 'r_')) ->fetch(); - static::assertSame([1], $source->limits); + static::assertCount(1, $source->scans); + static::assertSame(1, $source->scans[0]->limit); } public function test_a_joined_frames_own_error_handler_is_ignored(): void @@ -145,11 +146,11 @@ public function test_a_filter_pushed_into_a_source_both_sides_share_leaves_the_j ->crossJoin($joined, 'j_') ->fetch(); - $accepts2024 = static fn(Filter $filter): bool => $filter->accept(PartitionedSourceMother::file( + $accepts2024 = static fn(FileScan $scan): bool => $scan->pathFilter->accept(PartitionedSourceMother::file( 'year=2024/month=08', )); - static::assertCount(2, $source->pathFilters); - static::assertCount(1, array_filter($source->pathFilters, $accepts2024)); + static::assertCount(2, $source->scans); + static::assertCount(1, array_filter($source->scans, $accepts2024)); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootTest.php index 6c3b61c2a4..726a8895fd 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootTest.php @@ -414,7 +414,7 @@ public function test_a_later_limit_does_not_narrow_an_earlier_sink(): void $rows = df()->read($extractor)->write($spy)->limit(2)->fetch(); static::assertCount(2, $rows); - static::assertNull($extractor->limits[0]); + static::assertNull($extractor->scans[0]->limit); static::assertSame([3], $spy->loadedRowCounts()); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/FileExtractorScanTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/FileExtractorScanTest.php new file mode 100644 index 0000000000..2e664ad1ab --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/FileExtractorScanTest.php @@ -0,0 +1,155 @@ +cacheDir->path() . '/scan-' . $family; + $uri = path($directory . '/rows.' . $family); + $xlsx = path($directory . '/rows.xlsx'); + + [$sink, $extractor] = match ($family) { + 'csv' => [to_csv($uri), from_csv($uri)], + 'json' => [to_json($uri), from_json($uri)], + 'text' => [to_text($uri), from_text($uri)], + 'xml' => [to_xml($uri), from_xml($uri)], + 'excel' => [to_excel($xlsx), from_excel($xlsx)], + 'parquet' => [to_parquet($uri), from_parquet($uri)], + 'floe' => [to_floe($uri), from_floe($uri)], + 'files' => [to_csv($uri), files($directory . '/*')], + default => [to_csv(path($directory . '/id=a/rows.csv')), from_path_partitions($directory . '/*/*.csv')], + }; + + df() + ->read(from_array([['id' => 'a'], ['id' => 'b'], ['id' => 'c']])) + ->write($sink) + ->run(); + + $context = flow_context(); + $scanned = rows(schema()); + + foreach ($extractor->scan($context, new FileScan(limit: 2)) as $batch) { + $scanned = $scanned->isEmpty() ? $batch : $scanned->concat($context->backend(), $batch); + } + + static::assertNotSame([], $scanned->toArray()); + static::assertEquals(ExtractedRows::of($extractor, $context, new FileScan(2))->toArray(), $scanned->toArray()); + } + + #[TestWith(['csv'])] + #[TestWith(['json'])] + #[TestWith(['text'])] + #[TestWith(['xml'])] + #[TestWith(['excel'])] + #[TestWith(['parquet'])] + #[TestWith(['floe'])] + #[TestWith(['files'])] + #[TestWith(['path_partitions'])] + public function test_a_pushed_filter_returns_what_the_filter_node_returns(string $family): void + { + $directory = $this->cacheDir->path() . '/pushed-' . $family; + $uri = path($directory . '/rows.' . $family); + $xlsx = path($directory . '/rows.xlsx'); + + [$sink, $extractor, $predicate] = match ($family) { + 'csv' => [to_csv($uri), static fn(): FileExtractor => from_csv($uri), ref('id')->notEquals(lit('b'))], + 'json' => [to_json($uri), static fn(): FileExtractor => from_json($uri), ref('id')->notEquals(lit('b'))], + 'text' => [to_text($uri), static fn(): FileExtractor => from_text($uri), ref('text')->notEquals(lit('b'))], + 'xml' => [ + to_xml($uri), + static fn(): FileExtractor => from_xml($uri, 'rows/row'), + ref('node')->xpath('id')->domElementValue()->notEquals(lit('b')), + ], + 'excel' => [ + to_excel($xlsx), + static fn(): FileExtractor => from_excel($xlsx), + ref('id')->notEquals(lit('b')), + ], + 'parquet' => [ + to_parquet($uri), + static fn(): FileExtractor => from_parquet($uri), + ref('id')->notEquals(lit('b')), + ], + 'floe' => [to_floe($uri), static fn(): FileExtractor => from_floe($uri), ref('id')->notEquals(lit('b'))], + 'files' => [null, static fn(): FileExtractor => files($directory . '/*/*'), ref('id')->notEquals(lit('b'))], + default => [ + null, + static fn(): FileExtractor => from_path_partitions($directory . '/*/*'), + ref('id')->notEquals(lit('b')), + ], + }; + + if ($sink === null) { + foreach (['a', 'b', 'c'] as $id) { + df() + ->read(from_array([['id' => $id]])) + ->write(to_csv(path($directory . '/id=' . $id . '/rows.csv'))) + ->run(); + } + } else { + df() + ->read(from_array([['id' => 'a'], ['id' => 'b'], ['id' => 'c']])) + ->write($sink) + ->run(); + } + + $pushed = df()->read($extractor())->filter($predicate)->fetch(); + $plain = df(config_builder()->optimizer(Optimizer::default()->without(PushFilterIntoSource::class))) + ->read($extractor()) + ->filter($predicate) + ->fetch(); + + static::assertCount(2, $plain); + static::assertEquals($plain->toArray(), $pushed->toArray()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php index f048299c3a..4a3aefbd36 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php @@ -6,6 +6,7 @@ use Flow\ETL\Cardinality; use Flow\ETL\Column\PhpBackend; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\PathPartitionsExtractor; use Flow\ETL\Rows; use Flow\ETL\Tests\Context\ExtractedRows; @@ -16,7 +17,9 @@ use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_path_partitions; +use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\map_schema; +use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -243,4 +246,20 @@ public function test_path_partitions_extractor_honours_the_batch_contract(): voi ), ); } + + public function test_a_scan_returns_only_the_files_matching_its_filter(): void + { + $batches = iterator_to_array(from_path_partitions('memory://dir/**/*.txt', MemoryFiles::with([ + 'memory://dir/year=2021/a.txt' => 'a', + 'memory://dir/year=2022/b.txt' => 'b', + 'memory://dir/year=2023/c.txt' => 'c', + 'memory://dir/year=2024/d.txt' => 'd', + 'memory://dir/year=2025/e.txt' => 'e', + ])) + ->withBatchSize(2) + ->scan(flow_context(), new FileScan(conjuncts: [ref('year')->isIn(lit(['2023', '2024']))])), false); + + static::assertCount(1, $batches); + static::assertSame(['2023', '2024'], $batches[0]->column('year')->values()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/PartitionedSourceMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/PartitionedSourceMother.php index a6d27a9281..bbdbb81314 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Mother/PartitionedSourceMother.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/PartitionedSourceMother.php @@ -22,8 +22,8 @@ public static function partitions(): Schema } /** - * One row per partition. The double ignores the pushed path filter, so only a Filter node the planner kept can - * drop the other partition's row. + * One row per partition. The double ignores the pushed path filter, so only the scan's filter or a Filter node + * the planner kept can drop the other partition's row. */ public static function yearMonth(): RecordingFileExtractor { diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalForTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalForTest.php index 42ee0b927d..77dcd65dd9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalForTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalForTest.php @@ -130,7 +130,7 @@ public static function supported(): Generator foreach (self::types() as $name => [$type]) { yield $name => [ $type, - !in_array($name, ['positive integer', 'non-empty string', 'numeric string', 'class string'], true), + !in_array($name, ['positive integer', 'non-empty string', 'class string'], true), ]; } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Telemetry/TelemetryContextTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Telemetry/TelemetryContextTest.php index 83d3cdcb0c..06d2628298 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Telemetry/TelemetryContextTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Telemetry/TelemetryContextTest.php @@ -990,8 +990,8 @@ public function test_optimizer_rules_are_logged(): void [ CombineLimits::class, CombineSortAndLimit::class, - PushLimitIntoSource::class, PushFilterIntoSource::class, + PushLimitIntoSource::class, CountFromStatistics::class, PushProjectionIntoExpand::class, ], diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/DataFrameTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/DataFrameTest.php index e9e50824f2..9a7205e89f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/DataFrameTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/DataFrameTest.php @@ -958,9 +958,9 @@ public function test_a_pushed_limit_reaches_extract_and_leaves_the_users_extract static::assertSame([['id' => 1, 'doubled' => 2]], $frame->fetch()->toArray()); static::assertSame([['id' => 1, 'doubled' => 2]], $frame->fetch()->toArray()); - static::assertCount(2, $extractor->limits); - static::assertSame(1, $extractor->limits[0]); - static::assertSame(1, $extractor->limits[1]); + static::assertCount(2, $extractor->scans); + static::assertSame(1, $extractor->scans[0]->limit); + static::assertSame(1, $extractor->scans[1]->limit); static::assertStringEndsWith( "#1 Read\n Extractor: RecordingFileExtractor\n Source: file:///dev/null", $frame->explain()->toString(Stage::unoptimized), @@ -985,7 +985,7 @@ public function test_a_limit_inside_a_joins_right_side_is_pushed_into_that_sourc ->join($right, join_on(['id' => 'id'], 'r_')) ->fetch(); - static::assertSame(1, $extractor->limits[0]); + static::assertSame(1, $extractor->scans[0]->limit); } public function test_an_empty_fetch_plans_once(): void @@ -1112,7 +1112,7 @@ public function test_a_limit_inside_a_read_frame_is_pushed_into_its_source(): vo $inner = df()->read($extractor)->limit(1); static::assertSame([['id' => 1]], df()->read(from_data_frame($inner))->fetch()->toArray()); - static::assertSame(1, $extractor->limits[0]); + static::assertSame(1, $extractor->scans[0]->limit); } public function test_explain_without_sinks_prints_one_result_over_the_chain(): void @@ -1122,8 +1122,8 @@ public function test_explain_without_sinks_prints_one_result_over_the_chain(): v static::assertSame(<<<'PLAN' #3 Result preserving · transparent · streaming │ Rows this plan hands out: to the trigger, or to the node reading it - └─ #2 Select preserving · transparent · streaming - │ Columns: id + └─ #2 Select preserving · transparent · streaming · redefines all but id + │ Keeps only: id └─ #1 Read source · transparent · streaming Extractor: ArrayExtractor PLAN, $dataFrame->explain()->toString(format: Format::declarations)); @@ -1140,8 +1140,8 @@ public function test_explain_with_a_sink_prints_the_outputs_sharing_the_chain(): Outputs preserving · opaque · streaming ├─ #3 Result preserving · transparent · streaming │ │ Rows this plan hands out: to the trigger, or to the node reading it - │ └─ #2 Select preserving · transparent · streaming - │ │ Columns: id + │ └─ #2 Select preserving · transparent · streaming · redefines all but id + │ │ Keeps only: id │ └─ #1 Read source · transparent · streaming │ Extractor: ArrayExtractor └─ #4 Write preserving · opaque · streaming @@ -1173,8 +1173,8 @@ public function test_explain_of_the_run_trigger_keeps_a_result_over_a_chain_end_ Outputs preserving · opaque · streaming ├─ #3 Result preserving · transparent · streaming │ │ Rows this plan hands out: to the trigger, or to the node reading it - │ └─ #2 Select preserving · transparent · streaming - │ │ Columns: id + │ └─ #2 Select preserving · transparent · streaming · redefines all but id + │ │ Keeps only: id │ └─ #1 Read source · transparent · streaming │ Extractor: ArrayExtractor └─ #4 Write preserving · opaque · streaming @@ -1203,15 +1203,14 @@ public function test_explain_prints_the_logical_plan_without_reading_a_row(): vo $dataFrame = df()->read($extractor)->filter(ref('id')->isNotNull())->write(to_memory(new ArrayMemory())); static::assertSame(<<<'PLAN' - #3 Write preserving · opaque · streaming + #2 Write preserving · opaque · streaming │ Loader: MemoryLoader - └─ #2 Filter reducing · transparent · streaming - │ Condition: IsNotNull - └─ #1 Read source · transparent · streaming - Extractor: RecordingFileExtractor - Source: file:///dev/null + └─ #1 Read source · transparent · streaming + Extractor: RecordingFileExtractor + Source: file:///dev/null + Filter: IsNotNull PLAN, $dataFrame->explain()->toString(format: Format::declarations)); - static::assertSame([], $extractor->limits); + static::assertSame([], $extractor->scans); } public function test_from_data_frame_reads_the_frame_as_a_source(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/PipelineTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/PipelineTest.php index c7da1b6421..21a448113d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/PipelineTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/PipelineTest.php @@ -7,9 +7,9 @@ use Flow\ETL\Executor\Pipeline; use Flow\ETL\Executor\Segments; use Flow\ETL\Executor\SourceRows; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\NodeMother; -use Flow\Filesystem\Path\Filter\OnlyFiles; use Flow\Filesystem\Tests\Double\RejectingFilter; use function Flow\ETL\DSL\from_array; @@ -21,18 +21,16 @@ public function test_accessors_return_the_values_it_was_built_with(): void $segments = new Segments(); $context = NodeMother::context(); $input = new Pipeline(0, new Segments(from_array([['id' => 1]])), $context); - $filter = new RejectingFilter(); - + $scan = new FileScan(3, new RejectingFilter()); $sources = new SourceRows(); - $pipeline = new Pipeline(1, $segments, $context, $input, 3, $filter, $sources); + $pipeline = new Pipeline(1, $segments, $context, $input, $scan, $sources); static::assertSame(1, $pipeline->id); static::assertSame($segments, $pipeline->segments()); static::assertSame($context, $pipeline->context()); static::assertSame($input, $pipeline->input()); - static::assertSame(3, $pipeline->limit()); - static::assertSame($filter, $pipeline->pathFilter()); + static::assertSame($scan, $pipeline->scan()); static::assertSame($sources, $pipeline->sources()); } @@ -41,8 +39,7 @@ public function test_a_leaf_pipeline_defaults_to_no_input_no_limit_and_only_file $pipeline = new Pipeline(0, new Segments(from_array([['id' => 1]])), NodeMother::context()); static::assertNull($pipeline->input()); - static::assertNull($pipeline->limit()); - static::assertInstanceOf(OnlyFiles::class, $pipeline->pathFilter()); + static::assertEquals(new FileScan(), $pipeline->scan()); static::assertNull($pipeline->sources()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/ExecutorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/ExecutorTest.php index efeb84abb4..2b4f76aa06 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/ExecutorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/ExecutorTest.php @@ -12,6 +12,7 @@ use Flow\ETL\Executor\Pipeline; use Flow\ETL\Executor\Raw; use Flow\ETL\Executor\Segments; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Optimizer; use Flow\ETL\Plan\Node\Transform; use Flow\ETL\Processor\BatchingProcessor; @@ -186,17 +187,16 @@ public function test_a_pipeline_without_a_source_throws(): void iterator_to_array((new Executor())->executePipeline(new Pipeline(7, new Segments(), NodeMother::context()))); } - public function test_a_file_source_receives_the_pipelines_limit_and_path_filter(): void + public function test_a_file_source_receives_the_pipelines_scan(): void { $extractor = new RecordingFileExtractor(schema(int_schema('id')), RowsMother::sequentialIds(1)); - $filter = new RejectingFilter(); + $scan = new FileScan(7, new RejectingFilter()); iterator_to_array((new Executor())->executePipeline( - new Pipeline(0, new Segments($extractor), NodeMother::context(), limit: 7, pathFilter: $filter), + new Pipeline(0, new Segments($extractor), NodeMother::context(), scan: $scan), )); - static::assertSame([7], $extractor->limits); - static::assertSame([$filter], $extractor->pathFilters); + static::assertSame([$scan], $extractor->scans); } public function test_a_source_that_lists_no_files_receives_the_pipelines_limit(): void @@ -204,7 +204,7 @@ public function test_a_source_that_lists_no_files_receives_the_pipelines_limit() $extractor = new RecordingExtractor(schema(int_schema('id')), RowsMother::sequentialIds(1)); iterator_to_array((new Executor())->executePipeline( - new Pipeline(0, new Segments($extractor), NodeMother::context(), limit: 7), + new Pipeline(0, new Segments($extractor), NodeMother::context(), scan: new FileScan(7)), )); static::assertSame([7], $extractor->limits); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/DataFrameExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/DataFrameExtractorTest.php index dc607111f5..4bc0922602 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/DataFrameExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/DataFrameExtractorTest.php @@ -162,7 +162,8 @@ public function test_a_limit_given_to_extract_reaches_the_frames_source(): void $batches = iterator_to_array(from_data_frame(df()->read($source))->extract(flow_context(config()), 2), false); - static::assertSame([2], $source->limits); + static::assertCount(1, $source->scans); + static::assertSame(2, $source->scans[0]->limit); static::assertSame( [['id' => 1], ['id' => 2]], array_merge(...array_map(static fn($rows) => $rows->toArray(), $batches)), diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileMetadataRowTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileMetadataRowTest.php new file mode 100644 index 0000000000..a29fd6bfd2 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileMetadataRowTest.php @@ -0,0 +1,35 @@ + '/dir/a.csv', + 'protocol' => 'memory', + 'file_name' => 'a', + 'base_name' => 'a.csv', + 'is_file' => true, + 'is_dir' => false, + 'extension' => 'csv', + ], + (new FileMetadataRow())->row(new FileStatus(path('memory://dir/a.csv'), true)), + ); + } + + public function test_a_file_without_an_extension_has_a_null_extension(): void + { + static::assertNull((new FileMetadataRow())->row(new FileStatus(path('memory://dir/a'), true))['extension']); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileReadLoopTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileReadLoopTest.php index 7357642e5f..5a22c67cd2 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileReadLoopTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileReadLoopTest.php @@ -4,7 +4,6 @@ namespace Flow\ETL\Tests\Unit\Extractor\File; -use Flow\ETL\Column\PhpBackend; use Flow\ETL\Extractor\File\FileReadLoop; use Flow\ETL\Extractor\File\ReadWindow; use Flow\ETL\Extractor\File\SourceFile; @@ -17,7 +16,10 @@ use function array_keys; use function array_map; +use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\lit; +use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Filesystem\DSL\path; @@ -40,7 +42,7 @@ public function test_every_batch_carries_its_files_constants(): void ], $batches, 10, - new PhpBackend(), + flow_context(), new ReadWindow(), )); @@ -56,7 +58,7 @@ public function test_stop_after_the_first_batch_ends_the_read(): void [new SourceFile(path('memory://a.csv')), new SourceFile(path('memory://b.csv'))], new InMemoryFileBatches(['memory://a.csv' => [1, 2, 3], 'memory://b.csv' => [4]]), 1, - new PhpBackend(), + flow_context(), new ReadWindow(), ); @@ -78,7 +80,7 @@ public function test_a_limit_inside_a_file_ends_the_read_and_is_pushed_down(): v [new SourceFile(path('memory://a.csv')), new SourceFile(path('memory://b.csv'))], $batches, 10, - new PhpBackend(), + flow_context(), new ReadWindow(limit: 2), )); @@ -100,7 +102,7 @@ public function test_a_limit_at_a_file_boundary_opens_no_further_file(): void [new SourceFile(path('memory://a.csv')), new SourceFile(path('memory://b.csv'))], $batches, 1, - new PhpBackend(), + flow_context(), new ReadWindow(limit: 2), )); @@ -127,7 +129,7 @@ public function test_the_offset_a_file_leaves_and_the_rows_still_wanted_reach_th ], $batches, 10, - new PhpBackend(), + flow_context(), new ReadWindow(offset: 3, limit: 2), )); @@ -152,7 +154,7 @@ public function test_no_sources_read_nothing(): void iterator_to_array((new FileReadLoop( FileColumnsContext::discovering(names: []), schema(int_schema('id')), - ))->read([], new InMemoryFileBatches([]), 10, new PhpBackend(), new ReadWindow())), + ))->read([], new InMemoryFileBatches([]), 10, flow_context(), new ReadWindow())), ); } @@ -173,7 +175,7 @@ public function test_the_loop_slices_the_offset_a_format_cannot_skip_across_file ], $batches, 10, - new PhpBackend(), + flow_context(), new ReadWindow(offset: 3, limit: 2), ), false, @@ -202,7 +204,7 @@ public function test_the_loop_skips_the_offset_and_trims_to_the_limit_for_a_form [new SourceFile(path('memory://a.csv'))], $batches, 10, - new PhpBackend(), + flow_context(), new ReadWindow(offset: 3, limit: 2), ), false, @@ -214,4 +216,127 @@ public function test_the_loop_skips_the_offset_and_trims_to_the_limit_for_a_form ); static::assertEquals(['memory://a.csv' => [new ReadWindow(0, 5)]], $batches->windows); } + + public function test_a_filter_drops_the_rows_it_does_not_match_and_skips_a_batch_it_empties(): void + { + $rows = iterator_to_array( + (new FileReadLoop(FileColumnsContext::discovering(names: []), schema(int_schema('id')), [ref( + 'id', + )->greaterThan(lit( + 2, + ))]))->read([new SourceFile(path('memory://a.csv'))], new InMemoryFileBatches(['memory://a.csv' => [1, 2, 3, 4, 5]]), 2, flow_context(), new ReadWindow()), + false, + ); + + static::assertSame( + [[['id' => 3], ['id' => 4]], [['id' => 5]]], + array_map(static fn(Rows $batch): array => $batch->toArray(), $rows), + ); + } + + public function test_a_filter_mixing_a_body_and_a_partition_column_sees_the_partition_values(): void + { + $rows = iterator_to_array( + (new FileReadLoop(FileColumnsContext::discovering(), schema(int_schema('id'), str_schema('year')), [ + ref('year')->equals(lit('2024')), + ref('id')->greaterThan(lit(3)), + ]))->read([ + new SourceFile(path('memory://orders/year=2023/a.csv')), + new SourceFile(path('memory://orders/year=2024/b.csv')), + ], new InMemoryFileBatches([ + 'memory://orders/year=2023/a.csv' => [4, 5], + 'memory://orders/year=2024/b.csv' => [3, 4], + ]), 10, flow_context(), new ReadWindow()), + false, + ); + + static::assertSame( + [[['id' => 4, 'year' => '2024']]], + array_map(static fn(Rows $batch): array => $batch->toArray(), $rows), + ); + } + + public function test_the_offset_counts_raw_rows_and_the_limit_counts_filtered_ones(): void + { + $batches = new InMemoryFileBatches(['memory://a.csv' => [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]]); + + $rows = iterator_to_array( + (new FileReadLoop(FileColumnsContext::discovering(names: []), schema(int_schema('id')), [ref( + 'id', + )->greaterThan(lit( + 5, + ))]))->read([new SourceFile(path('memory://a.csv'))], $batches, 10, flow_context(), new ReadWindow(offset: 6, limit: 2)), + false, + ); + + static::assertSame( + [[['id' => 7], ['id' => 8]]], + array_map(static fn(Rows $batch): array => $batch->toArray(), $rows), + ); + static::assertEquals(['memory://a.csv' => [new ReadWindow(6)]], $batches->windows); + } + + public function test_the_loop_drops_the_raw_offset_before_the_filter_for_a_format_that_cannot_skip(): void + { + $batches = new InMemoryUnskippableFileBatches(['memory://a.csv' => [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]]); + + $rows = iterator_to_array( + (new FileReadLoop(FileColumnsContext::discovering(names: []), schema(int_schema('id')), [ref( + 'id', + )->greaterThan(lit( + 5, + ))]))->read([new SourceFile(path('memory://a.csv'))], $batches, 10, flow_context(), new ReadWindow(offset: 6, limit: 2)), + false, + ); + + static::assertSame( + [[['id' => 7], ['id' => 8]]], + array_map(static fn(Rows $batch): array => $batch->toArray(), $rows), + ); + static::assertEquals(['memory://a.csv' => [new ReadWindow()]], $batches->windows); + } + + public function test_a_limit_beside_a_filter_counts_matching_rows_across_files_and_opens_no_further_file(): void + { + $batches = new InMemoryFileBatches([ + 'memory://a.csv' => [1, 2], + 'memory://b.csv' => [3, 4], + 'memory://c.csv' => [5], + ]); + + $rows = iterator_to_array( + (new FileReadLoop(FileColumnsContext::discovering(names: []), schema(int_schema('id')), [ref( + 'id', + )->greaterThan(lit( + 1, + ))]))->read([ + new SourceFile(path('memory://a.csv')), + new SourceFile(path('memory://b.csv')), + new SourceFile(path('memory://c.csv')), + ], $batches, 10, flow_context(), new ReadWindow(limit: 2)), + false, + ); + + static::assertSame( + [[['id' => 2]], [['id' => 3]]], + array_map(static fn(Rows $batch): array => $batch->toArray(), $rows), + ); + static::assertEquals( + ['memory://a.csv' => [new ReadWindow()], 'memory://b.csv' => [new ReadWindow()]], + $batches->windows, + ); + } + + public function test_each_conjunct_of_the_filter_sees_only_the_rows_the_ones_before_it_kept(): void + { + $rows = iterator_to_array( + (new FileReadLoop(FileColumnsContext::discovering(names: []), schema(int_schema('id')), [ + ref('id')->notEquals(lit(0)), + lit(10)->divide(ref('id'))->greaterThan(lit(1)), + ]))->read([new SourceFile(path('memory://a.csv'))], new InMemoryFileBatches(['memory://a.csv' => [0, 2, 10]]), 10, flow_context(), new ReadWindow()), + false, + ); + + static::assertSame([[['id' => 2]]], array_map(static fn(Rows $batch): array => $batch->toArray(), $rows)); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileScanTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileScanTest.php new file mode 100644 index 0000000000..8a6a115393 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileScanTest.php @@ -0,0 +1,91 @@ +limit); + static::assertInstanceOf(OnlyFiles::class, $scan->pathFilter); + static::assertSame([], $scan->conjuncts); + static::assertFalse($scan->narrows()); + } + + #[TestWith([0])] + #[TestWith([-1])] + public function test_a_limit_below_one_is_refused(int $limit): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Limit must be greater than 0'); + + new FileScan($limit); + } + + public function test_a_filter_appends_its_conjuncts_in_push_order(): void + { + $first = ref('id')->greaterThan(lit(1)); + $second = ref('id')->lessThan(lit(10)); + $third = ref('id')->notEquals(lit(5)); + + $scan = (new FileScan(5)) + ->withFilter($first) + ->withFilter($second->and($third)); + + static::assertSame([$first, $second, $third], $scan->conjuncts); + static::assertSame(5, $scan->limit); + static::assertTrue($scan->narrows()); + } + + public function test_a_second_limit_only_lowers_the_first_and_keeps_the_conjuncts(): void + { + $filter = ref('id')->greaterThan(lit(1)); + $scan = (new FileScan()) + ->withFilter($filter) + ->withLimit(5); + + static::assertSame(3, $scan->withLimit(3)->limit); + static::assertSame(5, $scan->withLimit(8)->limit); + static::assertSame([$filter], $scan->withLimit(8)->conjuncts); + static::assertTrue( + (new FileScan()) + ->withLimit(1) + ->narrows(), + ); + } + + public function test_path_filters_compose_and_keep_the_conjuncts(): void + { + $filter = ref('id')->greaterThan(lit(1)); + $first = new RejectingFilter(); + $second = new RejectingFilter(); + + $scan = (new FileScan()) + ->withFilter($filter) + ->withPathFilter($first) + ->withPathFilter($second); + + static::assertEquals(new Filters(new OnlyFiles(), $first, $second), $scan->pathFilter); + static::assertSame([$filter], $scan->conjuncts); + static::assertTrue( + (new FileScan()) + ->withPathFilter($first) + ->narrows(), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/ListingScanTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/ListingScanTest.php new file mode 100644 index 0000000000..fa5ffc405f --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/ListingScanTest.php @@ -0,0 +1,154 @@ + 'a', 'memory://dir/b' => 'b', 'memory://dir/c' => 'c']); + + static::assertSame( + [['memory://dir/a', 'memory://dir/b'], ['memory://dir/c']], + array_map( + static fn(Rows $rows): array => $rows->reduceToArray('path'), + iterator_to_array( + (new ListingScan( + $filesystem, + path('memory://dir/*'), + new FileColumns(new PartitionColumns($filesystem), [], new PartitionTypes(), false), + new PathPartitionsRow(), + ))->scan( + flow_context(), + new FileScan(), + schema(str_schema('path'), map_schema('partitions', type_map(type_string(), type_string()))), + 2, + ), + false, + ), + ), + ); + } + + public function test_a_batch_holds_only_the_files_every_conjunct_keeps_and_an_emptied_batch_is_skipped(): void + { + $filesystem = MemoryFiles::with(['memory://dir/a' => 'a', 'memory://dir/b' => 'b', 'memory://dir/c' => 'c']); + + static::assertSame( + [['memory://dir/c']], + array_map( + static fn(Rows $rows): array => $rows->reduceToArray('path'), + iterator_to_array( + (new ListingScan( + $filesystem, + path('memory://dir/*'), + new FileColumns(new PartitionColumns($filesystem), [], new PartitionTypes(), false), + new PathPartitionsRow(), + ))->scan( + flow_context(), + new FileScan(conjuncts: [ + ref('path')->notEquals(lit('memory://dir/a')), + ref('path')->notEquals(lit('memory://dir/b')), + ]), + schema(str_schema('path'), map_schema('partitions', type_map(type_string(), type_string()))), + 2, + ), + false, + ), + ), + ); + } + + public function test_the_scan_stops_once_the_limit_of_rows_is_handed_out(): void + { + $filesystem = MemoryFiles::with(['memory://dir/a' => 'a', 'memory://dir/b' => 'b', 'memory://dir/c' => 'c']); + + static::assertCount(1, iterator_to_array( + (new ListingScan( + $filesystem, + path('memory://dir/*'), + new FileColumns(new PartitionColumns($filesystem), [], new PartitionTypes(), false), + new PathPartitionsRow(), + ))->scan( + flow_context(), + new FileScan(limit: 1), + schema(str_schema('path'), map_schema('partitions', type_map(type_string(), type_string()))), + 1, + ), + false, + )); + } + + public function test_a_batch_past_the_limit_is_trimmed_to_it(): void + { + $filesystem = MemoryFiles::with(['memory://dir/a' => 'a', 'memory://dir/b' => 'b', 'memory://dir/c' => 'c']); + + static::assertSame( + [['memory://dir/a', 'memory://dir/b']], + array_map( + static fn(Rows $rows): array => $rows->reduceToArray('path'), + iterator_to_array( + (new ListingScan( + $filesystem, + path('memory://dir/*'), + new FileColumns(new PartitionColumns($filesystem), [], new PartitionTypes(), false), + new PathPartitionsRow(), + ))->scan( + flow_context(), + new FileScan(limit: 2), + schema(str_schema('path'), map_schema('partitions', type_map(type_string(), type_string()))), + 3, + ), + false, + ), + ), + ); + } + + public function test_a_stop_signal_ends_the_scan(): void + { + $filesystem = MemoryFiles::with(['memory://dir/a' => 'a', 'memory://dir/b' => 'b']); + $scan = (new ListingScan( + $filesystem, + path('memory://dir/*'), + new FileColumns(new PartitionColumns($filesystem), [], new PartitionTypes(), false), + new PathPartitionsRow(), + ))->scan( + flow_context(), + new FileScan(), + schema(str_schema('path'), map_schema('partitions', type_map(type_string(), type_string()))), + 1, + ); + + static::assertSame(['memory://dir/a'], $scan->current()->reduceToArray('path')); + + $scan->send(Signal::STOP); + + static::assertFalse($scan->valid()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/PathPartitionsRowTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/PathPartitionsRowTest.php new file mode 100644 index 0000000000..f302978165 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/PathPartitionsRowTest.php @@ -0,0 +1,30 @@ + 'memory://dir/year=2024/month=01/a.csv', 'partitions' => ['year' => '2024', 'month' => '01']], + (new PathPartitionsRow())->row(new FileStatus(path('memory://dir/year=2024/month=01/a.csv'), true)), + ); + } + + public function test_a_file_outside_partitions_has_none(): void + { + static::assertSame( + [], + (new PathPartitionsRow())->row(new FileStatus(path('memory://dir/a.csv'), true))['partitions'], + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FilesExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FilesExtractorTest.php index 62e717d70f..b9308f80b8 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FilesExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FilesExtractorTest.php @@ -5,6 +5,7 @@ namespace Flow\ETL\Tests\Unit\Extractor; use Flow\ETL\Cardinality; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\FilesExtractor; use Flow\ETL\Extractor\Signal; use Flow\ETL\Schema; @@ -16,6 +17,8 @@ use function Flow\ETL\DSL\files; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\lit; +use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function iterator_to_array; @@ -139,7 +142,7 @@ public function test_extracting_files_from_directory_with_limit(): void { $extractor = files(__DIR__ . '/Fixtures/FileListExtractor/**/*')->withBatchSize(1); - self::assertExtractedRowsCount(2, $extractor, limit: 2); + self::assertExtractedRowsCount(2, $extractor, scan: new FileScan(2)); } public function test_is_repeatable(): void @@ -202,4 +205,20 @@ public function test_files_extractor_honours_the_batch_contract(): void ExtractedRows::of(files(__DIR__ . '/Fixtures/FileListExtractor/**/*')->withBatchSize(1)), ); } + + public function test_a_scan_returns_only_the_files_matching_its_filter(): void + { + $batches = iterator_to_array(files('memory://dir/*.txt', MemoryFiles::with([ + 'memory://dir/a.txt' => 'a', + 'memory://dir/b.txt' => 'b', + 'memory://dir/c.txt' => 'c', + 'memory://dir/d.txt' => 'd', + 'memory://dir/e.txt' => 'e', + ])) + ->withBatchSize(2) + ->scan(flow_context(), new FileScan(conjuncts: [ref('base_name')->isIn(lit(['c.txt', 'd.txt']))])), false); + + static::assertCount(1, $batches); + static::assertSame(['c.txt', 'd.txt'], $batches[0]->column('base_name')->values()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowPhpExtensionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowPhpExtensionTest.php index 1ecd4d604a..25f21489dc 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowPhpExtensionTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowPhpExtensionTest.php @@ -26,7 +26,7 @@ public function test_a_loaded_extension_of_another_abi_is_refused(): void { $this->expectException(RuntimeException::class); $this->expectExceptionMessage( - 'The loaded flow_php extension (0.46.0, ABI 2) does not match flow-php/etl (ABI 1). Rebuild or upgrade ' + 'The loaded flow_php extension (0.46.0, ABI 3) does not match flow-php/etl (ABI 2). Rebuild or upgrade ' . 'flow_php to the version of flow-php/etl, or stop loading it to use the PHP implementation.', ); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BetweenTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BetweenTest.php index 17cf7fa38c..94c23fa6fb 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BetweenTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BetweenTest.php @@ -12,11 +12,14 @@ use Flow\Types\Exception\InvalidArgumentException as TypesInvalidArgumentException; use function Flow\ETL\DSL\between; +use function Flow\ETL\DSL\definition_from_type; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\Types\DSL\type_numeric_string; final class BetweenTest extends FlowTestCase { @@ -165,4 +168,34 @@ public function test_between_refuses_incomparable_bounds_at_bind(): void ->between(lit(1), lit(new DateTimeImmutable('2024-01-01'))) ->returns(); } + + public function test_between_orders_strings_by_bytes(): void + { + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('a')->between(lit('10'), lit('9')), + ['a' => '100'], + schema(str_schema('a')), + )); + } + + public function test_between_orders_a_numeric_string_column_numerically(): void + { + $context = flow_context(); + + static::assertFalse((new FunctionContext($context))->eval( + ref('a')->between(lit('10'), lit('9')), + ['a' => '100'], + schema(definition_from_type('a', type_numeric_string())), + )); + static::assertTrue((new FunctionContext($context))->eval( + ref('a')->between(lit('9'), lit('18446744073709551615'), lit(Boundary::INCLUSIVE)), + ['a' => '18446744073709551615'], + schema(definition_from_type('a', type_numeric_string())), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('a')->between(lit('9'), lit('18446744073709551614'), lit(Boundary::INCLUSIVE)), + ['a' => '18446744073709551615'], + schema(definition_from_type('a', type_numeric_string())), + )); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ConjunctsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ConjunctsTest.php new file mode 100644 index 0000000000..e96211fbf3 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ConjunctsTest.php @@ -0,0 +1,41 @@ +greaterThan(lit(1)); + + static::assertSame([$function], (new Conjuncts())->of($function)); + } + + public function test_a_nested_and_is_flattened(): void + { + $a = ref('a')->equals(lit(1)); + $b = ref('b')->equals(lit(2)); + $c = ref('c')->equals(lit(3)); + $d = ref('d')->equals(lit(4)); + + static::assertSame([$a, $b, $c, $d], (new Conjuncts())->of(new All($a, new All($b, new All($c)), $d))); + } + + public function test_an_or_is_kept_whole(): void + { + $a = ref('a')->equals(lit(1)); + $or = new Any(ref('b')->equals(lit(2)), new All(ref('c')->equals(lit(3)), ref('d')->equals(lit(4)))); + + static::assertSame([$a, $or], (new Conjuncts())->of(new All($a, $or))); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/FilterTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/FilterTest.php new file mode 100644 index 0000000000..c46224a20a --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/FilterTest.php @@ -0,0 +1,110 @@ + 1], ['a' => 2]], schema(int_schema('a'))); + + static::assertSame($rows, Filter::bind($rows->schema())->filter($rows, flow_context(config()))); + } + + public function test_empty_rows_stay_the_same_rows(): void + { + $rows = array_to_rows([], schema(int_schema('a'))); + + static::assertSame($rows, Filter::bind($rows->schema(), ref('a')->greaterThan(lit(0)))->filter( + $rows, + flow_context(config()), + )); + } + + public function test_a_predicate_every_row_matches_keeps_the_same_rows(): void + { + $rows = array_to_rows([['a' => 1], ['a' => 2]], schema(int_schema('a'))); + + static::assertSame($rows, Filter::bind($rows->schema(), ref('a')->greaterThan(lit(0)))->filter( + $rows, + flow_context(config()), + )); + } + + public function test_a_predicate_no_row_matches_keeps_no_row(): void + { + $rows = array_to_rows([['a' => 1], ['a' => 2]], schema(int_schema('a'))); + + static::assertSame( + 0, + Filter::bind($rows->schema(), ref('a')->greaterThan(lit(5))) + ->filter($rows, flow_context(config())) + ->count(), + ); + } + + public function test_a_null_result_drops_the_row_as_false_does(): void + { + $rows = array_to_rows([['a' => 1], ['a' => null], ['a' => 3]], schema(int_schema('a', nullable: true))); + + static::assertSame( + [['a' => 3]], + Filter::bind($rows->schema(), ref('a')->greaterThan(lit(1))) + ->filter($rows, flow_context(config())) + ->toArray(), + ); + } + + public function test_a_predicate_is_evaluated_only_on_the_rows_the_ones_before_it_kept(): void + { + $rows = array_to_rows([['a' => 0], ['a' => 2], ['a' => 10]], schema(int_schema('a'))); + + static::assertSame( + [['a' => 2]], + Filter::bind($rows->schema(), ref('a')->notEquals(lit(0)), lit(10)->divide(ref('a'))->greaterThan(lit(1))) + ->filter($rows, flow_context(config())) + ->toArray(), + ); + } + + public function test_a_reference_the_schema_lacks_is_refused(): void + { + $this->expectException(SchemaDefinitionNotFoundException::class); + + Filter::bind(schema(int_schema('a')), ref('missing')->equals(lit(1))); + } + + public function test_an_expanding_predicate_is_refused(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'filter() cannot contain array_expand(), it turns one row into many rows. Expand with withEntry() first, then use the new column.', + ); + + Filter::bind(ListColumnsMother::schema(), ref('tags')->expand()->equals(lit('x'))); + } + + public function test_a_predicate_returning_no_boolean_is_refused(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('filter() requires a predicate returning boolean'); + + Filter::bind(schema(int_schema('a')), ref('a')); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanEqualTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanEqualTest.php index 80daa88ae0..3cc7299f6b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanEqualTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanEqualTest.php @@ -11,6 +11,7 @@ use Flow\Types\Exception\InvalidArgumentException; use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\definition_from_type; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; @@ -22,6 +23,9 @@ use function Flow\ETL\DSL\time_schema; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_numeric_string; + +use const NAN; final class GreaterThanEqualTest extends FlowTestCase { @@ -217,4 +221,45 @@ public function test_greater_than_equal_time_intervals(): void schema(time_schema('v')), )); } + + public function test_greater_than_equal_orders_strings_by_bytes(): void + { + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('a')->greaterThanEqual(lit('10')), + ['a' => '9'], + schema(str_schema('a')), + )); + } + + public function test_greater_than_equal_orders_a_numeric_string_column_numerically(): void + { + $context = flow_context(); + + static::assertFalse((new FunctionContext($context))->eval( + ref('a')->greaterThanEqual(lit('10')), + ['a' => '9'], + schema(definition_from_type('a', type_numeric_string())), + )); + static::assertTrue((new FunctionContext($context))->eval( + ref('a')->greaterThanEqual(lit('18446744073709551614')), + ['a' => '18446744073709551615'], + schema(definition_from_type('a', type_numeric_string())), + )); + } + + public function test_greater_than_equal_is_false_against_nan_in_either_direction(): void + { + $context = flow_context(); + + static::assertFalse((new FunctionContext($context))->eval( + ref('a')->greaterThanEqual(lit(1.0)), + ['a' => NAN], + schema(float_schema('a')), + )); + static::assertFalse((new FunctionContext($context))->eval( + lit(1.0)->greaterThanEqual(ref('a')), + ['a' => NAN], + schema(float_schema('a')), + )); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanTest.php index a5d92c5ab2..d66561daa4 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanTest.php @@ -11,6 +11,7 @@ use Flow\Types\Exception\InvalidArgumentException; use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\definition_from_type; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; @@ -22,6 +23,9 @@ use function Flow\ETL\DSL\time_schema; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_numeric_string; + +use const NAN; final class GreaterThanTest extends FlowTestCase { @@ -227,4 +231,45 @@ public function test_greater_than_time_intervals(): void schema(time_schema('v')), )); } + + public function test_greater_than_orders_strings_by_bytes(): void + { + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('a')->greaterThan(lit('10')), + ['a' => '9'], + schema(str_schema('a')), + )); + } + + public function test_greater_than_orders_a_numeric_string_column_numerically(): void + { + $context = flow_context(); + + static::assertFalse((new FunctionContext($context))->eval( + ref('a')->greaterThan(lit('10')), + ['a' => '9'], + schema(definition_from_type('a', type_numeric_string())), + )); + static::assertTrue((new FunctionContext($context))->eval( + ref('a')->greaterThan(lit('18446744073709551614')), + ['a' => '18446744073709551615'], + schema(definition_from_type('a', type_numeric_string())), + )); + } + + public function test_greater_than_is_false_against_nan_in_either_direction(): void + { + $context = flow_context(); + + static::assertFalse((new FunctionContext($context))->eval( + ref('a')->greaterThan(lit(1.0)), + ['a' => NAN], + schema(float_schema('a')), + )); + static::assertFalse((new FunctionContext($context))->eval( + lit(1.0)->greaterThan(ref('a')), + ['a' => NAN], + schema(float_schema('a')), + )); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreatestTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreatestTest.php index 8bf15b7382..81a1383fef 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreatestTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreatestTest.php @@ -10,12 +10,15 @@ use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\definition_from_type; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\greatest; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\Types\DSL\type_numeric_string; final class GreatestTest extends FlowTestCase { @@ -89,4 +92,22 @@ public function test_greatest_with_null(): void schema(int_schema('int')), )); } + + public function test_greatest_orders_strings_by_bytes(): void + { + static::assertSame('9', (new FunctionContext(flow_context()))->eval( + greatest(ref('a'), lit('10'), lit('9')), + ['a' => null], + schema(str_schema('a', nullable: true)), + )); + } + + public function test_greatest_orders_a_numeric_string_column_numerically(): void + { + static::assertSame('10', (new FunctionContext(flow_context()))->eval( + greatest(ref('a'), ref('b')), + ['a' => '9', 'b' => '10'], + schema(definition_from_type('a', type_numeric_string()), definition_from_type('b', type_numeric_string())), + )); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LeastTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LeastTest.php index dab159f076..63729cfb2d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LeastTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LeastTest.php @@ -10,12 +10,15 @@ use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\definition_from_type; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\least; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\Types\DSL\type_numeric_string; final class LeastTest extends FlowTestCase { @@ -76,4 +79,22 @@ public function test_least_with_null(): void schema(int_schema('int')), )); } + + public function test_least_orders_strings_by_bytes(): void + { + static::assertSame('10', (new FunctionContext(flow_context()))->eval( + least(ref('a'), lit('10'), lit('9')), + ['a' => null], + schema(str_schema('a', nullable: true)), + )); + } + + public function test_least_orders_a_numeric_string_column_numerically(): void + { + static::assertSame('9', (new FunctionContext(flow_context()))->eval( + least(ref('a'), ref('b')), + ['a' => '9', 'b' => '10'], + schema(definition_from_type('a', type_numeric_string()), definition_from_type('b', type_numeric_string())), + )); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanEqualTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanEqualTest.php index 7f3b0d1620..2670ac50e0 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanEqualTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanEqualTest.php @@ -11,6 +11,7 @@ use Flow\Types\Exception\InvalidArgumentException; use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\definition_from_type; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; @@ -22,6 +23,9 @@ use function Flow\ETL\DSL\time_schema; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_numeric_string; + +use const NAN; final class LessThanEqualTest extends FlowTestCase { @@ -217,4 +221,45 @@ public function test_less_than_equal_time_intervals(): void schema(time_schema('v')), )); } + + public function test_less_than_equal_orders_strings_by_bytes(): void + { + static::assertFalse((new FunctionContext(flow_context()))->eval( + ref('a')->lessThanEqual(lit('10')), + ['a' => '9'], + schema(str_schema('a')), + )); + } + + public function test_less_than_equal_orders_a_numeric_string_column_numerically(): void + { + $context = flow_context(); + + static::assertTrue((new FunctionContext($context))->eval( + ref('a')->lessThanEqual(lit('10')), + ['a' => '9'], + schema(definition_from_type('a', type_numeric_string())), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('a')->lessThanEqual(lit('18446744073709551614')), + ['a' => '18446744073709551615'], + schema(definition_from_type('a', type_numeric_string())), + )); + } + + public function test_less_than_equal_is_false_against_nan_in_either_direction(): void + { + $context = flow_context(); + + static::assertFalse((new FunctionContext($context))->eval( + ref('a')->lessThanEqual(lit(1.0)), + ['a' => NAN], + schema(float_schema('a')), + )); + static::assertFalse((new FunctionContext($context))->eval( + lit(1.0)->lessThanEqual(ref('a')), + ['a' => NAN], + schema(float_schema('a')), + )); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanTest.php index e24c88e4c0..d1c19da8f6 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanTest.php @@ -11,6 +11,7 @@ use Flow\Types\Exception\InvalidArgumentException; use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\definition_from_type; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; @@ -22,6 +23,9 @@ use function Flow\ETL\DSL\time_schema; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_numeric_string; + +use const NAN; final class LessThanTest extends FlowTestCase { @@ -217,4 +221,45 @@ public function test_less_than_time_intervals(): void schema(time_schema('v')), )); } + + public function test_less_than_orders_strings_by_bytes(): void + { + static::assertFalse((new FunctionContext(flow_context()))->eval( + ref('a')->lessThan(lit('10')), + ['a' => '9'], + schema(str_schema('a')), + )); + } + + public function test_less_than_orders_a_numeric_string_column_numerically(): void + { + $context = flow_context(); + + static::assertTrue((new FunctionContext($context))->eval( + ref('a')->lessThan(lit('10')), + ['a' => '9'], + schema(definition_from_type('a', type_numeric_string())), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('a')->lessThan(lit('18446744073709551614')), + ['a' => '18446744073709551615'], + schema(definition_from_type('a', type_numeric_string())), + )); + } + + public function test_less_than_is_false_against_nan_in_either_direction(): void + { + $context = flow_context(); + + static::assertFalse((new FunctionContext($context))->eval( + ref('a')->lessThan(lit(1.0)), + ['a' => NAN], + schema(float_schema('a')), + )); + static::assertFalse((new FunctionContext($context))->eval( + lit(1.0)->lessThan(ref('a')), + ['a' => NAN], + schema(float_schema('a')), + )); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ValueOrderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ValueOrderTest.php new file mode 100644 index 0000000000..2c63dd6110 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ValueOrderTest.php @@ -0,0 +1,81 @@ +compare($left, $right)); + } + + #[TestWith(['9', '10', -1])] + #[TestWith(['18446744073709551615', '18446744073709551614', 1])] + #[TestWith(['100', '100', 0])] + public function test_a_numeric_string_operand_orders_strings_numerically( + string $left, + string $right, + int $expected, + ): void { + static::assertSame($expected, (new ValueOrder(type_optional(type_numeric_string()), type_string()))->compare( + $left, + $right, + )); + } + + public function test_an_order_without_operand_types_orders_strings_by_bytes(): void + { + static::assertSame(1, (new ValueOrder())->compare('9', '10')); + } + + public function test_numbers_order_as_php_compares_them(): void + { + $order = new ValueOrder(); + + static::assertSame(-1, $order->compare(1, 1.5)); + static::assertSame(0, $order->compare(2, 2.0)); + static::assertSame(1, $order->compare('5', 3)); + } + + public function test_nan_is_never_below_anything_in_either_direction(): void + { + $order = new ValueOrder(); + + static::assertSame(1, $order->compare(NAN, 1.0)); + static::assertSame(1, $order->compare(1.0, NAN)); + } + + public function test_date_times_order_by_instant(): void + { + static::assertSame(-1, (new ValueOrder())->compare( + new DateTimeImmutable('2024-01-01 10:00:00+00:00'), + new DateTimeImmutable('2024-01-01 11:30:00+01:00'), + )); + } + + public function test_intervals_order_by_length(): void + { + $order = new ValueOrder(); + + static::assertSame(-1, $order->compare(new DateInterval('PT59M'), new DateInterval('PT1H'))); + static::assertSame(0, $order->compare(new DateInterval('PT60M'), new DateInterval('PT1H'))); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/FilterWalkTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/FilterWalkTest.php index b1f9030bbf..1846332960 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/FilterWalkTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/FilterWalkTest.php @@ -61,7 +61,7 @@ public function test_a_transparent_preserving_chain_reaches_the_leaf(): void $read = NodeMother::read(); static::assertNotNull((new FilterWalk())->predicateAtLeaf( - new Filter(NodeMother::select($read), lit(true)), + new Filter(NodeMother::select($read, 'year'), lit(true)), $read, ref('year')->isNotNull(), )); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/JoinSidesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/JoinSidesTest.php index 3da06e64ad..9cfc78dda3 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/JoinSidesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/JoinSidesTest.php @@ -18,17 +18,17 @@ public function test_a_node_that_is_not_a_join_is_returned_as_it_is(): void { $limit = NodeMother::limit(NodeMother::read(), 5); - static::assertSame($limit, (new JoinSides(Optimizer::default(), NodeMother::context()))->of($limit)); + static::assertSame($limit, (new JoinSides(Optimizer::default(), NodeMother::context()))->of($limit, $limit)); } public function test_a_joins_right_side_is_optimized_as_its_own_plan(): void { $join = NodeMother::join(NodeMother::read(), NodeMother::plan(NodeMother::limit(NodeMother::read(), 3))->root); - $rewritten = (new JoinSides(Optimizer::default(), NodeMother::context()))->of($join); + $rewritten = (new JoinSides(Optimizer::default(), NodeMother::context()))->of($join, $join); static::assertInstanceOf(Join::class, $rewritten); - static::assertSame(3, (new LogicalPlan($rewritten->right()))->source()->limit()); + static::assertSame(3, (new LogicalPlan($rewritten->right()))->source()->scan()->limit); static::assertSame($join->children()[0], $rewritten->children()[0]); } @@ -39,16 +39,16 @@ public function test_a_cross_joins_right_side_is_optimized_as_its_own_plan(): vo NodeMother::plan(NodeMother::limit(NodeMother::read(), 3))->root, ); - $rewritten = (new JoinSides(Optimizer::default(), NodeMother::context()))->of($join); + $rewritten = (new JoinSides(Optimizer::default(), NodeMother::context()))->of($join, $join); static::assertInstanceOf(CrossJoin::class, $rewritten); - static::assertSame(3, (new LogicalPlan($rewritten->right()))->source()->limit()); + static::assertSame(3, (new LogicalPlan($rewritten->right()))->source()->scan()->limit); } public function test_a_join_whose_right_side_does_not_change_keeps_its_identity(): void { $join = NodeMother::crossJoin(NodeMother::read(), NodeMother::plan(NodeMother::read())->root); - static::assertSame($join, (new JoinSides(new Optimizer(), NodeMother::context()))->of($join)); + static::assertSame($join, (new JoinSides(new Optimizer(), NodeMother::context()))->of($join, $join)); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CountFromStatisticsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CountFromStatisticsTest.php index 1c8079b83b..9e4cafef44 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CountFromStatisticsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CountFromStatisticsTest.php @@ -9,6 +9,7 @@ use Flow\ETL\Extractor\Statistics; use Flow\ETL\Memory\ArrayMemory; use Flow\ETL\Optimizer\Rule\CountFromStatistics; +use Flow\ETL\Plan\Node\Read; use Flow\ETL\Plan\Trigger; use Flow\ETL\Tests\Double\DeclaringExtractor; use Flow\ETL\Tests\FlowTestCase; @@ -130,10 +131,62 @@ public function test_a_read_with_a_pushed_limit_is_counted_by_running(): void static::assertSame($plan, (new CountFromStatistics())->apply($plan, NodeMother::context())); } - public function test_a_read_with_a_pushed_partition_filter_is_counted_by_running(): void + public function test_a_parquet_read_of_exact_statistics_is_counted_from_them(): void { - $plan = Trigger::count->plan(NodeMother::read()->withPathFilter(new RejectingFilter())); + $memory = memory_filesystem(); + df() + ->read(from_sequence_number('id', 1, 25)) + ->write(to_parquet(path('memory://count/file.parquet'), filesystem: $memory)) + ->run(); + $plan = Trigger::count->plan(new Read(from_parquet(path('memory://count/file.parquet'), filesystem: $memory))); + + static::assertNotSame($plan, (new CountFromStatistics())->apply($plan, NodeMother::context())); + } + + public function test_a_read_with_a_pushed_path_filter_is_counted_by_running(): void + { + $memory = memory_filesystem(); + df() + ->read(from_sequence_number('id', 1, 25)) + ->write(to_parquet(path('memory://count/file.parquet'), filesystem: $memory)) + ->run(); + $plan = Trigger::count->plan((new Read(from_parquet( + path('memory://count/file.parquet'), + filesystem: $memory, + )))->withPathFilter(new RejectingFilter())); + + static::assertSame($plan, (new CountFromStatistics())->apply($plan, NodeMother::context())); + } + + public function test_a_read_with_a_pushed_filter_is_counted_by_running(): void + { + $memory = memory_filesystem(); + df() + ->read(from_sequence_number('id', 1, 25)) + ->write(to_parquet(path('memory://count/file.parquet'), filesystem: $memory)) + ->run(); + $plan = Trigger::count->plan((new Read(from_parquet( + path('memory://count/file.parquet'), + filesystem: $memory, + )))->withFilter(ref('id')->greaterThan(lit(20)))); + + static::assertSame($plan, (new CountFromStatistics())->apply($plan, NodeMother::context())); + } + + public function test_a_count_over_a_filtered_parquet_read_is_counted_by_running(): void + { + $memory = memory_filesystem(); + df() + ->read(from_sequence_number('id', 1, 25)) + ->write(to_parquet(path('memory://count/file.parquet'), filesystem: $memory)) + ->run(); + $frame = df() + ->read(from_parquet(path('memory://count/file.parquet'), filesystem: $memory)) + ->filter(ref('id')->greaterThan(lit(20))); + + $plan = $frame->explain(Trigger::count)->logical; + static::assertSame(5, $frame->count()); static::assertSame($plan, (new CountFromStatistics())->apply($plan, NodeMother::context())); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushFilterIntoSourceTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushFilterIntoSourceTest.php index 7c55ab75b5..7849c68f54 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushFilterIntoSourceTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushFilterIntoSourceTest.php @@ -6,9 +6,12 @@ use DateTimeImmutable; use Flow\ETL\Exception\EvaluationException; +use Flow\ETL\Exception\SchemaDefinitionNotFoundException; +use Flow\ETL\Function\All; use Flow\ETL\Memory\ArrayMemory; use Flow\ETL\Optimizer; use Flow\ETL\Optimizer\Rule\PushFilterIntoSource; +use Flow\ETL\Plan\Node; use Flow\ETL\Plan\Node\Filter; use Flow\ETL\Plan\Node\Read; use Flow\ETL\Plan\Stage; @@ -21,6 +24,7 @@ use Flow\ETL\WithEntry; use Flow\Filesystem\Path\Filter\Filters; use Flow\Filesystem\Path\Filter\OnlyFiles; +use PHPUnit\Framework\Attributes\DataProvider; use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; @@ -45,11 +49,11 @@ public function test_a_partition_only_predicate_is_pushed_into_the_source(): voi ->filter(ref('year')->equals(lit(2023))) ->fetch(); - $pathFilter = $extractor->pathFilters[0]; + $pathFilter = $extractor->scans[0]->pathFilter; static::assertInstanceOf(Filters::class, $pathFilter); static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07'))); static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07'))); - // the double reads every partition anyway, so only the Filter node the rule kept drops 2024 + // the double lists every partition, so only the scan's filter drops 2024 static::assertSame([2023], $rows->reduceToArray('year')); } @@ -63,7 +67,7 @@ public function test_two_stacked_partition_filters_are_both_pushed(): void ->filter(ref('month')->equals(lit('07'))) ->fetch(); - $pathFilter = $extractor->pathFilters[0]; + $pathFilter = $extractor->scans[0]->pathFilter; static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07'))); static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07'))); static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=08'))); @@ -80,7 +84,7 @@ public function test_three_stacked_partition_filters_all_reach_the_source(): voi ->filter(ref('month')->equals(lit('07'))) ->fetch(); - $pathFilter = $extractor->pathFilters[0]; + $pathFilter = $extractor->scans[0]->pathFilter; static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07'))); static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2022/month=07'))); static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07'))); @@ -97,38 +101,53 @@ public function test_a_body_predicate_below_a_partition_filter_does_not_block_it ->filter(ref('year')->equals(lit(2023))) ->fetch(); - $pathFilter = $extractor->pathFilters[0]; + $pathFilter = $extractor->scans[0]->pathFilter; static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07'))); static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07'))); static::assertSame(['a'], $rows->reduceToArray('value')); } - public function test_a_body_column_predicate_is_not_pushed(): void + public function test_a_body_column_predicate_is_pushed_as_a_filter_and_never_as_a_path_filter(): void { $extractor = PartitionedSourceMother::yearMonth(); df() ->read($extractor) - ->filter(ref('value')->equals(lit('a'))) + ->filter($predicate = ref('value')->equals(lit('a'))) + ->fetch(); + + static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter); + static::assertSame([$predicate], $extractor->scans[0]->conjuncts); + } + + public function test_a_predicate_referencing_no_column_is_pushed_as_a_filter_and_never_as_a_path_filter(): void + { + $extractor = PartitionedSourceMother::yearMonth(); + + df() + ->read($extractor) + ->filter($predicate = lit(1)->equals(lit(1))) ->fetch(); - static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]); + static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter); + static::assertSame([$predicate], $extractor->scans[0]->conjuncts); } - public function test_a_mixed_predicate_pushes_only_its_partition_conjunct(): void + public function test_a_mixed_predicate_pushes_both_conjuncts_and_only_the_partition_one_as_a_path_filter(): void { $extractor = PartitionedSourceMother::yearMonth(); $rows = df() ->read($extractor) - ->filter(ref('year')->equals(lit(2023))->and(ref('value')->notEquals(lit('a')))) + ->filter(($year = ref('year')->equals(lit(2023)))->and($value = ref('value')->notEquals(lit('a')))) ->fetch(); - $pathFilter = $extractor->pathFilters[0]; + $pathFilter = $extractor->scans[0]->pathFilter; static::assertInstanceOf(Filters::class, $pathFilter); static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07'))); static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07'))); - // the body conjunct still runs in the Filter node: the only 2023 row has value "a" + static::assertEquals([$year, $value], $extractor->scans[0]->conjuncts); + // the scan applies the body conjunct too: the only 2023 row has value "a" static::assertCount(0, $rows); } @@ -146,7 +165,7 @@ public function test_a_mixed_predicate_with_two_partition_conjuncts_pushes_both( ) ->fetch(); - $pathFilter = $extractor->pathFilters[0]; + $pathFilter = $extractor->scans[0]->pathFilter; static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07'))); static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07'))); static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=08'))); @@ -165,7 +184,7 @@ public function test_a_conjunct_that_is_not_deterministic_is_skipped_and_the_res ) ->fetch(); - $pathFilter = $extractor->pathFilters[0]; + $pathFilter = $extractor->scans[0]->pathFilter; static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07'))); static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07'))); } @@ -179,22 +198,23 @@ public function test_an_or_over_partition_columns_is_pushed_whole(): void ->filter(ref('year')->equals(lit(2023))->or(ref('year')->equals(lit(2024)))) ->fetch(); - $pathFilter = $extractor->pathFilters[0]; + $pathFilter = $extractor->scans[0]->pathFilter; static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07'))); static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07'))); static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2025/month=07'))); } - public function test_an_or_mixing_a_body_column_is_not_pushed(): void + public function test_an_or_mixing_a_body_column_is_pushed_whole_as_a_filter_and_never_as_a_path_filter(): void { $extractor = PartitionedSourceMother::yearMonth(); df() ->read($extractor) - ->filter(ref('year')->equals(lit(2023))->or(ref('value')->equals(lit('x')))) + ->filter($predicate = ref('year')->equals(lit(2023))->or(ref('value')->equals(lit('x')))) ->fetch(); - static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]); + static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter); + static::assertSame([$predicate], $extractor->scans[0]->conjuncts); } public function test_a_nested_and_inside_an_and_pushes_the_partition_conjuncts(): void @@ -211,7 +231,7 @@ public function test_a_nested_and_inside_an_and_pushes_the_partition_conjuncts() ) ->fetch(); - $pathFilter = $extractor->pathFilters[0]; + $pathFilter = $extractor->scans[0]->pathFilter; static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07'))); static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07'))); static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=08'))); @@ -228,7 +248,7 @@ public function test_a_limit_below_the_filter_blocks_the_push(): void ->filter(ref('year')->equals(lit(2023))) ->fetch(); - static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]); + static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter); } public function test_a_with_entry_redefining_the_partition_column_blocks_the_push(): void @@ -241,7 +261,7 @@ public function test_a_with_entry_redefining_the_partition_column_blocks_the_pus ->filter(ref('year')->equals(lit(2023))) ->fetch(); - static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]); + static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter); } public function test_a_duplicate_row_defining_the_partition_column_blocks_the_push(): void @@ -254,7 +274,7 @@ public function test_a_duplicate_row_defining_the_partition_column_blocks_the_pu ->filter(ref('year')->equals(lit(2023))) ->fetch(); - static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]); + static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter); // the 2024 row's duplicate carries year=2023: pruning its partition would lose it static::assertCount(3, $rows); } @@ -269,7 +289,7 @@ public function test_a_duplicate_row_defining_another_column_does_not_block_the_ ->filter(ref('year')->equals(lit(2023))) ->fetch(); - static::assertInstanceOf(Filters::class, $extractor->pathFilters[0]); + static::assertInstanceOf(Filters::class, $extractor->scans[0]->pathFilter); } public function test_a_filter_a_sink_root_does_not_consume_through_is_not_pushed(): void @@ -282,7 +302,7 @@ public function test_a_filter_a_sink_root_does_not_consume_through_is_not_pushed ->filter(ref('year')->equals(lit(2023))) ->fetch(); - static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]); + static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter); } public function test_a_non_file_extractor_is_never_pushed_into(): void @@ -294,7 +314,7 @@ public function test_a_non_file_extractor_is_never_pushed_into(): void static::assertSame($plan, (new PushFilterIntoSource())->apply($plan, NodeMother::context())); } - public function test_a_source_without_partition_columns_is_never_pushed_into(): void + public function test_a_data_conjunct_is_pushed_into_a_source_without_partition_columns(): void { $extractor = new RecordingFileExtractor( schema(int_schema('year')), @@ -303,10 +323,37 @@ public function test_a_source_without_partition_columns_is_never_pushed_into(): df() ->read($extractor) - ->filter(ref('year')->equals(lit(2023))) + ->filter($predicate = ref('year')->equals(lit(2023))) + ->fetch(); + + static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter); + static::assertSame([$predicate], $extractor->scans[0]->conjuncts); + } + + public function test_a_non_deterministic_data_conjunct_is_not_pushed(): void + { + $extractor = PartitionedSourceMother::yearMonth(); + + df() + ->read($extractor) + ->filter(ref('value')->equals(random_string(2))) + ->fetch(); + + static::assertSame([], $extractor->scans[0]->conjuncts); + } + + public function test_a_data_conjunct_under_a_reducing_node_is_not_pushed(): void + { + $extractor = PartitionedSourceMother::yearMonth(); + + df() + ->read($extractor) + ->limit(5) + ->filter(ref('value')->equals(lit('a'))) ->fetch(); - static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]); + static::assertSame(5, $extractor->scans[0]->limit); + static::assertSame([], $extractor->scans[0]->conjuncts); } public function test_a_non_deterministic_predicate_is_not_pushed(): void @@ -318,7 +365,7 @@ public function test_a_non_deterministic_predicate_is_not_pushed(): void ->filter(ref('month')->equals(random_string(2))) ->fetch(); - static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]); + static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter); } public function test_a_predicate_calling_a_user_callable_is_not_pushed(): void @@ -330,16 +377,26 @@ public function test_a_predicate_calling_a_user_callable_is_not_pushed(): void ->filter(ref('year')->call(lit('is_string'), type_boolean())) ->fetch(); - static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]); + static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter); } - public function test_a_literal_only_predicate_is_not_pushed(): void + public function test_a_constant_conjunct_is_pushed_and_stops_nothing_above_it(): void { $extractor = PartitionedSourceMother::yearMonth(); - df()->read($extractor)->filter(lit(true))->fetch(); + $rows = df() + ->read($extractor) + ->filter($constant = lit(true)) + ->filter($year = ref('year')->equals(lit(2023))) + ->fetch(); + + static::assertSame([$constant, $year], $extractor->scans[0]->conjuncts); + static::assertSame([2023], $rows->reduceToArray('year')); + } - static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]); + public function test_a_constant_false_conjunct_returns_no_row(): void + { + static::assertSame(0, df()->read(PartitionedSourceMother::yearMonth())->filter(lit(false))->fetch()->count()); } public function test_a_filter_inside_a_joined_frame_is_pushed_with_that_frames_context(): void @@ -351,7 +408,7 @@ public function test_a_filter_inside_a_joined_frame_is_pushed_with_that_frames_c ->join(df()->read($right)->filter(ref('year')->equals(lit(2023))), join_on(['id' => 'year'], 'r_')) ->fetch(); - $pathFilter = $right->pathFilters[0]; + $pathFilter = $right->scans[0]->pathFilter; static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07'))); static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07'))); } @@ -379,7 +436,7 @@ public function test_the_extractor_instance_is_never_mutated(): void $frame->explain()->toString(Stage::unoptimized), ); static::assertStringEndsWith( - "Extractor: RecordingFileExtractor\n Source: file:///dev/null\n Files: Filters", + "Extractor: RecordingFileExtractor\n Source: file:///dev/null\n Files: Filters\n Filter: Equals", $frame->explain()->toString(), ); } @@ -394,7 +451,7 @@ public function test_a_filter_on_a_renamed_partition_column_is_pushed_as_the_ori ->filter(ref('y')->equals(lit(2023))) ->fetch(); - $pathFilter = $extractor->pathFilters[0]; + $pathFilter = $extractor->scans[0]->pathFilter; static::assertInstanceOf(Filters::class, $pathFilter); static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07'))); static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07'))); @@ -411,23 +468,228 @@ public function test_a_filter_on_an_alias_of_a_partition_column_is_pushed_as_the ->filter(ref('y')->equals(lit(2023))) ->fetch(); - $pathFilter = $extractor->pathFilters[0]; + $pathFilter = $extractor->scans[0]->pathFilter; static::assertInstanceOf(Filters::class, $pathFilter); static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07'))); static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07'))); static::assertSame([2023], $rows->reduceToArray('y')); } - public function test_a_filter_on_an_alias_of_a_body_column_is_not_pushed(): void + public function test_a_filter_on_an_alias_of_a_body_column_is_pushed_as_the_original_column_and_never_as_a_path_filter(): void { $extractor = PartitionedSourceMother::yearMonth(); df() ->read($extractor) ->withEntry('v', ref('value')) - ->filter(ref('v')->equals(lit('a'))) + ->filter(ref('v')->equals($literal = lit('a'))) + ->fetch(); + + static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter); + static::assertEquals([ref('value')->equals($literal)], $extractor->scans[0]->conjuncts); + } + + public function test_a_filter_node_whose_only_conjunct_is_pushed_is_removed(): void + { + $plan = NodeMother::plan( + new Filter(new Read(PartitionedSourceMother::yearMonth()), $year = ref('year')->equals(lit(2023))), + ); + + $read = (new PushFilterIntoSource())->apply($plan, NodeMother::context())->root->children()[0]; + + static::assertInstanceOf(Read::class, $read); + static::assertSame([$year], $read->scan()->conjuncts); + } + + public function test_a_pushed_conjunct_leaves_its_filter_node_and_the_rest_stays_in_it(): void + { + $plan = NodeMother::plan( + new Filter( + new Read(PartitionedSourceMother::yearMonth()), + ($year = ref('year')->equals(lit(2023)))->and($random = ref('value')->equals(random_string(2))), + ), + ); + + $filter = (new PushFilterIntoSource())->apply($plan, NodeMother::context())->root->children()[0]; + + static::assertInstanceOf(Filter::class, $filter); + static::assertSame($random, $filter->function); + $read = $filter->children()[0]; + static::assertInstanceOf(Read::class, $read); + static::assertSame([$year], $read->scan()->conjuncts); + } + + public function test_stacked_filter_nodes_are_pushed_leaf_first(): void + { + $plan = NodeMother::plan( + new Filter( + new Filter(new Read(PartitionedSourceMother::yearMonth()), $year = ref('year')->equals(lit(2023))), + $month = ref('month')->equals(lit('07')), + ), + ); + + $read = (new PushFilterIntoSource())->apply($plan, NodeMother::context())->root->children()[0]; + + static::assertInstanceOf(Read::class, $read); + static::assertEquals([$year, $month], $read->scan()->conjuncts); + } + + public function test_a_filter_node_that_keeps_a_conjunct_stops_the_push_of_the_filters_above_it(): void + { + $plan = NodeMother::plan( + new Filter( + new Filter( + new Read(PartitionedSourceMother::yearMonth()), + ($year = ref('year')->equals(lit(2023)))->and($random = ref('value')->equals(random_string(2))), + ), + $month = ref('month')->equals(lit('07')), + ), + ); + + $top = (new PushFilterIntoSource())->apply($plan, NodeMother::context())->root->children()[0]; + + static::assertInstanceOf(Filter::class, $top); + static::assertSame($month, $top->function); + $bottom = $top->children()[0]; + static::assertInstanceOf(Filter::class, $bottom); + static::assertSame($random, $bottom->function); + $read = $bottom->children()[0]; + static::assertInstanceOf(Read::class, $read); + static::assertSame([$year], $read->scan()->conjuncts); + } + + public function test_a_filter_node_keeping_two_conjuncts_keeps_them_as_one_all(): void + { + $plan = NodeMother::plan( + new Filter( + new Read(PartitionedSourceMother::yearMonth()), + ref('year') + ->equals(lit(2023)) + ->and($first = ref('value')->equals(random_string(2))) + ->and($second = ref('value')->notEquals(random_string(3))), + ), + ); + + $filter = (new PushFilterIntoSource())->apply($plan, NodeMother::context())->root->children()[0]; + + static::assertInstanceOf(Filter::class, $filter); + static::assertEquals(new All($first, $second), $filter->function); + } + + public function test_a_limit_above_a_fully_pushed_filter_is_pushed_beside_it(): void + { + $extractor = PartitionedSourceMother::yearMonth(); + + $rows = df() + ->read($extractor) + ->filter($year = ref('year')->equals(lit(2023))) + ->limit(1) + ->fetch(); + + static::assertSame(1, $extractor->scans[0]->limit); + static::assertSame([$year], $extractor->scans[0]->conjuncts); + static::assertSame([2023], $rows->reduceToArray('year')); + } + + public function test_a_limit_above_a_filter_node_that_keeps_a_conjunct_is_not_pushed(): void + { + $extractor = PartitionedSourceMother::yearMonth(); + + df() + ->read($extractor) + ->filter(ref('year')->equals(lit(2023))->and(ref('value')->equals(random_string(2)))) + ->limit(1) + ->fetch(); + + static::assertNull($extractor->scans[0]->limit); + } + + public function test_a_kept_conjunct_keeps_every_conjunct_after_it_in_its_node(): void + { + $plan = NodeMother::plan( + new Filter( + new Read(PartitionedSourceMother::yearMonth()), + ($year = ref('year')->equals(lit(2023)))->and($random = ref('value')->equals(random_string(2)))->and( + $month = ref('month')->equals(lit('07')), + ), + ), + ); + + $filter = (new PushFilterIntoSource())->apply($plan, NodeMother::context())->root->children()[0]; + + static::assertInstanceOf(Filter::class, $filter); + static::assertEquals(new All($random, $month), $filter->function); + $read = $filter->children()[0]; + static::assertInstanceOf(Read::class, $read); + static::assertSame([$year], $read->scan()->conjuncts); + } + + /** + * @return iterable + */ + public static function nodes_removing_year(): iterable + { + $read = new Read(PartitionedSourceMother::yearMonth()); + + yield 'rename' => [new Node\Rename($read, 'year', 'y')]; + yield 'drop' => [new Node\Drop($read, ['year'])]; + yield 'select' => [new Node\Select($read, ['month', 'value'])]; + } + + #[DataProvider('nodes_removing_year')] + public function test_a_filter_on_a_column_a_node_removes_is_not_pushed_past_it(Node $node): void + { + $plan = NodeMother::plan(new Filter($node, $year = ref('year')->equals(lit(2023)))); + + $filter = (new PushFilterIntoSource())->apply($plan, NodeMother::context())->root->children()[0]; + + static::assertInstanceOf(Filter::class, $filter); + static::assertSame($year, $filter->function); + } + + public function test_a_filter_on_a_renamed_away_column_fails(): void + { + $this->expectException(SchemaDefinitionNotFoundException::class); + + df() + ->read(PartitionedSourceMother::yearMonth()) + ->rename('year', 'y') + ->filter(ref('year')->equals(lit(2023))) + ->fetch(); + } + + public function test_a_filter_on_a_dropped_column_fails(): void + { + $this->expectException(SchemaDefinitionNotFoundException::class); + + df() + ->read(PartitionedSourceMother::yearMonth()) + ->drop('year') + ->filter(ref('year')->equals(lit(2023))) + ->fetch(); + } + + public function test_a_filter_on_a_column_select_does_not_keep_fails(): void + { + $this->expectException(SchemaDefinitionNotFoundException::class); + + df() + ->read(PartitionedSourceMother::yearMonth()) + ->select('month') + ->filter(ref('year')->equals(lit(2023))) + ->fetch(); + } + + public function test_a_filter_on_a_column_select_keeps_is_pushed(): void + { + $extractor = PartitionedSourceMother::yearMonth(); + + df() + ->read($extractor) + ->select('year') + ->filter($year = ref('year')->equals(lit(2023))) ->fetch(); - static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]); + static::assertSame([$year], $extractor->scans[0]->conjuncts); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushLimitIntoSourceTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushLimitIntoSourceTest.php index e72ba207dc..579ddaa477 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushLimitIntoSourceTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushLimitIntoSourceTest.php @@ -42,7 +42,7 @@ public function test_a_limit_directly_above_the_read_is_pushed(): void NodeMother::context(), ); - static::assertSame(10, $plan->source()->limit()); + static::assertSame(10, $plan->source()->scan()->limit); } public function test_a_limit_passes_a_select(): void @@ -52,7 +52,7 @@ public function test_a_limit_passes_a_select(): void NodeMother::context(), ); - static::assertSame(10, $plan->source()->limit()); + static::assertSame(10, $plan->source()->scan()->limit); } #[DataProvider('expanding_expressions')] @@ -63,7 +63,7 @@ public function test_a_limit_does_not_pass_an_expand_column(ScalarFunction $func NodeMother::context(), ); - static::assertNull($plan->source()->limit()); + static::assertNull($plan->source()->scan()->limit); } public static function expanding_expressions(): Generator @@ -82,7 +82,7 @@ public function test_a_limit_does_not_pass_a_filter(): void NodeMother::context(), ); - static::assertNull($plan->source()->limit()); + static::assertNull($plan->source()->scan()->limit); } public function test_a_limit_does_not_pass_a_write(): void @@ -92,7 +92,7 @@ public function test_a_limit_does_not_pass_a_write(): void NodeMother::context(), ); - static::assertNull($plan->source()->limit()); + static::assertNull($plan->source()->scan()->limit); } public function test_a_blocker_discards_the_limit_collected_above_it_and_the_walk_continues(): void @@ -105,7 +105,7 @@ public function test_a_blocker_discards_the_limit_collected_above_it_and_the_wal NodeMother::context(), ); - static::assertSame(5, $plan->source()->limit()); + static::assertSame(5, $plan->source()->scan()->limit); } public function test_the_push_rewrites_the_leaf_and_keeps_the_spine(): void @@ -120,10 +120,10 @@ public function test_the_push_rewrites_the_leaf_and_keeps_the_spine(): void static::assertInstanceOf(Node\Limit::class, $limit); static::assertSame(10, $limit->limit); static::assertInstanceOf(Node\Select::class, $limit->children()[0]); - static::assertSame(10, $pushed->source()->limit()); + static::assertSame(10, $pushed->source()->scan()->limit); static::assertSame($extractor, $pushed->source()->extractor()); - static::assertNull($plan->source()->limit()); - static::assertSame([], $extractor->limits); + static::assertNull($plan->source()->scan()->limit); + static::assertSame([], $extractor->scans); } public function test_the_minimum_of_two_limits_is_pushed(): void @@ -133,7 +133,7 @@ public function test_the_minimum_of_two_limits_is_pushed(): void NodeMother::context(), ); - static::assertSame(3, $plan->source()->limit()); + static::assertSame(3, $plan->source()->scan()->limit); } public function test_a_limit_narrows_an_already_pushed_limit(): void @@ -143,7 +143,7 @@ public function test_a_limit_narrows_an_already_pushed_limit(): void NodeMother::context(), ); - static::assertSame(4, $plan->source()->limit()); + static::assertSame(4, $plan->source()->scan()->limit); } public function test_a_plan_without_a_limit_pushes_nothing(): void @@ -153,7 +153,7 @@ public function test_a_plan_without_a_limit_pushes_nothing(): void NodeMother::context(), ); - static::assertNull($plan->source()->limit()); + static::assertNull($plan->source()->scan()->limit); } public function test_a_row_input_chain_that_does_not_end_in_a_read_throws(): void @@ -185,7 +185,7 @@ public function test_a_transaction_contributes_one_walk_per_child(): void NodeMother::context(), ); - static::assertSame(3, $plan->source()->limit()); + static::assertSame(3, $plan->source()->scan()->limit); } public function test_one_transaction_child_whose_walk_yields_no_limit_blocks_the_push(): void @@ -207,7 +207,7 @@ public function test_one_transaction_child_whose_walk_yields_no_limit_blocks_the NodeMother::context(), ); - static::assertNull($plan->source()->limit()); + static::assertNull($plan->source()->scan()->limit); } public function test_a_limit_below_a_sink_is_pushed(): void @@ -219,7 +219,7 @@ public function test_a_limit_below_a_sink_is_pushed(): void NodeMother::context(), ); - static::assertSame(3, $plan->source()->limit()); + static::assertSame(3, $plan->source()->scan()->limit); } public function test_a_limit_above_a_sink_does_not_narrow_the_sink(): void @@ -233,7 +233,7 @@ public function test_a_limit_above_a_sink_does_not_narrow_the_sink(): void NodeMother::context(), ); - static::assertNull($plan->source()->limit()); + static::assertNull($plan->source()->scan()->limit); } public function test_the_widest_limit_of_every_root_is_pushed(): void @@ -247,7 +247,7 @@ public function test_the_widest_limit_of_every_root_is_pushed(): void NodeMother::context(), ); - static::assertSame(5, $plan->source()->limit()); + static::assertSame(5, $plan->source()->scan()->limit); } public function test_a_sink_whose_walk_yields_no_limit_blocks_the_push(): void @@ -267,7 +267,7 @@ public function test_a_sink_whose_walk_yields_no_limit_blocks_the_push(): void NodeMother::context(), ); - static::assertNull($plan->source()->limit()); + static::assertNull($plan->source()->scan()->limit); } public function test_the_push_reaches_the_sinks_through_the_shared_leaf(): void @@ -289,6 +289,6 @@ public function test_a_limit_above_an_offset_pushes_the_limit_plus_the_offset(): NodeMother::context(), ); - static::assertSame(110, $plan->source()->limit()); + static::assertSame(110, $plan->source()->scan()->limit); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/TopNRewriteTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/TopNRewriteTest.php index 053bafae5b..2985a2f83c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/TopNRewriteTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/TopNRewriteTest.php @@ -21,14 +21,14 @@ public function test_a_node_that_is_not_a_limit_is_returned_as_is(): void { $sort = NodeMother::sort(NodeMother::read()); - static::assertSame($sort, (new TopNRewrite())->of($sort)); + static::assertSame($sort, (new TopNRewrite())->of($sort, $sort)); } public function test_a_limit_over_anything_but_a_sort_is_returned_as_is(): void { $limit = NodeMother::limit(NodeMother::select(NodeMother::sort(NodeMother::read())), 5); - static::assertSame($limit, (new TopNRewrite())->of($limit)); + static::assertSame($limit, (new TopNRewrite())->of($limit, $limit)); } public function test_a_limit_over_a_sort_becomes_a_top_n_over_the_sorts_input_refs_and_limit(): void @@ -36,7 +36,8 @@ public function test_a_limit_over_a_sort_becomes_a_top_n_over_the_sorts_input_re $read = NodeMother::read(); $refs = refs(ref('id')); - $topN = (new TopNRewrite())->of(NodeMother::limit(new Sort($read, $refs), 5)); + $limit = NodeMother::limit(new Sort($read, $refs), 5); + $topN = (new TopNRewrite())->of($limit, $limit); static::assertInstanceOf(TopN::class, $topN); static::assertSame([$read], $topN->children()); @@ -48,10 +49,8 @@ public function test_a_limit_over_any_sort_is_rewritten_and_keeps_the_pinned_alg { $algorithm = external_sort()->memoryLimit(Unit::fromMb(1)); - $topN = (new TopNRewrite())->of(NodeMother::limit( - new Sort(NodeMother::read(), refs(ref('id')), $algorithm), - 1_000_000, - )); + $limit = NodeMother::limit(new Sort(NodeMother::read(), refs(ref('id')), $algorithm), 1_000_000); + $topN = (new TopNRewrite())->of($limit, $limit); static::assertInstanceOf(TopN::class, $topN); static::assertSame($algorithm, $topN->algorithm); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/OptimizerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/OptimizerTest.php index fa4487f34a..97075d094c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/OptimizerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/OptimizerTest.php @@ -57,14 +57,14 @@ public function test_an_optimizer_without_rules_returns_the_plan_it_was_given(): static::assertSame($plan, (new Optimizer())->optimize($plan, NodeMother::context())); } - public function test_default_registers_combine_limits_combine_sort_and_limit_push_limit_push_filter_then_count_from_statistics(): void + public function test_default_registers_combine_limits_combine_sort_and_limit_push_filter_push_limit_then_count_from_statistics(): void { static::assertSame( [ CombineLimits::class, CombineSortAndLimit::class, - PushLimitIntoSource::class, PushFilterIntoSource::class, + PushLimitIntoSource::class, CountFromStatistics::class, PushProjectionIntoExpand::class, ], diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/BoxLayoutTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/BoxLayoutTest.php index ce9a262b2c..db4411c12d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/BoxLayoutTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/BoxLayoutTest.php @@ -118,7 +118,7 @@ public function test_a_middle_child_branches_off_the_same_line(): void ┌─────────────┴─────────────┐┌─────────────┴─────────────┐┌─────────────┴─────────────┐ │ #1 Read ││ #1 Read ││ #4 Select │ │ ──────────────────── ││ (shared) ││ ──────────────────── │ - │ Extractor: ArrayExtractor ││ ││ Columns: id │ + │ Extractor: ArrayExtractor ││ ││ Keeps only: id │ └───────────────────────────┘└───────────────────────────┘└─────────────┬─────────────┘ ┌─────────────┴─────────────┐ │ #1 Read │ @@ -138,9 +138,9 @@ public function test_long_text_wraps_and_a_word_longer_than_the_box_is_split(): │ Rename: id → │ │ identifier_of_the_custome │ │ r_order │ - │ Defines columns: │ + │ Changes columns: │ │ identifier_of_the_custome │ - │ r_order │ + │ r_order, id │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ │ #1 Read │ diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/DetailsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/DetailsTest.php index 44e7139c9f..d1bf62490f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/DetailsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/DetailsTest.php @@ -16,6 +16,7 @@ use Flow\ETL\Plan\Node\Filter; use Flow\ETL\Plan\Node\Join as JoinNode; use Flow\ETL\Plan\Node\Offset; +use Flow\ETL\Plan\Node\Read; use Flow\ETL\Plan\Node\Rename; use Flow\ETL\Plan\Node\RenameEach; use Flow\ETL\Plan\Node\Result; @@ -26,6 +27,7 @@ use Flow\ETL\Plan\RequiredColumns; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\NodeMother; +use Flow\ETL\Tests\Mother\PartitionedSourceMother; use Flow\Filesystem\Path\Filter\KeepAll; use function Flow\ETL\DSL\hash_join; @@ -48,11 +50,23 @@ public function test_a_read_lists_its_extractor_only_when_nothing_was_pushed_int static::assertSame(['Extractor: ArrayExtractor'], (new Details())->lines(NodeMother::read())); } - public function test_a_read_lists_a_pushed_limit_and_file_filter(): void + public function test_a_read_lists_a_pushed_limit_file_filter_and_each_conjunct(): void { static::assertSame( - ['Extractor: ArrayExtractor', 'Limit: 3', 'Files: Filters'], - (new Details())->lines(NodeMother::read()->withLimit(3)->withPathFilter(new KeepAll())), + [ + 'Extractor: RecordingFileExtractor', + 'Source: file:///dev/null', + 'Limit: 3', + 'Files: Filters', + 'Filter: GreaterThan', + 'Filter: IsNotNull', + ], + (new Details())->lines( + (new Read(PartitionedSourceMother::yearMonth())) + ->withLimit(3) + ->withPathFilter(new KeepAll()) + ->withFilter(ref('year')->greaterThan(lit(1))->and(ref('month')->isNotNull())), + ), ); } @@ -80,7 +94,7 @@ public function test_a_join_labels_its_type_and_condition(): void 'Left on: id', 'Right on: id', 'Buffers all rows before passing them on', - 'Defines columns known only at run time', + 'Changes columns known only at run time', ], (new Details())->lines(NodeMother::join( NodeMother::read(), @@ -171,12 +185,12 @@ public function test_redefined_columns_follow_the_labelled_details(): void $details = new Details(); static::assertSame( - ['Column: n = Literal', 'Defines columns: n'], + ['Column: n = Literal', 'Changes columns: n'], $details->lines(new WithColumn($read, int_schema('n'), lit(1))), ); - static::assertSame(['Rename: n → m', 'Defines columns: m'], $details->lines(new Rename($read, 'n', 'm'))); + static::assertSame(['Rename: n → m', 'Changes columns: m, n'], $details->lines(new Rename($read, 'n', 'm'))); static::assertSame( - ['Defines columns known only at run time'], + ['Changes columns known only at run time'], $details->lines(new RenameEach($read, [rename_replace('_', '-')])), ); } @@ -188,7 +202,7 @@ public function test_declarations_are_the_row_count_transparency_materialization static::assertSame('preserving · transparent · blocking', $details->declarations(new Collect($read))); static::assertSame( - 'preserving · transparent · streaming · redefines m', + 'preserving · transparent · streaming · redefines m, n', $details->declarations(new Rename($read, 'n', 'm')), ); static::assertSame('preserving · transparent · streaming · redefines unknown', $details->declarations( diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/FlowLayoutTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/FlowLayoutTest.php index b6650dea73..d1ad96ed9c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/FlowLayoutTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/FlowLayoutTest.php @@ -72,7 +72,7 @@ public function test_every_reader_of_a_node_hangs_under_it(): void ├─ #3 Write │ Loader: MemoryLoader └─ #4 Select - │ Columns: id + │ Keeps only: id └─ #5 Write Loader: MemoryLoader PLAN, (new FlowLayout())->render((new Outline())->of($plan))); @@ -89,7 +89,7 @@ public function test_every_source_starts_its_own_tree_and_a_node_reached_again_i │ Extractor: ArrayExtractor └─ #3 CrossJoin │ Type: cross - │ Defines columns known only at run time + │ Changes columns known only at run time └─ #4 Result Rows this plan hands out: to the trigger, or to the node reading it #2 Read diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/PhysicalOutlineTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/PhysicalOutlineTest.php index d8d612ddeb..77c2935cad 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/PhysicalOutlineTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/PhysicalOutlineTest.php @@ -7,6 +7,7 @@ use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Plan\Explain\PhysicalOutline; use Flow\ETL\Plan\Explain\TreeLayout; +use Flow\ETL\Tests\Double\RecordingFileExtractor; use Flow\ETL\Tests\Double\UndescribableRowLessExtractor; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\PhysicalPlanMother; @@ -14,7 +15,11 @@ use function Flow\ETL\DSL\data_frame; use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\hash_join; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\join_on; +use function Flow\ETL\DSL\lit; +use function Flow\ETL\DSL\ref; +use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\to_output; final class PhysicalOutlineTest extends FlowTestCase @@ -112,6 +117,23 @@ public function test_a_pushed_limit_is_listed_under_the_source_that_was_handed_i ); } + public function test_a_pushed_filter_is_listed_under_the_file_source_that_was_offered_it(): void + { + $root = (new PhysicalOutline())->of(PhysicalPlanMother::of( + data_frame() + ->read(new RecordingFileExtractor(schema(int_schema('id')))) + ->filter(ref('id')->greaterThan(lit(1))), + )); + + static::assertSame( + [ + 'Extractor: RecordingFileExtractor', + ' Filter: GreaterThan', + ], + $root->children[0]->lines, + ); + } + public function test_a_plan_that_could_not_derive_its_schema_says_why(): void { $root = (new PhysicalOutline())->of(PhysicalPlanMother::reading(new UndescribableRowLessExtractor())); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/TreeLayoutTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/TreeLayoutTest.php index 3f51fdfc36..8c5991cf56 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/TreeLayoutTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/TreeLayoutTest.php @@ -91,7 +91,7 @@ public function test_every_child_but_the_last_keeps_the_rail_open(): void └─ #5 Write │ Loader: MemoryLoader └─ #4 Select - │ Columns: id + │ Keeps only: id └─ #1 Read (shared) PLAN, (new TreeLayout())->render((new Outline())->of($plan))); } @@ -105,8 +105,8 @@ public function test_with_declarations_a_single_chain_prints_one_line_per_node_w │ Rows this plan hands out: to the trigger, or to the node reading it └─ #3 Limit reducing · transparent · streaming │ Limit: 5 - └─ #2 Select preserving · transparent · streaming - │ Columns: id + └─ #2 Select preserving · transparent · streaming · redefines all but id + │ Keeps only: id └─ #1 Read source · transparent · streaming Extractor: ArrayExtractor PLAN, (new TreeLayout())->render((new Outline(declarations: true))->of($plan->root))); @@ -155,7 +155,7 @@ public function test_with_declarations_payloads_and_redefinitions(): void │ Until: Literal └─ #5 Offset reducing · transparent · streaming │ Skip: 2 - └─ #4 Rename preserving · transparent · streaming · redefines m + └─ #4 Rename preserving · transparent · streaming · redefines m, n │ Rename: n → m └─ #3 WithColumn preserving · transparent · streaming · redefines n │ Column: n = Literal @@ -202,8 +202,8 @@ public function test_with_declarations_a_run_trigger_keeps_a_result_beside_the_w Outputs preserving · opaque · streaming ├─ #3 Result preserving · transparent · streaming │ │ Rows this plan hands out: to the trigger, or to the node reading it - │ └─ #2 Select preserving · transparent · streaming - │ │ Columns: id + │ └─ #2 Select preserving · transparent · streaming · redefines all but id + │ │ Keeps only: id │ └─ #1 Read source · transparent · streaming │ Extractor: ArrayExtractor └─ #4 Write preserving · opaque · streaming @@ -241,8 +241,8 @@ public function test_with_declarations_a_transaction_root_child_lists_its_writes │ └─ #1 Read (shared) └─ #5 Write preserving · opaque · streaming │ Loader: MemoryLoader - └─ #4 Select preserving · transparent · streaming - │ Columns: id + └─ #4 Select preserving · transparent · streaming · redefines all but id + │ Keeps only: id └─ #1 Read (shared) PLAN, (new TreeLayout())->render((new Outline(declarations: true))->of($plan->root))); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/LogicalPlanTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/LogicalPlanTest.php index f3ba70577c..5c2f3c4cd7 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/LogicalPlanTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/LogicalPlanTest.php @@ -13,7 +13,7 @@ use Flow\ETL\Plan\Node\Result; use Flow\ETL\Plan\Node\Transaction; use Flow\ETL\Plan\Node\Write; -use Flow\ETL\Plan\ReplaceLeaf; +use Flow\ETL\Plan\ReplaceNodes; use Flow\ETL\Plan\Rewrite; use Flow\ETL\Tests\Double\ChildlessNode; use Flow\ETL\Tests\Double\RecordingTransaction; @@ -107,12 +107,12 @@ public function test_consumer_inputs_include_the_inputs_of_sinks_below_the_root( static::assertSame([$select, $limit], NodeMother::plan($select)->consumerInputs()); } - public function test_transform_up_with_replace_leaf_rewrites_the_leaf_and_keeps_the_spine(): void + public function test_transform_up_with_replace_nodes_rewrites_the_leaf_and_keeps_the_spine(): void { $replacement = NodeMother::read(from_array([['id' => 2]])); $plan = NodeMother::plan(NodeMother::select(NodeMother::read())); - $rewritten = $plan->transformUp(new ReplaceLeaf($plan->source(), $replacement)); + $rewritten = $plan->transformUp(ReplaceNodes::one($plan->source(), $replacement)); static::assertSame($replacement, $rewritten->source()); $select = $rewritten->root->children()[0]; @@ -148,9 +148,9 @@ public function test_transform_up_refuses_an_outputs_consumer_rewritten_to_a_non ); $plan->transformUp(new class implements Rewrite { - public function of(Node $node): Node + public function of(Node $original, Node $rebuilt): Node { - return $node instanceof Write ? $node->children()[0] : $node; + return $rebuilt instanceof Write ? $rebuilt->children()[0] : $rebuilt; } }); } @@ -186,9 +186,9 @@ public function test_transform_up_rebuilds_bottom_up(): void $plan = NodeMother::plan(NodeMother::limit(NodeMother::select($read), 5)); $rewritten = $plan->transformUp(new class implements Rewrite { - public function of(Node $node): Node + public function of(Node $original, Node $rebuilt): Node { - return $node instanceof Limit ? new Limit($node->children()[0], 1) : $node; + return $rebuilt instanceof Limit ? new Limit($rebuilt->children()[0], 1) : $rebuilt; } }); @@ -205,9 +205,9 @@ public function test_transform_up_that_changes_nothing_returns_the_same_plan(): $plan = NodeMother::plan(NodeMother::limit(NodeMother::select(NodeMother::read()), 5)); $rewritten = $plan->transformUp(new class implements Rewrite { - public function of(Node $node): Node + public function of(Node $original, Node $rebuilt): Node { - return $node; + return $rebuilt; } }); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/DropTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/DropTest.php index 494bd90af7..df0a4cb119 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/DropTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/DropTest.php @@ -44,6 +44,6 @@ public function test_declarations(): void static::assertSame(RowCount::preserving, $node->rowCount()); static::assertSame(Transparency::transparent, $node->transparency()); static::assertSame(Materialization::streaming, $node->materialization()); - static::assertEquals(Redefined::none(), $node->redefines()); + static::assertEquals(Redefined::names('id'), $node->redefines()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/ReadTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/ReadTest.php index 459547604d..f120798b2e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/ReadTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/ReadTest.php @@ -5,18 +5,21 @@ namespace Flow\ETL\Tests\Unit\Plan\Node; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Plan\Materialization; use Flow\ETL\Plan\Node\Read; use Flow\ETL\Plan\Redefined; use Flow\ETL\Plan\RowCount; use Flow\ETL\Plan\Transparency; use Flow\ETL\Tests\FlowTestCase; -use Flow\Filesystem\Path\Filter\Filters; +use Flow\ETL\Tests\Mother\PartitionedSourceMother; use Flow\Filesystem\Path\Filter\OnlyFiles; use Flow\Filesystem\Tests\Double\RejectingFilter; use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\lit; +use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\schema; final class ReadTest extends FlowTestCase @@ -37,62 +40,74 @@ public function test_a_default_read_pushes_no_limit_and_lists_only_files(): void { $read = new Read(from_array([['id' => 1]])); - static::assertNull($read->limit()); - static::assertInstanceOf(OnlyFiles::class, $read->pathFilter()); + static::assertNull($read->scan()->limit); + static::assertInstanceOf(OnlyFiles::class, $read->scan()->pathFilter); } - public function test_extractor_limit_and_path_filter_are_the_values_it_was_built_with(): void + public function test_extractor_and_scan_are_the_values_it_was_built_with(): void { $extractor = from_array([['id' => 1]]); - $filter = new RejectingFilter(); + $scan = new FileScan(5, new RejectingFilter()); - $read = new Read($extractor, 5, $filter); + $read = new Read($extractor, $scan); static::assertSame($extractor, $read->extractor()); - static::assertSame(5, $read->limit()); - static::assertSame($filter, $read->pathFilter()); + static::assertSame($scan, $read->scan()); } - public function test_with_path_filter_composes_with_the_filters_already_held(): void + public function test_with_path_filter_and_with_filter_push_into_the_scan_and_leave_the_original_alone(): void { - $extractor = from_array([['id' => 1]]); - $read = new Read($extractor, 3); - $first = new RejectingFilter(); - $second = new RejectingFilter(); + $extractor = PartitionedSourceMother::yearMonth(); + $read = new Read($extractor, new FileScan(3)); + $pathFilter = new RejectingFilter(); + $filter = ref('year')->greaterThan(lit(1)); - $filtered = $read->withPathFilter($first)->withPathFilter($second); + $pushed = $read->withPathFilter($pathFilter)->withFilter($filter); - static::assertInstanceOf(OnlyFiles::class, $read->pathFilter()); - static::assertEquals(new Filters(new OnlyFiles(), $first, $second), $filtered->pathFilter()); - static::assertSame(3, $filtered->limit()); - static::assertSame($extractor, $filtered->extractor()); + static::assertEquals(new FileScan(3), $read->scan()); + static::assertEquals( + (new FileScan(3)) + ->withPathFilter($pathFilter) + ->withFilter($filter), + $pushed->scan(), + ); + static::assertSame($extractor, $pushed->extractor()); } - public function test_with_limit_returns_a_new_read_and_leaves_the_original_alone(): void + public function test_with_limit_pushes_into_the_scan_of_any_extractor(): void { - $read = (new Read(from_array([['id' => 1]])))->withPathFilter($filter = new OnlyFiles()); + $read = new Read(from_array([['id' => 1]])); - $limited = $read->withLimit(5); + static::assertNull($read->scan()->limit); + static::assertSame(10, $read->withLimit(10)->withLimit(100)->scan()->limit); + } + + public function test_with_limit_refuses_zero_or_less(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Limit must be greater than 0'); - static::assertNull($read->limit()); - static::assertSame(5, $limited->limit()); - static::assertEquals(new Filters(new OnlyFiles(), $filter), $limited->pathFilter()); + (new Read(from_array([['id' => 1]])))->withLimit(0); } - public function test_with_limit_narrows_and_never_widens(): void + public function test_a_filter_is_refused_for_an_extractor_that_is_no_file_extractor(): void { - $read = new Read(from_array([['id' => 1]])); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'Only a FileExtractor reads a pushed filter, Flow\ETL\Extractor\ArrayExtractor does not', + ); - static::assertSame(10, $read->withLimit(10)->withLimit(100)->limit()); - static::assertSame(10, $read->withLimit(100)->withLimit(10)->limit()); + (new Read(from_array([['id' => 1]])))->withFilter(ref('id')->greaterThan(lit(1))); } - public function test_with_limit_refuses_zero_or_less(): void + public function test_a_path_filter_is_refused_for_an_extractor_that_is_no_file_extractor(): void { $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Limit must be greater than 0'); + $this->expectExceptionMessage( + 'Only a FileExtractor reads a pushed filter, Flow\ETL\Extractor\ArrayExtractor does not', + ); - (new Read(from_array([['id' => 1]])))->withLimit(0); + (new Read(from_array([['id' => 1]])))->withPathFilter(new RejectingFilter()); } public function test_schema_delegates_to_the_extractor(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/RenameTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/RenameTest.php index 5776bf92ca..72205a2f26 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/RenameTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/RenameTest.php @@ -45,6 +45,6 @@ public function test_declarations(): void static::assertSame(RowCount::preserving, $node->rowCount()); static::assertSame(Transparency::transparent, $node->transparency()); static::assertSame(Materialization::streaming, $node->materialization()); - static::assertEquals(Redefined::alias('user_id', 'id'), $node->redefines()); + static::assertEquals(Redefined::rename('id', 'user_id'), $node->redefines()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/SelectTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/SelectTest.php index f934c68ef9..845480c535 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/SelectTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/SelectTest.php @@ -44,6 +44,6 @@ public function test_declarations(): void static::assertSame(RowCount::preserving, $node->rowCount()); static::assertSame(Transparency::transparent, $node->transparency()); static::assertSame(Materialization::streaming, $node->materialization()); - static::assertEquals(Redefined::none(), $node->redefines()); + static::assertEquals(Redefined::allBut('id'), $node->redefines()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/RedefinedTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/RedefinedTest.php index 21ddb51400..b5e0442dae 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/RedefinedTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/RedefinedTest.php @@ -13,7 +13,8 @@ public function test_none_defines_nothing(): void { static::assertFalse(Redefined::none()->defines('year')); static::assertSame([], Redefined::none()->names); - static::assertFalse(Redefined::none()->unknown); + static::assertFalse(Redefined::none()->isUnknown()); + static::assertFalse(Redefined::none()->isAllBut()); } public function test_names_define_exactly_those_names_and_alias_none(): void @@ -30,7 +31,8 @@ public function test_unknown_defines_every_name_and_aliases_none(): void { static::assertTrue(Redefined::unknown()->defines('anything')); static::assertNull(Redefined::unknown()->aliasOf('anything')); - static::assertTrue(Redefined::unknown()->unknown); + static::assertTrue(Redefined::unknown()->isUnknown()); + static::assertFalse(Redefined::unknown()->isAllBut()); } public function test_names_with_no_names_defines_nothing(): void @@ -49,4 +51,27 @@ public function test_an_alias_defines_its_name_and_points_it_at_the_column_below static::assertNull($redefined->aliasOf('year')); static::assertSame(['y'], $redefined->names); } + + public function test_all_but_defines_every_name_except_the_kept_ones(): void + { + $redefined = Redefined::allBut('year'); + + static::assertFalse($redefined->defines('year')); + static::assertTrue($redefined->defines('month')); + static::assertNull($redefined->aliasOf('month')); + static::assertSame(['year'], $redefined->names); + static::assertTrue($redefined->isAllBut()); + static::assertFalse($redefined->isUnknown()); + } + + public function test_a_rename_defines_both_names_and_points_the_new_one_at_the_old_one(): void + { + $redefined = Redefined::rename('year', 'y'); + + static::assertTrue($redefined->defines('y')); + static::assertTrue($redefined->defines('year')); + static::assertSame('year', $redefined->aliasOf('y')); + static::assertNull($redefined->aliasOf('year')); + static::assertSame(['y', 'year'], $redefined->names); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/ReplaceLeafTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/ReplaceLeafTest.php deleted file mode 100644 index 720dace269..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/ReplaceLeafTest.php +++ /dev/null @@ -1,27 +0,0 @@ -of($target)); - } - - public function test_any_other_node_is_returned_unchanged(): void - { - $other = NodeMother::select(NodeMother::read()); - - static::assertSame($other, (new ReplaceLeaf(NodeMother::read(), NodeMother::read()))->of($other)); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/ReplaceNodesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/ReplaceNodesTest.php new file mode 100644 index 0000000000..3853d01c2f --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/ReplaceNodesTest.php @@ -0,0 +1,101 @@ +of($target, $target)); + static::assertSame($other, $rewrite->of($other, $other)); + } + + public function test_a_replacement_takes_the_targets_rewritten_children(): void + { + $target = NodeMother::select(NodeMother::read()); + $rebuilt = $target->withChildren([$read = NodeMother::read()]); + + $replaced = ReplaceNodes::one($target, NodeMother::select($target->children()[0], 'name'))->of( + $target, + $rebuilt, + ); + + static::assertSame([$read], $replaced->children()); + } + + public function test_a_removed_node_gives_way_to_its_rewritten_child(): void + { + $target = new Filter(NodeMother::read(), lit(true)); + $rebuilt = $target->withChildren([$read = NodeMother::read()]); + + /** @var SplObjectStorage $replacements */ + $replacements = new SplObjectStorage(); + + static::assertSame($read, (new ReplaceNodes($replacements, [$target]))->of($target, $rebuilt)); + } + + public function test_a_target_replaced_by_its_own_child_keeps_the_childs_rewritten_children(): void + { + $child = NodeMother::select(NodeMother::read()); + $target = new Filter($child, lit(true)); + $rebuilt = $target->withChildren([NodeMother::select(NodeMother::read())]); + + static::assertSame($rebuilt->children(), ReplaceNodes::one($target, $child)->of($target, $rebuilt)->children()); + } + + public function test_a_node_of_other_than_one_child_is_not_removed(): void + { + $target = NodeMother::read(); + /** @var SplObjectStorage $replacements */ + $replacements = new SplObjectStorage(); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Only a node of one child gives way to it, ' . $target::class . ' has 0'); + + (new ReplaceNodes($replacements, [$target]))->of($target, $target); + } + + public function test_a_node_both_replaced_and_removed_is_refused(): void + { + $target = new Filter(NodeMother::read(), lit(true)); + /** @var SplObjectStorage $replacements */ + $replacements = new SplObjectStorage(); + $replacements[$target] = NodeMother::read(); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('A node is either replaced or removed, ' . Filter::class . ' is both'); + + new ReplaceNodes($replacements, [$target]); + } + + public function test_several_targets_are_replaced_in_one_rewrite(): void + { + $first = NodeMother::read(); + $second = NodeMother::read(); + /** @var SplObjectStorage $replacements */ + $replacements = new SplObjectStorage(); + $replacements[$first] = $firstReplacement = NodeMother::read(); + $replacements[$second] = $secondReplacement = NodeMother::read(); + $rewrite = new ReplaceNodes($replacements); + + static::assertSame($firstReplacement, $rewrite->of($first, $first)); + static::assertSame($secondReplacement, $rewrite->of($second, $second)); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/TransformUpTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/TransformUpTest.php index 94628a6ba5..c74e53d68f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/TransformUpTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/TransformUpTest.php @@ -34,11 +34,11 @@ public function __construct( private array &$visited, ) {} - public function of(Node $node): Node + public function of(Node $original, Node $rebuilt): Node { - $this->visited[] = $node::class; + $this->visited[] = $rebuilt::class; - return $node; + return $rebuilt; } }); @@ -50,9 +50,9 @@ public function test_a_rewrite_that_changes_nothing_returns_the_same_root_object $root = NodeMother::limit(NodeMother::select(NodeMother::read()), 5); static::assertSame($root, (new TransformUp())->of($root, new class implements Rewrite { - public function of(Node $node): Node + public function of(Node $original, Node $rebuilt): Node { - return $node; + return $rebuilt; } })); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/PipelineSplitTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/PipelineSplitTest.php index 3ec9d45a48..62a19a2452 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/PipelineSplitTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/PipelineSplitTest.php @@ -665,8 +665,8 @@ public function test_the_leaf_pipeline_carries_the_reads_limit_and_path_filter() $plan = (new PipelineSplit())->of($logical, $planned, NodeMother::context()); - static::assertSame(5, $plan->root()->input()?->limit()); - static::assertNull($plan->root()->limit()); + static::assertSame(5, $plan->root()->input()?->scan()->limit); + static::assertNull($plan->root()->scan()->limit); } public function test_only_the_pipeline_that_reads_the_source_counts_its_rows(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/PlannerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/PlannerTest.php index 191d1d00aa..e5feb8008f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/PlannerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/PlannerTest.php @@ -90,7 +90,7 @@ public function test_the_default_rules_keep_every_sink_on_the_spine(): void ); $steps = PipelineSteps::of($plan->root()->segments()); - static::assertSame(4, $plan->root()->limit()); + static::assertSame(4, $plan->root()->scan()->limit); static::assertSame( [SinkFeed::class, LimitTransformer::class, MemoryLoader::class, SelectEntriesTransformer::class], array_map(static fn($step) => $step::class, $steps), @@ -141,8 +141,8 @@ public function test_a_linear_plan_is_one_pipeline_with_the_limit_pushed(): void static::assertSame(['id', 'total'], $plan->schema->references()->names()); static::assertSame($csv, $plan->root()->segments()->extractor()); - static::assertSame(5, $plan->root()->limit()); - static::assertNull($logical->source()->limit()); + static::assertSame(5, $plan->root()->scan()->limit); + static::assertNull($logical->source()->scan()->limit); } public function test_a_run_plan_over_one_sink_is_described_with_the_sinks_input_schema(): void @@ -189,8 +189,8 @@ public function test_a_sort_splits_the_plan_into_two_pipelines(): void static::assertSame(['id', 'total', 'seller_id'], $plan->schema->references()->names()); static::assertSame($csv, $upstream->segments()->extractor()); - static::assertNull($upstream->limit()); - static::assertNull($plan->root()->limit()); + static::assertNull($upstream->scan()->limit); + static::assertNull($plan->root()->scan()->limit); } public function test_a_join_each_plan_refuses_its_schema_and_has_no_frame_edge(): void @@ -234,7 +234,7 @@ public function test_a_planner_with_no_rules_plans_without_a_push(): void $plan = (new Planner())->plan($logical, NodeMother::context()); static::assertSame($csv, $plan->root()->segments()->extractor()); - static::assertNull($plan->root()->limit()); + static::assertNull($plan->root()->scan()->limit); } public function test_a_join_plans_its_right_side_as_a_physical_plan_of_its_own(): void @@ -316,7 +316,7 @@ public function test_a_read_frame_is_a_source_of_this_pipeline(): void PipelineSteps::classes($plan->root()->segments()), ); static::assertSame($extractor, $plan->root()->segments()->extractor()); - static::assertSame(5, $plan->root()->limit()); + static::assertSame(5, $plan->root()->scan()->limit); } public function test_a_read_frame_describes_the_rows_without_reading_them(): void @@ -412,10 +412,7 @@ public function test_a_limit_over_a_read_frame_is_pushed_into_its_extractor(): v static::assertSame( 1, - (new Planner(Optimizer::default())) - ->plan($logical, NodeMother::context()) - ->root() - ->limit(), + (new Planner(Optimizer::default()))->plan($logical, NodeMother::context())->root()->scan()->limit, ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/StringDefinitionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/StringDefinitionTest.php index 2fdc52f2c5..3f66734e82 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/StringDefinitionTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/StringDefinitionTest.php @@ -13,9 +13,14 @@ use PHPUnit\Framework\Attributes\DataProvider; use function Flow\ETL\DSL\bool_schema; +use function Flow\ETL\DSL\definition_from_array; +use function Flow\ETL\DSL\definition_from_type; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\null_schema; use function Flow\ETL\DSL\string_schema; +use function Flow\Types\DSL\type_equals; +use function Flow\Types\DSL\type_numeric_string; +use function Flow\Types\DSL\type_string; final class StringDefinitionTest extends FlowTestCase { @@ -67,6 +72,25 @@ public static function provideMergeCases(): Generator ]; } + public function test_a_numeric_string_definition_keeps_its_type_through_every_rebuild(): void + { + $def = definition_from_type('id', type_numeric_string()); + + static::assertInstanceOf(StringDefinition::class, $def); + static::assertTrue(type_equals(type_numeric_string(), $def->type())); + static::assertTrue(type_equals(type_numeric_string(), $def->makeNullable()->type())); + static::assertTrue(type_equals(type_numeric_string(), $def->rename('other')->type())); + static::assertTrue(type_equals(type_numeric_string(), $def->setMetadata(Metadata::with('k', 'v'))->type())); + static::assertTrue(type_equals(type_numeric_string(), $def->addMetadata('k', 'v')->type())); + static::assertTrue(type_equals( + type_numeric_string(), + $def->merge(definition_from_type('id', type_numeric_string()))->type(), + )); + static::assertTrue(type_equals(type_numeric_string(), $def->merge(null_schema('id'))->type())); + static::assertTrue(type_equals(type_string(), $def->merge(string_schema('id'))->type())); + static::assertTrue($def->isSame(definition_from_array($def->normalize()))); + } + public function test_add_metadata(): void { $def = string_schema('name'); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Formatter/PHPSchemaFormatterTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Formatter/PHPSchemaFormatterTest.php index 4a7d3f8ccb..9eda4c6c97 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Formatter/PHPSchemaFormatterTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Formatter/PHPSchemaFormatterTest.php @@ -12,6 +12,7 @@ use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\date_schema; use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\definition_from_type; use function Flow\ETL\DSL\enum_schema; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\html_element_schema; @@ -31,6 +32,7 @@ use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_list; use function Flow\Types\DSL\type_map; +use function Flow\Types\DSL\type_numeric_string; use function Flow\Types\DSL\type_optional; use function Flow\Types\DSL\type_string; use function Flow\Types\DSL\type_structure; @@ -88,6 +90,15 @@ public function test_formatting_map_schema(): void )))); } + public function test_formatting_numeric_string_schema(): void + { + static::assertEquals(<<<'PHP' + \Flow\ETL\DSL\schema( + \Flow\ETL\DSL\definition_from_type("id", type: \Flow\Types\DSL\type_numeric_string(), nullable: true, metadata: \Flow\ETL\DSL\schema_metadata()), + ); + PHP, (new PHPSchemaFormatter())->format(schema(definition_from_type('id', type_numeric_string(), true)))); + } + public function test_formatting_simple_schema(): void { static::assertEquals(<<explain()->toString(format: Format::declarations)); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionFilterTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionFilterTransformerTest.php index 313fda2876..68ee8e404a 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionFilterTransformerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionFilterTransformerTest.php @@ -231,4 +231,18 @@ public function test_bind_refuses_a_root_expand_over_booleans(): void (new ScalarFunctionFilterTransformer(ref('flags')->expand()))->bind(ListColumnsMother::schema()); } + + public function test_the_bound_transformer_keeps_the_rows_its_predicate_matches(): void + { + $input = schema(int_schema('a')); + $bound = (new ScalarFunctionFilterTransformer(ref('a')->greaterThan(lit(1))))->bind($input)->step; + + static::assertInstanceOf(ScalarFunctionFilterTransformer::class, $bound); + static::assertSame( + [['a' => 2], ['a' => 3]], + $bound + ->transform(array_to_rows([['a' => 1], ['a' => 2], ['a' => 3]], $input), flow_context(config())) + ->toArray(), + ); + } } diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeExtractorTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeExtractorTest.php index 6dc5defddd..3510f1bdda 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeExtractorTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeExtractorTest.php @@ -6,6 +6,7 @@ use Flow\ETL\Exception\InferredSchemaException; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Extractor\File\FileScan; use Flow\ETL\Extractor\Signal; use Flow\ETL\Tests\Double\CountingFilesystem; use Flow\ETL\Tests\Double\SpyBackend; @@ -489,7 +490,10 @@ public function test_a_path_filter_narrows_the_read_but_not_the_schema(): void $extractor = from_floe(path('memory://parts/*/*.floe'), filesystem: $counting); $schema = $extractor->schema(); - $batches = iterator_to_array($extractor->extract(flow_context(config()), pathFilter: new RejectingFilter())); + $batches = iterator_to_array($extractor->scan( + flow_context(config()), + new FileScan(pathFilter: new RejectingFilter()), + )); static::assertSame([], $batches); static::assertSame(1, $counting->readFromCalls); diff --git a/src/extension/arrow-ext/Cargo.lock b/src/extension/arrow-ext/Cargo.lock index d69bb02fa8..1e7d3c2577 100644 --- a/src/extension/arrow-ext/Cargo.lock +++ b/src/extension/arrow-ext/Cargo.lock @@ -44,15 +44,15 @@ dependencies = [ [[package]] name = "alloc-no-stdlib" -version = "2.0.4" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" +checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a" [[package]] name = "alloc-stdlib" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1" dependencies = [ "alloc-no-stdlib", ] @@ -87,9 +87,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e5f6adeffdf587d7a31db5d2266189624b526730cd3627f9ff9fedae97ad584" +checksum = "51dff6e4b9f158864a0aeb6a131ad53858ecbae7a2fc8989307b9b3c6b2f122e" dependencies = [ "ahash", "arrow-buffer", @@ -106,9 +106,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "097d193003ce7995d5d087089069ec2a6e0187faf5a6f8c9f38af2645d987182" +checksum = "c7235b863533e9db3ab86905b4521251de11226275b436ff3cb2d0010558d1c5" dependencies = [ "bytes", "half", @@ -116,11 +116,22 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-cmp" +version = "60.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73a0e9adb28d95739fabcdae637261a2c14e618caf58ba1b88857120f9ee2687" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", +] + [[package]] name = "arrow-data" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba2f832eaeca24b8f26143dba750e42ee4ab51cf7d65e701ca9607cfda9f358" +checksum = "5aa06d5e22786c4966ffc4d2f171d0cb65d95f6bdea108a14e8c335e9faa437f" dependencies = [ "arrow-buffer", "arrow-schema", @@ -131,9 +142,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcc41681ea80f521df14c36725b74d4c60702c47f0793af2be469c04527e2599" +checksum = "86c523472d22e31f984df7a44fc36e6323d3c9b05ffe53d267ee4c3aa7c128d2" dependencies = [ "arrow-array", "arrow-buffer", @@ -145,9 +156,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd" +checksum = "f85a9a58cd03560526b0066487574aacb9008e413504fa0ee23d61141e96c338" dependencies = [ "bitflags", "serde_core", @@ -156,13 +167,14 @@ dependencies = [ [[package]] name = "arrow-select" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc58569193c2525915f3cc6310edba3792f1200f65d6e9ed330aa33e691493b8" +checksum = "ba3482b7d03bf4a3bf0b195f08a7f2b071595ccd54897e35b975ac63d3104977" dependencies = [ "ahash", "arrow-array", "arrow-buffer", + "arrow-cmp", "arrow-data", "arrow-schema", "num-traits", @@ -207,7 +219,7 @@ dependencies = [ "regex", "rustc-hash", "shlex 2.0.1", - "syn 2.0.117", + "syn 3.0.6", ] [[package]] @@ -227,9 +239,9 @@ dependencies = [ [[package]] name = "brotli" -version = "8.0.2" +version = "9.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +checksum = "f8b851b75c23ca7873623d612fe49bd1989aeb03d08fb9432187eb253d3d4c6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -238,9 +250,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "5.0.0" +version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +checksum = "941cd9bd4ddab83cb46fa5a2d428f1c857b24ac78cb876cf7beb710840934bd7" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1115,9 +1127,9 @@ dependencies = [ [[package]] name = "parquet" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff322f54b1a0f9288e614ed1f2d329b380af5476420db19f46ffb865e1163d73" +checksum = "8af83d2940bc0510f9aef86d865f56fdc6095f87ab115ac885a80b7c5226d3ba" dependencies = [ "ahash", "arrow-array", @@ -1141,7 +1153,7 @@ dependencies = [ "simdutf8", "snap", "twox-hash", - "zstd", + "zstd 0.14.2", ] [[package]] @@ -2021,7 +2033,7 @@ dependencies = [ "typed-path", "zeroize", "zopfli", - "zstd", + "zstd 0.13.3", ] [[package]] @@ -2054,7 +2066,16 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe", + "zstd-safe 7.2.4", +] + +[[package]] +name = "zstd" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "057cfd910cfac363a0ada849592624b4c9ff2e10bef504c3433810d78ed96f93" +dependencies = [ + "zstd-safe 8.1.0", ] [[package]] @@ -2066,11 +2087,20 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "8.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd44c6a7284e91f3717755b24315a302edd9153a01f753c3cba3d765e8eafac" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.1+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "aeec9eaf2dffbbd09201e23bd0ffcbaa33bb8e9266a10734fd7ed90a85eca078" dependencies = [ "cc", "pkg-config", diff --git a/src/extension/arrow-ext/Cargo.toml b/src/extension/arrow-ext/Cargo.toml index e528e0f770..0bbb474083 100644 --- a/src/extension/arrow-ext/Cargo.toml +++ b/src/extension/arrow-ext/Cargo.toml @@ -8,10 +8,10 @@ crate-type = ["cdylib"] [dependencies] ext-php-rs = "0.16" -parquet = { version = "59", default-features = false, features = ["arrow", "arrow_canonical_extension_types", "snap", "flate2", "flate2-zlib-rs", "zstd", "lz4", "brotli", "simdutf8"] } -arrow-array = { version = "59", features = ["ffi"] } -arrow-buffer = "59" -arrow-schema = { version = "59", features = ["canonical_extension_types"] } +parquet = { version = "60", default-features = false, features = ["arrow", "arrow_canonical_extension_types", "snap", "flate2", "flate2-zlib-rs", "zstd", "lz4", "brotli", "simdutf8"] } +arrow-array = { version = "60", features = ["ffi"] } +arrow-buffer = "60" +arrow-schema = { version = "60", features = ["canonical_extension_types"] } bytes = "1" [build-dependencies] diff --git a/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustBatchReader.php b/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustBatchReader.php index 6ea532f5c8..473d266161 100644 --- a/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustBatchReader.php +++ b/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustBatchReader.php @@ -7,6 +7,7 @@ use Flow\Arrow\RustArrowSchema; use Flow\Arrow\RustParquetBatch; use Flow\Parquet\Engine\RustParquetFileReader; +use Flow\Parquet\Reader\ReadPlan; use RuntimeException; use function extension_loaded; @@ -21,7 +22,7 @@ final class RustBatchReader * @param list $columns root column names * @param int<1, max> $batchSize */ - public function __construct(RustParquetFileReader $file, array $columns, int $batchSize, ?int $offset, ?int $limit) + public function __construct(RustParquetFileReader $file, array $columns, int $batchSize, ReadPlan $plan) { throw new RuntimeException('arrow extension is not loaded'); } diff --git a/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustColumnsReader.php b/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustColumnsReader.php index bf7ee0a131..c39b3400d5 100644 --- a/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustColumnsReader.php +++ b/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustColumnsReader.php @@ -5,6 +5,7 @@ namespace Flow\Arrow\Parquet; use Flow\Parquet\Engine\RustParquetFileReader; +use Flow\Parquet\Reader\ReadPlan; use Iterator; use RuntimeException; @@ -25,7 +26,7 @@ final class RustColumnsReader implements Iterator * @param list $columns root names or struct paths * @param int<1, max> $batchSize */ - public function __construct(RustParquetFileReader $file, array $columns, int $batchSize, ?int $offset, ?int $limit) + public function __construct(RustParquetFileReader $file, array $columns, int $batchSize, ReadPlan $plan) { throw new RuntimeException('arrow extension is not loaded'); } diff --git a/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFileReader.php b/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFileReader.php index 7e041d8abc..38e400fded 100644 --- a/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFileReader.php +++ b/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFileReader.php @@ -6,10 +6,8 @@ use Flow\Arrow\Parquet\RustColumnsReader; use Flow\Filesystem\SourceStream; -use Flow\Parquet\ParquetFile\Metadata; -use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFileReader; -use Flow\Parquet\ThriftModel\FileMetaData; +use Flow\Parquet\Reader\ReadPlan; use RuntimeException; use function extension_loaded; @@ -30,49 +28,27 @@ public function __construct(SourceStream $stream, bool $int96AsDatetime = true) } /** - * Closes the stream; any later call throws \Flow\Parquet\Exception\RuntimeException('Reader is not open'). + * Closes the stream; any later call throws \Flow\Parquet\Exception\RuntimeException('Reader is not open'). A + * footer taken before stays readable. */ public function close(): void { throw new RuntimeException('arrow extension is not loaded'); } - public function metadata(): Metadata - { - throw new RuntimeException('arrow extension is not loaded'); - } - - /** - * @param list $columns resolved names, never empty - * @param int<1, max> $batchSize upper bound of rows per chunk - */ - public function readColumns(array $columns, int $batchSize, ?int $limit, ?int $offset): RustColumnsReader - { - throw new RuntimeException('arrow extension is not loaded'); - } - - public function rowsNumber(): int - { - throw new RuntimeException('arrow extension is not loaded'); - } - - public function schema(): Schema - { - throw new RuntimeException('arrow extension is not loaded'); - } - /** - * The whole footer, decoded on the first call. + * Created on the first call. */ - public function thrift(): FileMetaData + public function footer(): RustParquetFooter { throw new RuntimeException('arrow extension is not loaded'); } /** - * Σ row group total_byte_size (uncompressed). + * @param list $columns resolved names, never empty + * @param int $batchSize upper bound of rows per chunk, refused below 1 */ - public function totalByteSize(): int + public function readColumns(array $columns, int $batchSize, ReadPlan $plan): RustColumnsReader { throw new RuntimeException('arrow extension is not loaded'); } diff --git a/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFooter.php b/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFooter.php new file mode 100644 index 0000000000..6eaf75423a --- /dev/null +++ b/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFooter.php @@ -0,0 +1,67 @@ + $columns + */ + public function rowGroupChunks(array $columns): RowGroupChunks + { + throw new RuntimeException('arrow extension is not loaded'); + } + + public function rowsNumber(): int + { + throw new RuntimeException('arrow extension is not loaded'); + } + + public function schema(): Schema + { + throw new RuntimeException('arrow extension is not loaded'); + } + + /** + * The whole footer, decoded on the first call. + */ + public function thrift(): FileMetaData + { + throw new RuntimeException('arrow extension is not loaded'); + } + + /** + * Σ row group total_byte_size (uncompressed). + */ + public function totalByteSize(): int + { + throw new RuntimeException('arrow extension is not loaded'); + } +} diff --git a/src/extension/arrow-ext/src/interfaces.rs b/src/extension/arrow-ext/src/interfaces.rs index cc8589a753..316de354b9 100644 --- a/src/extension/arrow-ext/src/interfaces.rs +++ b/src/extension/arrow-ext/src/interfaces.rs @@ -1,4 +1,4 @@ -//! `Flow\Parquet\{ParquetEngine, ParquetFileReader, ParquetFileWriter}`: registered at MINIT with the package files' +//! `Flow\Parquet\{ParquetEngine, ParquetFileReader, ParquetFileWriter}` and `Flow\Parquet\ParquetFile\Footer`: registered at MINIT with the package files' //! exact signatures, so the extension's classes implement them and the package files (autoloaded only without arrow) //! stay unused. @@ -14,14 +14,18 @@ const SOURCE_STREAM: DataType = DataType::object("Flow\\Filesystem\\SourceStream const DESTINATION_STREAM: DataType = DataType::object("Flow\\Filesystem\\DestinationStream"); const SCHEMA: DataType = DataType::object("Flow\\Parquet\\ParquetFile\\Schema"); const METADATA: DataType = DataType::object("Flow\\Parquet\\ParquetFile\\Metadata"); +const READ_PLAN: DataType = DataType::object("Flow\\Parquet\\Reader\\ReadPlan"); +const ROW_GROUP_CHUNKS: DataType = DataType::object("Flow\\Parquet\\ParquetFile\\RowGroupChunks"); const COMPRESSIONS: DataType = DataType::object("Flow\\Parquet\\ParquetFile\\Compressions"); const OPTIONS: DataType = DataType::object("Flow\\Parquet\\Options"); +const FOOTER: DataType = DataType::object("Flow\\Parquet\\ParquetFile\\Footer"); const PARQUET_FILE_READER: DataType = DataType::object("Flow\\Parquet\\ParquetFileReader"); const PARQUET_FILE_WRITER: DataType = DataType::object("Flow\\Parquet\\ParquetFileWriter"); const ITERATOR: DataType = DataType::object("Iterator"); // ClassEntry::try_find reads EG(class_table), which MINIT has not set, so each CE is kept from its registration static PARQUET_ENGINE_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); +static FOOTER_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); static PARQUET_FILE_READER_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); static PARQUET_FILE_WRITER_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); @@ -30,6 +34,11 @@ fn store_parquet_engine(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) { PARQUET_ENGINE_CE.store(ce, Ordering::Release); } +fn store_footer(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) { + keep(arg_info); + FOOTER_CE.store(ce, Ordering::Release); +} + fn store_parquet_file_reader(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) { keep(arg_info); PARQUET_FILE_READER_CE.store(ce, Ordering::Release); @@ -53,6 +62,10 @@ pub fn parquet_engine_ce() -> &'static ClassEntry { load(&PARQUET_ENGINE_CE, "Flow\\Parquet\\ParquetEngine") } +pub fn footer_ce() -> &'static ClassEntry { + load(&FOOTER_CE, "Flow\\Parquet\\ParquetFile\\Footer") +} + pub fn parquet_file_reader_ce() -> &'static ClassEntry { load(&PARQUET_FILE_READER_CE, "Flow\\Parquet\\ParquetFileReader") } @@ -82,25 +95,36 @@ fn interface( } pub fn register() -> Result<()> { + interface( + "Flow\\Parquet\\ParquetFile\\Footer", + store_footer, + vec![ + method("metadata", vec![], METADATA), + method( + "rowGroupChunks", + vec![Arg::new("columns", DataType::Array)], + ROW_GROUP_CHUNKS, + ), + method("rowsNumber", vec![], DataType::Long), + method("schema", vec![], SCHEMA), + method("totalByteSize", vec![], DataType::Long), + ], + )?; interface( "Flow\\Parquet\\ParquetFileReader", store_parquet_file_reader, vec![ method("close", vec![], DataType::Void), - method("metadata", vec![], METADATA), + method("footer", vec![], FOOTER), method( "readColumns", vec![ Arg::new("columns", DataType::Array), Arg::new("batchSize", DataType::Long), - Arg::new("limit", DataType::Long).allow_null(), - Arg::new("offset", DataType::Long).allow_null(), + Arg::new("plan", READ_PLAN), ], ITERATOR, ), - method("rowsNumber", vec![], DataType::Long), - method("schema", vec![], SCHEMA), - method("totalByteSize", vec![], DataType::Long), ], )?; interface( diff --git a/src/extension/arrow-ext/src/lib.rs b/src/extension/arrow-ext/src/lib.rs index 8804b3bff3..5d194b8aa6 100644 --- a/src/extension/arrow-ext/src/lib.rs +++ b/src/extension/arrow-ext/src/lib.rs @@ -29,8 +29,8 @@ pub extern "C" fn php_module_info(_module: *mut ModuleEntry) { } /// The contract version flow-php/parquet expects (`Flow\Parquet\Engine\ArrowExtension::ABI`); both change together -/// whenever a registered class or interface changes. -const FLOW_ARROW_ABI: i64 = 1; +/// whenever a registered class or interface, or the schema, options or values crossing them, change. +const FLOW_ARROW_ABI: i64 = 2; /// # Safety /// @@ -55,6 +55,7 @@ pub fn get_module(module: ModuleBuilder) -> ModuleBuilder { .request_startup_function(php::request_startup) .request_shutdown_function(php::request_shutdown) .class::() + .class::() .class::() .class::() .class::() diff --git a/src/extension/arrow-ext/src/parquet/batch.rs b/src/extension/arrow-ext/src/parquet/batch.rs index 60d5fde10f..af91b7ccdc 100644 --- a/src/extension/arrow-ext/src/parquet/batch.rs +++ b/src/extension/arrow-ext/src/parquet/batch.rs @@ -15,8 +15,8 @@ use ext_php_rs::types::{ZendObject, Zval}; use ext_php_rs::zend::ClassEntry; use crate::parquet::error::Error; -use crate::parquet::library::{batch_size, rows, RustParquetFileReader, INVALID_ARGUMENT}; -use crate::parquet::read::{ParquetReader, ReadPlan}; +use crate::parquet::library::{batch_size, read_plan, RustParquetFileReader, INVALID_ARGUMENT}; +use crate::parquet::read::ParquetReader; use crate::parquet::source::PhpStream; use crate::php::{call_handle_transparent, ce_method_ref}; use crate::render::{exception, parquet_exception, Side}; @@ -83,22 +83,14 @@ impl RustBatchReader { file: &RustParquetFileReader, columns: Vec, batch_size: i64, - offset: Option, - limit: Option, + plan: &Zval, ) -> PhpResult { let batch_size = self::batch_size(batch_size)?; - let (offset, limit) = (rows(offset, "offset")?.unwrap_or(0), rows(limit, "limit")?); - let (stream, size, meta) = file.read()?; + let (stream, meta) = file.read()?; let stream = Arc::clone(stream); - let reader = ParquetReader::open( - Arc::clone(&stream), - size, - meta, - &columns, - ReadPlan::new(meta, offset, limit), - batch_size, - ) - .map_err(|error| parquet_exception(error, &stream, Side::Read))?; + let plan = read_plan(meta, plan, &stream)?; + let reader = ParquetReader::open(Arc::clone(&stream), meta, &columns, plan, batch_size) + .map_err(|error| parquet_exception(error, &stream, Side::Read))?; let fields = columns .iter() .zip(reader.fields()) @@ -188,26 +180,13 @@ pub fn import(batch: &Zval) -> PhpResult { ))); } - let schema = unsafe { &*(address(object, ce, "arrowSchemaAddress")? as *const FFI_ArrowSchema) }; - let source = unsafe { &mut *(address(object, ce, "arrowArrayAddress")? as *mut FFI_ArrowArray) }; - - if schema.format.is_null() || unsafe { CStr::from_ptr(schema.format) }.to_bytes() != b"+s" { - return Err(refused("Arrow C Data batch must be a struct array".to_string())); - } - - if source.release.is_none() { - return Err(refused("Arrow C Data batch is already imported".to_string())); - } + let schema = address(object, ce, "arrowSchemaAddress")? as *const FFI_ArrowSchema; + let source = address(object, ce, "arrowArrayAddress")? as *mut FFI_ArrowArray; - if source.n_children != schema.n_children || source.length < 0 { - return Err(refused(format!( - "Arrow C Data batch of {} children does not match its schema of {}", - source.n_children, schema.n_children - ))); - } + unsafe { check(schema, source) }.map_err(refused)?; - let moved = unsafe { std::ptr::read(source) }; - source.release = None; + let schema = unsafe { &*schema }; + let moved = unsafe { FFI_ArrowArray::from_raw(source) }; let data = unsafe { from_ffi(moved, schema) } .and_then(|data| data.validate().map(|()| data)) @@ -216,6 +195,59 @@ pub fn import(batch: &Zval) -> PhpResult { Ok(StructArray::from(data)) } +/// The leading fields of `struct ArrowSchema` and `struct ArrowArray` as the Arrow C Data Interface specification +/// declares them (https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions): read before the +/// arrow-rs accessors, which panic on a null format and cast a negative count to `usize`. +#[repr(C)] +struct SchemaHead { + format: *const c_char, + name: *const c_char, + metadata: *const c_char, + flags: i64, + n_children: i64, +} + +#[repr(C)] +struct ArrayHead { + length: i64, + null_count: i64, + offset: i64, + n_buffers: i64, + n_children: i64, +} + +/// # Safety +/// +/// `schema` and `array` point to live C Data Interface structs. +unsafe fn check(schema: *const FFI_ArrowSchema, array: *const FFI_ArrowArray) -> Result<(), String> { + let schema_head = unsafe { &*(schema as *const SchemaHead) }; + let array_head = unsafe { &*(array as *const ArrayHead) }; + + if schema_head.format.is_null() || unsafe { CStr::from_ptr(schema_head.format) }.to_bytes() != b"+s" { + return Err("Arrow C Data batch must be a struct array".to_string()); + } + + if unsafe { &*array }.is_released() { + return Err("Arrow C Data batch is already imported".to_string()); + } + + if array_head.length < 0 { + return Err(format!( + "Arrow C Data batch length must not be negative, {} given", + array_head.length + )); + } + + if array_head.n_children < 0 || schema_head.n_children < 0 || array_head.n_children != schema_head.n_children { + return Err(format!( + "Arrow C Data batch of {} children does not match its schema of {}", + array_head.n_children, schema_head.n_children + )); + } + + Ok(()) +} + #[cfg(test)] mod tests { use std::sync::Arc; @@ -271,4 +303,53 @@ mod tests { assert_eq!(allocated_bytes(), baseline); } + + fn refusal(mutate: impl FnOnce(*mut super::SchemaHead, *mut super::ArrayHead)) -> Result<(), String> { + let (mut array, mut schema) = to_ffi(&batch().to_data()).unwrap(); + let schema_ptr = &mut schema as *mut arrow_array::ffi::FFI_ArrowSchema; + let array_ptr = &mut array as *mut arrow_array::ffi::FFI_ArrowArray; + let schema_head = schema_ptr as *mut super::SchemaHead; + let array_head = array_ptr as *mut super::ArrayHead; + let (format, length, children) = + unsafe { ((*schema_head).format, (*array_head).length, (*array_head).n_children) }; + + mutate(schema_head, array_head); + let result = unsafe { super::check(schema_ptr, array_ptr) }; + + unsafe { + (*schema_head).format = format; + (*array_head).length = length; + (*array_head).n_children = children; + } + + result + } + + #[test] + fn a_null_format_is_refused_before_the_accessors() { + assert_eq!( + refusal(|schema, _| unsafe { (*schema).format = std::ptr::null() }), + Err("Arrow C Data batch must be a struct array".to_string()) + ); + } + + #[test] + fn a_negative_length_is_refused() { + assert_eq!( + refusal(|_, array| unsafe { (*array).length = -1 }), + Err("Arrow C Data batch length must not be negative, -1 given".to_string()) + ); + } + + #[test] + fn a_negative_or_mismatched_child_count_is_refused() { + assert_eq!( + refusal(|_, array| unsafe { (*array).n_children = -1 }), + Err("Arrow C Data batch of -1 children does not match its schema of 2".to_string()) + ); + assert_eq!( + refusal(|_, array| unsafe { (*array).n_children = 1 }), + Err("Arrow C Data batch of 1 children does not match its schema of 2".to_string()) + ); + } } diff --git a/src/extension/arrow-ext/src/parquet/canonical.rs b/src/extension/arrow-ext/src/parquet/canonical.rs index 80b5dfaca2..fc19dbae2a 100644 --- a/src/extension/arrow-ext/src/parquet/canonical.rs +++ b/src/extension/arrow-ext/src/parquet/canonical.rs @@ -2,8 +2,10 @@ //! storage (the layout flow_php's columns store), keyed on the arrow `DataType` alone. Temporal and decimal //! casts are this module's own - `arrow_cast` rounds toward zero and nulls out overflows. +use std::fmt::Write; use std::sync::Arc; +use arrow_array::builder::LargeBinaryBuilder; use arrow_array::cast::AsArray; use arrow_array::types::{ Date32Type, Date64Type, Decimal128Type, Decimal256Type, DecimalType, DurationMicrosecondType, Float32Type, @@ -13,11 +15,12 @@ use arrow_array::types::{ }; use arrow_array::{ make_array, Array, ArrayRef, ArrowPrimitiveType, BinaryArray, FixedSizeBinaryArray, Float64Array, GenericByteArray, - LargeBinaryArray, ListArray, MapArray, PrimitiveArray, StructArray, + LargeBinaryArray, ListArray, MapArray, PrimitiveArray, StructArray, UInt64Array, }; use arrow_buffer::{OffsetBuffer, ScalarBuffer}; use arrow_schema::{DataType, Field, FieldRef, Fields, TimeUnit}; +use crate::parquet::cells::{unsigned, UNSIGNED_EXPECTED}; use crate::parquet::error::{parent_row, Error}; const UTC: &str = "UTC"; @@ -62,19 +65,16 @@ fn canonical_data_type(data_type: &DataType) -> Result { | DataType::Float64 | DataType::Date32 | DataType::FixedSizeBinary(16) => data_type.clone(), - DataType::Int8 - | DataType::Int16 - | DataType::Int32 - | DataType::UInt8 - | DataType::UInt16 - | DataType::UInt32 - | DataType::UInt64 => DataType::Int64, + DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::UInt8 | DataType::UInt16 | DataType::UInt32 => { + DataType::Int64 + } DataType::Float32 | DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => DataType::Float64, DataType::Utf8 | DataType::Binary | DataType::LargeUtf8 | DataType::LargeBinary - | DataType::FixedSizeBinary(_) => DataType::LargeBinary, + | DataType::FixedSizeBinary(_) + | DataType::UInt64 => DataType::LargeBinary, DataType::Timestamp(_, _) => timestamp(), DataType::Time64(TimeUnit::Microsecond | TimeUnit::Nanosecond) | DataType::Time32(TimeUnit::Millisecond | TimeUnit::Second) => DataType::Duration(TimeUnit::Microsecond), @@ -146,6 +146,47 @@ fn checked( Ok(PrimitiveArray::new(ScalarBuffer::from(values), input.nulls().cloned())) } +/// UINT_64 from its decimal text (a `numeric_string` column): a canonical digit string up to u64::MAX, refused at its +/// row otherwise. +fn unsigned_from_text(array: &ArrayRef) -> Result { + let input = array.as_binary::(); + let mut values = Vec::with_capacity(input.len()); + + for row in 0..input.len() { + values.push(if input.is_null(row) { + 0 + } else { + unsigned(input.value(row)).ok_or_else(|| Error::Value { + column: String::new(), + row, + expected: UNSIGNED_EXPECTED, + got: "string".to_string(), + })? + }); + } + + Ok(UInt64Array::new(ScalarBuffer::from(values), input.nulls().cloned())) +} + +/// UINT_64 as its decimal text: one PHP type for the column, lossless above `PHP_INT_MAX`. +fn decimal_text(array: &ArrayRef) -> ArrayRef { + let input = array.as_primitive::(); + let mut builder = LargeBinaryBuilder::with_capacity(input.len(), input.len() * 20); + let mut text = String::with_capacity(20); + + for (row, value) in input.values().iter().enumerate() { + if input.is_null(row) { + builder.append_null(); + } else { + text.clear(); + write!(text, "{value}").expect("writing to a String does not fail"); + builder.append_value(&text); + } + } + + Arc::new(builder.finish()) +} + fn retagged(array: &ArrayRef, to: DataType) -> ArrayRef { make_array(unsafe { array.to_data().into_builder().data_type(to).build_unchecked() }) } @@ -252,9 +293,7 @@ fn cast(array: &ArrayRef) -> Result { DataType::UInt8 => widened::(array), DataType::UInt16 => widened::(array), DataType::UInt32 => widened::(array), - DataType::UInt64 => Arc::new(checked::(array, |value| { - i64::try_from(value).ok() - })?), + DataType::UInt64 => decimal_text(array), DataType::Float32 => Arc::new(array.as_primitive::().unary::<_, Float64Type>(f64::from)), DataType::Utf8 | DataType::Binary => widened_binary(array), DataType::LargeUtf8 | DataType::LargeBinary => retagged(array, DataType::LargeBinary), @@ -369,6 +408,7 @@ pub fn to_parquet(array: &ArrayRef, target: &DataType) -> Result Arc::new(narrowed_int::(array)?), (DataType::Int64, DataType::UInt32) => Arc::new(narrowed_int::(array)?), (DataType::Int64, DataType::UInt64) => Arc::new(narrowed_int::(array)?), + (DataType::LargeBinary, DataType::UInt64) => Arc::new(unsigned_from_text(array)?), (DataType::Float64, DataType::Float32) => Arc::new( array .as_primitive::() @@ -701,28 +741,39 @@ mod tests { int64(&read(Arc::new(UInt32Array::from(vec![Some(u32::MAX), None])))), vec![Some(i64::from(u32::MAX)), None] ); - assert_eq!( - int64(&read(Arc::new(UInt64Array::from(vec![Some(i64::MAX as u64), None])))), - vec![Some(i64::MAX), None] - ); } #[test] - fn uint64_above_int64_is_refused_at_its_row() { - let error = refusal(Arc::new(UInt64Array::from(vec![ - Some(1), + fn uint64_reads_as_its_decimal_text() { + let canonical = read(Arc::new(UInt64Array::from(vec![ + Some(0), None, - Some(i64::MAX as u64 + 1), + Some(1 << 63), + Some(u64::MAX), ]))); - assert!(matches!(error, Error::Overflow { column, row: 2 } if column == "column")); + assert_eq!( + canonical.as_binary::().iter().collect::>(), + vec![ + Some(&b"0"[..]), + None, + Some(&b"9223372036854775808"[..]), + Some(&b"18446744073709551615"[..]), + ] + ); } #[test] fn a_null_slot_never_overflows() { - let values = UInt64Array::new(vec![1, u64::MAX].into(), Some(vec![true, false].into())); + let values = TimestampMillisecondArray::new(vec![1, i64::MAX].into(), Some(vec![true, false].into())); - assert_eq!(int64(&read(Arc::new(values))), vec![Some(1), None]); + assert_eq!( + read(Arc::new(values)) + .as_primitive::() + .iter() + .collect::>(), + vec![Some(1_000), None] + ); } #[test] @@ -952,9 +1003,13 @@ mod tests { #[test] fn a_nested_overflow_is_refused_at_its_parent_row() { - let values = Arc::new(UInt64Array::from(vec![1, 2, u64::MAX])); + let values = Arc::new(TimestampMillisecondArray::from(vec![1, 2, i64::MAX])); let list = ListArray::new( - Arc::new(Field::new("element", DataType::UInt64, false)), + Arc::new(Field::new( + "element", + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + )), arrow_buffer::OffsetBuffer::new(vec![0, 2, 3].into()), values, None, @@ -1025,15 +1080,16 @@ mod tests { vec![Some(-128), None, Some(127)] ); - let unsigned: ArrayRef = Arc::new(Int64Array::from(vec![Some(4_000_000_000), Some(i64::MAX)])); - assert_eq!( - to_parquet(&unsigned, &DataType::UInt64) - .unwrap() - .as_primitive::() - .values() - .to_vec(), - vec![4_000_000_000, i64::MAX as u64] + to_parquet( + &(Arc::new(Int64Array::from(vec![Some(4_000_000_000), None])) as ArrayRef), + &DataType::UInt64 + ) + .unwrap() + .as_primitive::() + .iter() + .collect::>(), + vec![Some(4_000_000_000), None] ); for target in [ @@ -1381,4 +1437,35 @@ mod tests { Err(Error::Unsupported { parquet, .. }) if parquet == "Float32" )); } + + #[test] + fn to_parquet_reads_uint64_from_digit_text() { + let text: ArrayRef = Arc::new(LargeBinaryArray::from(vec![ + Some(&b"18446744073709551615"[..]), + None, + Some(&b"0"[..]), + ])); + + assert_eq!( + to_parquet(&text, &DataType::UInt64) + .unwrap() + .as_primitive::() + .iter() + .collect::>(), + vec![Some(u64::MAX), None, Some(0)] + ); + } + + #[test] + fn to_parquet_refuses_uint64_text_that_is_not_a_canonical_digit_string() { + for text in [&b"18446744073709551616"[..], b"007", b"-1", b"abc", b""] { + let array: ArrayRef = Arc::new(LargeBinaryArray::from(vec![Some(&b"1"[..]), Some(text)])); + + assert!( + matches!(to_parquet(&array, &DataType::UInt64), Err(Error::Value { row: 1, .. })), + "{:?}", + text + ); + } + } } diff --git a/src/extension/arrow-ext/src/parquet/cells.rs b/src/extension/arrow-ext/src/parquet/cells.rs index f5ce878e63..0c8ecd7ba0 100644 --- a/src/extension/arrow-ext/src/parquet/cells.rs +++ b/src/extension/arrow-ext/src/parquet/cells.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use arrow_array::builder::{ ArrayBuilder, BooleanBuilder, Date32Builder, DurationMicrosecondBuilder, FixedSizeBinaryBuilder, Float64Builder, Int64Builder, LargeBinaryBuilder, ListBuilder, MapBuilder, MapFieldNames, TimestampMicrosecondBuilder, + UInt64Builder, }; use arrow_array::cast::AsArray; use arrow_array::types::{Date32Type, DurationMicrosecondType, Float64Type, Int64Type, TimestampMicrosecondType}; @@ -381,6 +382,7 @@ pub enum Scalar<'a> { Null, Bool(bool), Int(i64), + UInt(u64), Float(f64), Bytes(&'a [u8]), Uuid([u8; 16]), @@ -396,6 +398,7 @@ impl Scalar<'_> { Scalar::Null => Scalar::Null, Scalar::Bool(value) => Scalar::Bool(value), Scalar::Int(value) => Scalar::Int(value), + Scalar::UInt(value) => Scalar::UInt(value), Scalar::Float(value) => Scalar::Float(value), Scalar::Bytes(_) => Scalar::Bytes(bytes), Scalar::Uuid(value) => Scalar::Uuid(value), @@ -483,6 +486,7 @@ impl<'a> Checked<'a> for Validated<'a> { pub enum ColumnBuilder { Bool(BooleanBuilder), Int(Int64Builder), + UInt(UInt64Builder), Float(Float64Builder), Bytes(LargeBinaryBuilder), Uuid(FixedSizeBinaryBuilder), @@ -532,6 +536,7 @@ impl ColumnBuilder { match canonical { DataType::Boolean => ColumnBuilder::Bool(BooleanBuilder::with_capacity(capacity)), DataType::Int64 => ColumnBuilder::Int(Int64Builder::with_capacity(capacity)), + DataType::UInt64 => ColumnBuilder::UInt(UInt64Builder::with_capacity(capacity)), DataType::Float64 => ColumnBuilder::Float(Float64Builder::with_capacity(capacity)), DataType::LargeBinary => ColumnBuilder::Bytes(LargeBinaryBuilder::with_capacity(capacity, 1024)), DataType::FixedSizeBinary(size) => { @@ -584,6 +589,7 @@ impl ArrayBuilder for ColumnBuilder { match self { ColumnBuilder::Bool(builder) => builder.len(), ColumnBuilder::Int(builder) => builder.len(), + ColumnBuilder::UInt(builder) => builder.len(), ColumnBuilder::Float(builder) => builder.len(), ColumnBuilder::Bytes(builder) => builder.len(), ColumnBuilder::Uuid(builder) => builder.len(), @@ -600,6 +606,7 @@ impl ArrayBuilder for ColumnBuilder { match self { ColumnBuilder::Bool(builder) => Arc::new(builder.finish()), ColumnBuilder::Int(builder) => Arc::new(builder.finish()), + ColumnBuilder::UInt(builder) => Arc::new(builder.finish()), ColumnBuilder::Float(builder) => Arc::new(builder.finish()), ColumnBuilder::Bytes(builder) => Arc::new(builder.finish()), ColumnBuilder::Uuid(builder) => Arc::new(builder.finish()), @@ -616,6 +623,7 @@ impl ArrayBuilder for ColumnBuilder { match self { ColumnBuilder::Bool(builder) => Arc::new(builder.finish_cloned()), ColumnBuilder::Int(builder) => Arc::new(builder.finish_cloned()), + ColumnBuilder::UInt(builder) => Arc::new(builder.finish_cloned()), ColumnBuilder::Float(builder) => Arc::new(builder.finish_cloned()), ColumnBuilder::Bytes(builder) => Arc::new(builder.finish_cloned()), ColumnBuilder::Uuid(builder) => Arc::new(builder.finish_cloned()), @@ -653,6 +661,8 @@ fn offsets_fit(current: usize, added: usize) -> bool { pub enum WriteCell { Bool, Int, + /// UINT_64: an int >= 0, or a canonical digit string up to 18446744073709551615. + UInt64, Float, /// DECIMAL: a float or an int. Decimal, @@ -679,8 +689,8 @@ impl WriteCell { | DataType::Int64 | DataType::UInt8 | DataType::UInt16 - | DataType::UInt32 - | DataType::UInt64 => (WriteCell::Int, DataType::Int64), + | DataType::UInt32 => (WriteCell::Int, DataType::Int64), + DataType::UInt64 => (WriteCell::UInt64, DataType::UInt64), DataType::Float32 | DataType::Float64 => (WriteCell::Float, DataType::Float64), DataType::Decimal128(_, _) => (WriteCell::Decimal, DataType::Float64), DataType::Utf8 | DataType::Binary => (WriteCell::Bytes, DataType::LargeBinary), @@ -754,6 +764,13 @@ impl WriteCell { Ok(match self { WriteCell::Bool => Scalar::Bool(value.bool().ok_or_else(|| refused("bool", value))?), WriteCell::Int => Scalar::Int(value.long().ok_or_else(|| refused("int", value))?), + WriteCell::UInt64 => Scalar::UInt( + match value.long() { + Some(int) => u64::try_from(int).ok(), + None => value.zend_str().and_then(|text| unsigned(text.as_bytes())), + } + .ok_or_else(|| refused(UNSIGNED_EXPECTED, value))?, + ), WriteCell::Float => Scalar::Float(value.double().ok_or_else(|| refused("float", value))?), WriteCell::Decimal => Scalar::Float(match (value.double(), value.long()) { (Some(float), _) => float, @@ -830,7 +847,7 @@ impl WriteCell { } } - Ok(()) + Ok::<(), PhpException>(()) })?; if let Some(refusal) = failure { @@ -846,7 +863,7 @@ impl WriteCell { ht_for_each(items, |name, index, item| { if failure.is_some() { - return Ok(()); + return Ok::<(), PhpException>(()); } let entry_key = match name { @@ -986,6 +1003,7 @@ impl WriteCell { (_, builder, Scalar::Null) => self.append_null(builder), (WriteCell::Bool, ColumnBuilder::Bool(builder), Scalar::Bool(value)) => builder.append_value(*value), (WriteCell::Int, ColumnBuilder::Int(builder), Scalar::Int(value)) => builder.append_value(*value), + (WriteCell::UInt64, ColumnBuilder::UInt(builder), Scalar::UInt(value)) => builder.append_value(*value), (WriteCell::Float | WriteCell::Decimal, ColumnBuilder::Float(builder), Scalar::Float(value)) => { builder.append_value(*value); } @@ -1022,6 +1040,7 @@ impl WriteCell { } (_, ColumnBuilder::Bool(builder)) => builder.append_null(), (_, ColumnBuilder::Int(builder)) => builder.append_null(), + (_, ColumnBuilder::UInt(builder)) => builder.append_null(), (_, ColumnBuilder::Float(builder)) => builder.append_null(), (_, ColumnBuilder::Bytes(builder)) => builder.append_null(), (_, ColumnBuilder::Uuid(builder)) => builder.append_null(), @@ -1037,6 +1056,18 @@ impl WriteCell { } } +/// What a UINT_64 cell takes, as its refusals name it. +pub const UNSIGNED_EXPECTED: &str = "int >= 0 or digit string"; + +/// A canonical digit string (no sign, no leading zero) within u64, as `UnsignedInt64Converter::toParquetType()` takes it. +pub fn unsigned(text: &[u8]) -> Option { + if text.is_empty() || !text.iter().all(u8::is_ascii_digit) || (text.len() > 1 && text[0] == b'0') { + return None; + } + + std::str::from_utf8(text).ok()?.parse().ok() +} + /// `UuidConverter::toParquetType()`'s text of an object: `toString()`, else `__toString()`; `None` for neither. fn uuid_text_of(object: &ZendObject) -> Result, PhpException> { let ce = unsafe { object.ce.as_ref() }.ok_or_else(|| ext_exception("arrow expected a class entry"))?; diff --git a/src/extension/arrow-ext/src/parquet/chunks.rs b/src/extension/arrow-ext/src/parquet/chunks.rs new file mode 100644 index 0000000000..c16c830449 --- /dev/null +++ b/src/extension/arrow-ext/src/parquet/chunks.rs @@ -0,0 +1,335 @@ +//! The column chunks of the leaves a filter reads and their column orders, sliced from the footer's thrift bytes +//! as the file holds them: the PHP planner decodes the bytes the PHP engine decodes, so both engines prune alike. + +use std::ops::Range; + +use crate::parquet::error::Error; +use crate::thrift::{elements as list, fields, Field, Malformed}; + +const FILE_ROW_GROUPS: i16 = 4; +const FILE_COLUMN_ORDERS: i16 = 7; +const FILE_ENCRYPTION_ALGORITHM: i16 = 8; +const ROW_GROUP_COLUMNS: i16 = 1; +const CHUNK_CRYPTO_METADATA: i16 = 8; +const CHUNK_ENCRYPTED_COLUMN_METADATA: i16 = 9; + +/// Byte ranges into the footer, one entry per leaf asked for, in the order asked. +#[derive(Debug, PartialEq, Eq)] +pub struct ChunkSpans { + /// Each leaf's `ColumnChunk` in every row group, in file order. + pub chunks: Vec>>, + /// Each leaf's `ColumnOrder`; `None` when the file has no `column_orders`. + pub orders: Option>>, +} + +/// `leaves`: indices into the schema's leaves. +pub fn chunk_spans(footer: &[u8], leaves: &[usize]) -> Result { + let mut chunks = vec![Vec::new(); leaves.len()]; + let mut orders = None; + + for field in fields(footer) { + let field = field.map_err(malformed)?; + + match field.id { + FILE_ROW_GROUPS => { + for group in elements(footer, &field, 0)? { + let group = group?; + let columns = row_group_columns(footer, &group)?.collect::, _>>()?; + + for (position, leaf) in leaves.iter().enumerate() { + chunks[position].push( + columns + .get(*leaf) + .cloned() + .ok_or_else(|| corrupt("a row group without the leaf"))?, + ); + } + } + } + FILE_COLUMN_ORDERS => { + let columns = elements(footer, &field, 0)?.collect::, _>>()?; + + orders = Some( + leaves + .iter() + .map(|leaf| { + columns + .get(*leaf) + .cloned() + .ok_or_else(|| corrupt("column orders without the leaf")) + }) + .collect::, _>>()?, + ); + } + _ => {} + } + } + + Ok(ChunkSpans { chunks, orders }) +} + +/// Refuses a footer that holds an encryption algorithm or an encrypted column chunk: arrow-rs decodes a plaintext +/// footer with encrypted columns without a word. +pub fn refuse_encrypted(footer: &[u8]) -> Result<(), Error> { + for field in fields(footer) { + let field = field.map_err(malformed)?; + + match field.id { + FILE_ROW_GROUPS => { + for group in elements(footer, &field, 0)? { + for chunk in row_group_columns(footer, &group?)? { + for field in fields(&footer[chunk?]) { + let field = field.map_err(malformed)?; + + if field.id == CHUNK_CRYPTO_METADATA || field.id == CHUNK_ENCRYPTED_COLUMN_METADATA { + return Err(Error::Encrypted); + } + } + } + } + } + FILE_ENCRYPTION_ALGORITHM => return Err(Error::Encrypted), + _ => {} + } + } + + Ok(()) +} + +/// The `ColumnChunk`s of the row group at `group` in `footer`, as ranges into `footer`. +fn row_group_columns<'a>( + footer: &'a [u8], + group: &Range, +) -> Result, Error>> + 'a, Error> { + let mut columns = None; + + for field in fields(&footer[group.clone()]) { + let field = field.map_err(malformed)?; + + if field.id == ROW_GROUP_COLUMNS { + columns = Some(field); + break; + } + } + + elements( + footer, + &columns.ok_or_else(|| corrupt("a row group without columns"))?, + group.start, + ) +} + +/// The elements of the list `field` holds, `field` read from the struct at `offset` in `footer`, as ranges into +/// `footer`. +fn elements<'a>( + footer: &'a [u8], + field: &Field, + offset: usize, +) -> Result, Error>> + 'a, Error> { + if !field.is_list() { + return Err(corrupt("another value where a list belongs")); + } + + let start = offset + field.value.start; + + Ok(list(&footer[start..offset + field.value.end]) + .map_err(malformed)? + .map(move |element| { + element + .map(|element| element.start + start..element.end + start) + .map_err(malformed) + })) +} + +fn corrupt(what: &str) -> Error { + Error::NotParquet(format!("Invalid Parquet file. The footer holds {what}")) +} + +fn malformed(malformed: Malformed) -> Error { + Error::NotParquet(format!( + "Invalid Parquet file. The footer is not thrift: {}", + malformed.0 + )) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::{ArrayRef, Int64Array, RecordBatch, StringArray}; + use arrow_schema::{DataType, Field, Schema}; + use bytes::Bytes; + use parquet::arrow::ArrowWriter; + use parquet::file::properties::WriterProperties; + + use super::{chunk_spans, refuse_encrypted}; + use crate::parquet::error::Error; + use crate::parquet::footer::Footer; + use crate::thrift::{elements, fields}; + + /// `id` Int64 and `name` Utf8 in two row groups of 2 rows. + fn two_groups() -> Footer { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, true), + ])); + let mut file = Vec::new(); + let mut writer = ArrowWriter::try_new( + &mut file, + Arc::clone(&schema), + Some(WriterProperties::builder().set_max_row_group_row_count(Some(2)).build()), + ) + .unwrap(); + writer + .write( + &RecordBatch::try_new( + schema, + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3, 4])) as ArrayRef, + Arc::new(StringArray::from(vec!["a", "b", "c", "d"])), + ], + ) + .unwrap(), + ) + .unwrap(); + writer.close().unwrap(); + + Footer::read(&Bytes::from(file)).unwrap() + } + + /// `bytes` with `insert` put at `at` and the `remove` bytes after it dropped. + fn spliced(bytes: &[u8], at: usize, remove: usize, insert: &[u8]) -> Vec { + let mut spliced = bytes[..at].to_vec(); + spliced.extend_from_slice(insert); + spliced.extend_from_slice(&bytes[at + remove..]); + + spliced + } + + /// The footer with `field` (of compact wire type `wire`, then `value`) appended to the first chunk of `id`. + fn with_chunk_field(bytes: &[u8], field: u8, wire: u8, value: &[u8]) -> Vec { + let chunk = chunk_spans(bytes, &[0]).unwrap().chunks[0][0].clone(); + let last = fields(&bytes[chunk.clone()]).last().unwrap().unwrap().id as u8; + let mut insert = vec![((field - last) << 4) | wire]; + insert.extend_from_slice(value); + + spliced(bytes, chunk.end - 1, 0, &insert) + } + + fn assert_corrupt(result: Result, message: &str) { + match result { + Err(Error::NotParquet(actual)) => assert!(actual.contains(message), "{actual}"), + other => panic!("expected NotParquet containing {message:?}, got {other:?}"), + } + } + + /// `ColumnChunk.meta_data.path_in_schema` of the chunk `span` holds. + fn path(footer: &[u8], span: &std::ops::Range) -> Vec { + let chunk = &footer[span.clone()]; + let meta = fields(chunk) + .map(Result::unwrap) + .find(|field| field.id == 3) + .unwrap() + .value; + let path = fields(&chunk[meta.clone()]) + .map(Result::unwrap) + .find(|field| field.id == 3) + .unwrap() + .value; + let elements = elements(&chunk[meta.start + path.start..meta.start + path.end]) + .unwrap() + .collect::, _>>() + .unwrap(); + + chunk[meta.start + path.start + elements[0].start + 1..meta.start + path.start + elements[0].end].to_vec() + } + + #[test] + fn the_spans_are_each_asked_leafs_chunk_per_row_group_and_its_column_order() { + let footer = two_groups(); + let bytes = footer.bytes(); + let spans = chunk_spans(bytes, &[1, 0]).unwrap(); + + assert_eq!(spans.chunks.len(), 2); + assert_eq!(spans.chunks[0].len(), 2); + assert_eq!(spans.chunks[1].len(), 2); + assert_eq!(path(bytes, &spans.chunks[0][0]), b"name"); + assert_eq!(path(bytes, &spans.chunks[0][1]), b"name"); + assert_eq!(path(bytes, &spans.chunks[1][1]), b"id"); + + let orders = spans.orders.unwrap(); + + // a ColumnOrder union holding TYPE_ORDER (field 1, an empty struct) + assert_eq!(&bytes[orders[0].clone()], b"\x1c\x00\x00"); + assert_eq!(&bytes[orders[1].clone()], b"\x1c\x00\x00"); + } + + #[test] + fn a_file_without_column_orders_has_no_orders() { + let footer = two_groups(); + let bytes = footer.bytes(); + let orders = fields(bytes) + .map(Result::unwrap) + .find(|field| field.id == 7) + .unwrap() + .value; + + // column_orders is the last field: its header is one byte, delta 1 and wire type LIST + assert_eq!(bytes[orders.start - 1], 0x19); + + let without = spliced(bytes, orders.start - 1, orders.len() + 1, &[]); + let spans = chunk_spans(&without, &[1]).unwrap(); + + assert_eq!(spans.orders, None); + assert_eq!(path(&without, &spans.chunks[0][1]), b"name"); + } + + #[test] + fn a_row_group_without_the_asked_leaf_is_corrupt() { + assert_corrupt(chunk_spans(two_groups().bytes(), &[2]), "a row group without the leaf"); + } + + #[test] + fn a_truncated_footer_is_not_thrift() { + let footer = two_groups(); + let bytes = footer.bytes(); + + assert_corrupt(chunk_spans(&bytes[..bytes.len() / 2], &[0]), "The footer is not thrift"); + } + + #[test] + fn a_list_field_of_another_wire_type_is_corrupt() { + // row_groups (field 4) as the i32 1: header delta 4, wire type I32, zigzag 2, then the struct's stop + assert_corrupt(chunk_spans(b"\x45\x02\x00", &[0]), "another value where a list belongs"); + } + + #[test] + fn an_encrypted_column_chunk_is_refused() { + let footer = two_groups(); + let bytes = footer.bytes(); + + // crypto_metadata (8): ENCRYPTION_WITH_FOOTER_KEY, an empty struct in a union + assert!(matches!( + refuse_encrypted(&with_chunk_field(bytes, 8, 12, b"\x1c\x00\x00")), + Err(Error::Encrypted) + )); + // encrypted_column_metadata (9): one byte of binary + assert!(matches!( + refuse_encrypted(&with_chunk_field(bytes, 9, 8, b"\x01\xaa")), + Err(Error::Encrypted) + )); + assert!(refuse_encrypted(bytes).is_ok()); + } + + #[test] + fn an_encryption_algorithm_is_refused() { + let footer = two_groups(); + let bytes = footer.bytes(); + + // encryption_algorithm (8) after column_orders (7): AES_GCM_V1, an empty struct in a union + assert!(matches!( + refuse_encrypted(&spliced(bytes, bytes.len() - 1, 0, b"\x1c\x1c\x00\x00")), + Err(Error::Encrypted) + )); + } +} diff --git a/src/extension/arrow-ext/src/parquet/error.rs b/src/extension/arrow-ext/src/parquet/error.rs index 4817f37754..81ec31eebb 100644 --- a/src/extension/arrow-ext/src/parquet/error.rs +++ b/src/extension/arrow-ext/src/parquet/error.rs @@ -42,6 +42,11 @@ pub enum Error { /// Bytes that are not a Parquet file: no footer where it must be. NotParquet(String), Options(String), + /// A read plan arrow cannot follow: selectors that do not cover the file's rows, a malformed limit or row filter, + /// a row filter path that is no leaf of the file or bounds its column cannot take. + Plan(String), + /// A footer or a column chunk encrypted with Parquet modular encryption. + Encrypted, Stream(String), } diff --git a/src/extension/arrow-ext/src/parquet/fetch.rs b/src/extension/arrow-ext/src/parquet/fetch.rs deleted file mode 100644 index ce141cc755..0000000000 --- a/src/extension/arrow-ext/src/parquet/fetch.rs +++ /dev/null @@ -1,328 +0,0 @@ -//! A planned read's column chunks, fetched a row group at a time: the first page request inside a planned row group -//! reads every projected column chunk of that group - adjacent chunks in one read - and every later page header and -//! page body of the group is a slice of those buffers. arrow-rs's sync page reader asks for each page header -//! separately (`get_read`), so answering from the stream per request would read the file many times over. -//! -//! arrow-rs advances each column's pages on its own, so one batch may take column `a` into the next row group while -//! column `b` still reads the previous one: a group's buffers are dropped once every projected column has moved past -//! it. Memory: the projected compressed chunks of the row groups one batch spans - the bound arrow-rs's async reader -//! and DuckDB accept, row groups being the writer's unit. - -use std::collections::btree_map::Entry; -use std::collections::BTreeMap; -use std::ops::Range; -use std::sync::{Arc, Mutex}; - -use bytes::buf::Reader; -use bytes::{Buf, Bytes}; -use parquet::errors::ParquetError; -use parquet::file::metadata::ParquetMetaData; -use parquet::file::reader::{ChunkReader, Length}; - -use crate::parquet::source::{exact, ReadAt}; - -/// A column chunk of the file and, when the read plans it, where: its planned group, its projected column, and the -/// coalesced read of that group holding it. -struct Chunk { - range: Range, - planned: Option<(usize, usize, usize)>, -} - -#[derive(Default)] -struct Fetched { - /// Planned group → its reads' bytes. - buffers: BTreeMap>, - /// Per projected column, the planned group it reads now. - progress: Vec, -} - -pub struct ChunkFetch { - source: Arc, - size: u64, - /// Per planned group, its projected chunks with every run of adjacent ones merged, in file order. - reads: Vec>>, - /// Every column chunk of the file, by start: page requests find theirs by binary search. - chunks: Vec, - fetched: Mutex, -} - -impl ChunkFetch { - /// `row_groups`: the planned groups; `leaves`: the projected leaf columns. - pub fn new(source: Arc, size: u64, meta: &ParquetMetaData, row_groups: &[usize], leaves: &[usize]) -> Self { - let range = |(start, length): (u64, u64)| start..start + length; - let planned = row_groups - .iter() - .map(|group| { - let columns = meta.row_group(*group).columns(); - - leaves.iter().map(|leaf| range(columns[*leaf].byte_range())).collect() - }) - .collect(); - let mut is_planned = vec![false; meta.num_row_groups()]; - let mut is_projected = vec![false; meta.file_metadata().schema_descr().num_columns()]; - row_groups.iter().for_each(|group| is_planned[*group] = true); - leaves.iter().for_each(|leaf| is_projected[*leaf] = true); - let (is_planned, is_projected) = (&is_planned, &is_projected); - let unplanned = meta - .row_groups() - .iter() - .enumerate() - .flat_map(|(group, columns)| { - columns - .columns() - .iter() - .enumerate() - .filter(move |(leaf, _)| !is_planned[group] || !is_projected[*leaf]) - .map(|(_, chunk)| range(chunk.byte_range())) - }) - .collect(); - - Self::from_ranges(source, size, planned, unplanned) - } - - /// `planned`: per planned group, its projected chunks, by projected column; `unplanned`: every other chunk. - fn from_ranges(source: Arc, size: u64, planned: Vec>>, unplanned: Vec>) -> Self { - let columns = planned.first().map_or(0, Vec::len); - let reads = planned - .iter() - .map(|chunks| coalesced(chunks.clone())) - .collect::>(); - let mut chunks = unplanned - .into_iter() - .map(|range| Chunk { range, planned: None }) - .chain(planned.into_iter().enumerate().flat_map(|(group, chunks)| { - let reads = &reads[group]; - - chunks - .into_iter() - .enumerate() - .map(move |(column, range)| { - let read = reads - .iter() - .position(|read| read.start <= range.start && range.end <= read.end); - - Chunk { - planned: read.map(|read| (group, column, read)), - range, - } - }) - .collect::>() - })) - .collect::>(); - chunks.sort_by_key(|chunk| chunk.range.start); - - Self { - source, - size, - reads, - chunks, - fetched: Mutex::new(Fetched { - buffers: BTreeMap::new(), - progress: vec![0; columns], - }), - } - } - - /// The column chunk holding `offset`. - fn chunk_at(&self, offset: u64) -> Option<&Chunk> { - let after = self.chunks.partition_point(|chunk| chunk.range.start <= offset); - - after - .checked_sub(1) - .map(|index| &self.chunks[index]) - .filter(|chunk| offset < chunk.range.end) - } - - /// The fetched bytes of [start, end) when a planned column chunk holds it, fetching its row group first. - fn planned(&self, start: u64, end: u64) -> Result, ParquetError> { - let Some((group, column, read)) = self.chunk_at(start).and_then(|chunk| chunk.planned) else { - return Ok(None); - }; - let range = &self.reads[group][read]; - - if end > range.end { - return Ok(None); - } - - let mut fetched = self - .fetched - .lock() - .map_err(|_| ParquetError::General("chunk fetch poisoned".into()))?; - - if let Entry::Vacant(entry) = fetched.buffers.entry(group) { - entry.insert( - self.reads[group] - .iter() - .map(|range| exact(self.source.as_ref(), range.start, (range.end - range.start) as usize)) - .collect::, _>>()?, - ); - } - - let bytes = fetched.buffers[&group][read].slice((start - range.start) as usize..(end - range.start) as usize); - fetched.progress[column] = group; - let slowest = fetched.progress.iter().copied().min().unwrap_or(group); - fetched.buffers.retain(|fetched_group, _| *fetched_group >= slowest); - - Ok(Some(bytes)) - } -} - -/// Sorted ranges with every run of adjacent ones merged. -fn coalesced(mut ranges: Vec>) -> Vec> { - ranges.sort_by_key(|range| range.start); - let mut merged: Vec> = Vec::with_capacity(ranges.len()); - - for range in ranges { - match merged.last_mut() { - Some(last) if last.end == range.start => last.end = range.end, - _ => merged.push(range), - } - } - - merged -} - -impl Length for ChunkFetch { - fn len(&self) -> u64 { - self.size - } -} - -impl ChunkReader for ChunkFetch { - type T = Reader; - - /// A page header: the rest of its planned read; outside the plan, one exact read to the end of the column chunk - /// holding `start` (of the file when none does). - fn get_read(&self, start: u64) -> Result { - let end = self.chunk_at(start).map_or(self.size, |chunk| match chunk.planned { - Some((group, _, read)) => self.reads[group][read].end, - None => chunk.range.end, - }); - - Ok(match self.planned(start, end)? { - Some(bytes) => bytes, - None => exact(self.source.as_ref(), start, (end - start) as usize)?, - } - .reader()) - } - - fn get_bytes(&self, start: u64, length: usize) -> Result { - match self.planned(start, start + length as u64)? { - Some(bytes) => Ok(bytes), - None => exact(self.source.as_ref(), start, length), - } - } -} - -#[cfg(test)] -mod tests { - use std::io::Read; - use std::ops::Range; - use std::sync::Arc; - - use parquet::file::reader::ChunkReader; - - use super::{coalesced, ChunkFetch}; - use crate::parquet::source::fake::Counting; - - #[test] - fn adjacent_ranges_are_coalesced() { - assert_eq!(coalesced(vec![10..20, 0..10, 30..40, 20..25]), vec![0..25, 30..40]); - assert!(coalesced(vec![]).is_empty()); - } - - fn fetch( - source: &Arc, - planned: Vec>>, - unplanned: Vec>, - ) -> ChunkFetch { - ChunkFetch::from_ranges(Arc::clone(source), source.data.len() as u64, planned, unplanned) - } - - #[test] - fn a_read_outside_the_plan_is_one_exact_read_to_the_end_of_its_chunk() { - let source = Arc::new(Counting::new((0..100).collect())); - let fetch = fetch(&source, vec![vec![10..20]], vec![30..35, 40..60]); - let mut rest = Vec::new(); - - fetch.get_read(45).unwrap().read_to_end(&mut rest).unwrap(); - - assert_eq!(rest, (45..60).collect::>()); - assert_eq!(source.counted(), (1, 15)); - - rest.clear(); - fetch.get_read(90).unwrap().read_to_end(&mut rest).unwrap(); - - assert_eq!(rest, (90..100).collect::>()); - assert_eq!(fetch.get_bytes(70, 2).unwrap().to_vec(), vec![70, 71]); - assert_eq!(source.counted(), (3, 27)); - } - - #[test] - fn a_planned_group_is_fetched_once_in_coalesced_reads() { - let source = Arc::new(Counting::new((0..100).collect())); - let fetch = fetch(&source, vec![vec![0..10, 10..15, 20..30]], vec![]); - let mut header = Vec::new(); - - assert_eq!(fetch.get_bytes(22, 3).unwrap().to_vec(), vec![22, 23, 24]); - fetch.get_read(12).unwrap().read_to_end(&mut header).unwrap(); - assert_eq!(fetch.get_bytes(1, 2).unwrap().to_vec(), vec![1, 2]); - - assert_eq!(header, (12..15).collect::>()); - assert_eq!(source.counted(), (2, 25)); - } - - #[test] - fn a_group_is_dropped_once_every_column_moved_past_it() { - let source = Arc::new(Counting::new((0..100).collect())); - let fetch = fetch(&source, vec![vec![0..10, 20..30], vec![50..60, 70..80]], vec![]); - let groups = |fetch: &ChunkFetch| { - fetch - .fetched - .lock() - .unwrap() - .buffers - .keys() - .copied() - .collect::>() - }; - - fetch.get_bytes(0, 1).unwrap(); - fetch.get_bytes(50, 1).unwrap(); - assert_eq!(groups(&fetch), vec![0, 1]); - - fetch.get_bytes(21, 1).unwrap(); - assert_eq!(groups(&fetch), vec![0, 1]); - - fetch.get_bytes(71, 1).unwrap(); - assert_eq!(groups(&fetch), vec![1]); - assert_eq!(source.counted(), (4, 40)); - } - - #[test] - fn page_requests_find_their_chunk_among_many() { - let source = Arc::new(Counting::new((0..=255).cycle().take(40_000).collect())); - let planned = (0..100).map(|group| vec![group * 400..group * 400 + 100, group * 400 + 200..group * 400 + 300]); - let fetch = fetch( - &source, - planned.collect(), - (0..100).map(|group| group * 400 + 100..group * 400 + 200).collect(), - ); - - assert_eq!( - fetch.get_bytes(99 * 400 + 250, 2).unwrap().to_vec(), - source.data[39_850..39_852].to_vec() - ); - assert_eq!(source.counted(), (2, 200)); - - let mut header = Vec::new(); - fetch - .get_read(50 * 400 + 150) - .unwrap() - .read_to_end(&mut header) - .unwrap(); - - assert_eq!(header, source.data[20_150..20_200].to_vec()); - assert_eq!(source.counted(), (3, 250)); - } -} diff --git a/src/extension/arrow-ext/src/parquet/filter.rs b/src/extension/arrow-ext/src/parquet/filter.rs new file mode 100644 index 0000000000..a1d269de30 --- /dev/null +++ b/src/extension/arrow-ext/src/parquet/filter.rs @@ -0,0 +1,1234 @@ +//! The row filter `ReadPlanConverter::toExtension()` sends as `'filter'`, evaluated by arrow-rs's push decoder on the +//! predicate leaves of each row group before the projected leaves are decoded, one predicate per root conjunct. A row +//! is kept only when the filter is TRUE: `Not` is pushed down to the leaves as a polarity, so every node yields the +//! rows it is TRUE for, and a NULL (a null value, an absent column) is TRUE in neither polarity. Floats compare as +//! IEEE 754: NaN is false, -0.0 equals +0.0. + +use std::cmp::Ordering; + +use arrow_array::cast::AsArray; +use arrow_array::types::{ + Date32Type, Date64Type, Decimal128Type, Decimal256Type, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, + Int8Type, Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType, + TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, UInt16Type, + UInt32Type, UInt64Type, UInt8Type, +}; +use arrow_array::{Array, ArrayRef, ArrowPrimitiveType, BooleanArray, RecordBatch}; +use arrow_buffer::{i256, BooleanBuffer}; +use arrow_schema::{ArrowError, DataType, Schema, TimeUnit}; +use parquet::arrow::arrow_reader::ArrowPredicate; +use parquet::arrow::ProjectionMask; +use parquet::basic::{ConvertedType, LogicalType, Type}; +use parquet::schema::types::{ColumnDescriptor, SchemaDescriptor}; + +use crate::parquet::error::Error; +use crate::parquet::read::{struct_child, struct_field}; + +/// A PHP value as the converter's array holds it. +#[derive(Debug, Clone, PartialEq)] +pub enum Raw { + Null, + Bool(bool), + Int(i64), + Float(f64), + Bytes(Vec), + List(Vec), + Map(Vec<(Vec, Raw)>), +} + +/// A physical bound as PHP sends it: INT32/INT64 values (the UINT_64 bit pattern included), floats, booleans, bytes +/// (BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY, DECIMAL unscaled big-endian two's complement). +#[derive(Debug, Clone, PartialEq)] +enum Scalar { + Int(i64), + Float(f64), + Bool(bool), + Bytes(Vec), +} + +/// A null bound is unbounded. +#[derive(Debug, Clone, PartialEq)] +struct Interval { + lower: Option, + lower_inclusive: bool, + upper: Option, + upper_inclusive: bool, +} + +/// Per row of a leaf, whether its value matches; a null row's slot holds no value and is masked by its caller. +type Matcher = Box BooleanBuffer + Send>; + +/// The `ArrowPredicate` of one root conjunct, checked against the file's arrow schema when built. +pub struct PlanPredicate { + leaves: Vec, + mask: ProjectionMask, + root: Compiled, +} + +impl PlanPredicate { + /// One predicate per root conjunct of `filter`, in order: arrow-rs evaluates each on the rows the ones before kept. + pub fn conjuncts(filter: &Raw, schema: &Schema, descr: &SchemaDescriptor) -> Result, Error> { + conjuncts(filter)? + .into_iter() + .map(|conjunct| { + let mut leaves = Vec::new(); + let root = Compiled::parse(conjunct, schema, descr, &mut leaves)?; + + Ok(Self { + mask: ProjectionMask::leaves(descr, leaves.iter().copied()), + leaves, + root, + }) + }) + .collect() + } + + /// The parquet leaf of each path the conjunct reads. + pub fn leaves(&self) -> &[usize] { + &self.leaves + } +} + +impl ArrowPredicate for PlanPredicate { + fn projection(&self) -> &ProjectionMask { + &self.mask + } + + fn evaluate(&mut self, batch: RecordBatch) -> Result { + Ok(BooleanArray::new(self.root.truth(&batch, false)?, None)) + } +} + +/// A filter node with each leaf's match typed for the arrow array the leaf is read as. +enum Compiled { + And(Vec), + Or(Vec), + Not(Box), + Matches { + path: Vec, + matcher: Matcher, + }, + IsNull { + path: Vec, + }, + /// A leaf on a column absent from the file, null in every row: TRUE for an IS NULL, NULL for any other leaf. + Absent { + is_null: bool, + }, +} + +impl Compiled { + /// `leaves` gains the parquet leaf of each path the node reads, once. + fn parse(raw: &Raw, schema: &Schema, descr: &SchemaDescriptor, leaves: &mut Vec) -> Result { + let entries = map(raw, "a filter node")?; + let mut children = |raw: &Raw, what: &str| { + list(raw, what)? + .iter() + .map(|child| Self::parse(child, schema, descr, leaves)) + .collect::, _>>() + }; + + if let [(key, value)] = entries { + match key.as_slice() { + b"and" => return Ok(Self::And(children(value, "'and'")?)), + b"or" => return Ok(Self::Or(children(value, "'or'")?)), + b"not" => return Ok(Self::Not(Box::new(Self::parse(value, schema, descr, leaves)?))), + _ => {} + } + } + + Ok(match field(entries, b"leaf")? { + Raw::Bytes(kind) if kind == b"intervals" => { + let path = path(field(entries, b"path")?)?; + let intervals = list(field(entries, b"intervals")?, "'intervals'")? + .iter() + .map(interval) + .collect::, _>>()?; + let column = descr.column(leaf(descr, &path, leaves)?); + + Self::Matches { + matcher: bounds(&column, &path, &intervals, &data_type(schema, &path)?)?, + path, + } + } + Raw::Bytes(kind) if kind == b"is_null" => { + let path = path(field(entries, b"path")?)?; + leaf(descr, &path, leaves)?; + data_type(schema, &path)?; + + Self::IsNull { path } + } + Raw::Bytes(kind) if kind == b"starts_with" => { + let path = path(field(entries, b"path")?)?; + let prefix = match field(entries, b"prefix")? { + Raw::Bytes(prefix) => prefix.clone(), + other => return Err(malformed(format!("a prefix must be a string, got {other:?}"))), + }; + leaf(descr, &path, leaves)?; + let data_type = data_type(schema, &path)?; + + Self::Matches { + matcher: by_bytes( + ByteArray::of(&data_type).ok_or_else(|| unsupported(&path, &data_type))?, + move |value| value.starts_with(&prefix), + ), + path, + } + } + Raw::Bytes(kind) if kind == b"absent" => Self::Absent { + is_null: match field(entries, b"is_null")? { + Raw::Bool(is_null) => *is_null, + other => return Err(malformed(format!("'is_null' must be a bool, got {other:?}"))), + }, + }, + leaf => return Err(malformed(format!("unknown leaf {leaf:?}"))), + }) + } + + /// The rows the node is TRUE for: under `negated`, the rows its negation is TRUE for. + fn truth(&self, batch: &RecordBatch, negated: bool) -> Result { + let rows = batch.num_rows(); + + Ok(match self { + Self::And(children) | Self::Or(children) => { + let conjunction = matches!(self, Self::And(_)) != negated; + let mut result = if conjunction { + BooleanBuffer::new_set(rows) + } else { + BooleanBuffer::new_unset(rows) + }; + + for child in children { + let truth = child.truth(batch, negated)?; + result = if conjunction { + &result & &truth + } else { + &result | &truth + }; + } + + result + } + Self::Not(child) => child.truth(batch, !negated)?, + Self::Absent { is_null } if *is_null && !negated => BooleanBuffer::new_set(rows), + Self::Absent { .. } => BooleanBuffer::new_unset(rows), + Self::IsNull { path } => { + let (_, valid) = column(batch, path)?; + + if negated { + valid + } else { + !&valid + } + } + Self::Matches { path, matcher } => { + let (array, valid) = column(batch, path)?; + let matches = matcher(&array); + + if negated { + &valid & &!&matches + } else { + &valid & &matches + } + } + }) + } +} + +/// The root conjuncts of a filter, nested `and`s flattened. +fn conjuncts(raw: &Raw) -> Result, Error> { + match map(raw, "a filter node")? { + [(key, value)] if key == b"and" => Ok(list(value, "'and'")? + .iter() + .map(conjuncts) + .collect::, _>>()? + .into_iter() + .flatten() + .collect()), + _ => Ok(vec![raw]), + } +} + +/// The matcher of `intervals` for the arrow array arrow-rs reads `column` as: the file's arrow schema can turn an INT32 +/// DATE into Date64 milliseconds, a plain INT32/INT64 into a timestamp or a Date64, values unchanged. +fn bounds( + column: &ColumnDescriptor, + path: &[String], + intervals: &[Interval], + data_type: &DataType, +) -> Result { + let malformed_bounds = || malformed(format!("bounds of \"{}\" are not {data_type} values", path.join("."))); + let signed = |factor: i64| { + typed(intervals, |scalar| match scalar { + Scalar::Int(value) => value.checked_mul(factor), + _ => None, + }) + .ok_or_else(malformed_bounds) + }; + let unsigned = |mask: u64| { + typed(intervals, |scalar| match scalar { + Scalar::Int(value) => Some(*value as u64 & mask), + _ => None, + }) + .ok_or_else(malformed_bounds) + }; + let floats = || { + typed(intervals, |scalar| match scalar { + Scalar::Float(value) => Some(*value), + Scalar::Int(value) => Some(*value as f64), + _ => None, + }) + .ok_or_else(malformed_bounds) + }; + let stored_unit = timestamp_unit(column.logical_type_ref(), column.converted_type()); + + Ok(match data_type { + DataType::Int8 => primitive::(signed(1)?), + DataType::Int16 => primitive::(signed(1)?), + DataType::Int32 => primitive::(signed(1)?), + DataType::Int64 => primitive::(signed(1)?), + DataType::Date32 => primitive::(signed(1)?), + DataType::Date64 if column.physical_type() == Type::INT64 => primitive::(signed(1)?), + DataType::Date64 => primitive::(signed(86_400_000)?), + DataType::Time32(TimeUnit::Second) => primitive::(signed(1)?), + DataType::Time32(_) => primitive::(signed(1)?), + DataType::Time64(TimeUnit::Microsecond) => primitive::(signed(1)?), + DataType::Time64(_) => primitive::(signed(1)?), + DataType::Timestamp(unit, _) if stored_unit.is_none_or(|stored| stored == *unit) => match unit { + TimeUnit::Second => primitive::(signed(1)?), + TimeUnit::Millisecond => primitive::(signed(1)?), + TimeUnit::Microsecond => primitive::(signed(1)?), + TimeUnit::Nanosecond => primitive::(signed(1)?), + }, + DataType::UInt8 => primitive::(unsigned(0xFFFF_FFFF)?), + DataType::UInt16 => primitive::(unsigned(0xFFFF_FFFF)?), + DataType::UInt32 => primitive::(unsigned(0xFFFF_FFFF)?), + DataType::UInt64 => primitive::(unsigned(u64::MAX)?), + DataType::Float32 => primitive::(floats()?), + DataType::Float64 => primitive::(floats()?), + DataType::Boolean => { + let intervals = typed(intervals, |scalar| match scalar { + Scalar::Bool(value) => Some(*value), + _ => None, + }) + .ok_or_else(malformed_bounds)?; + + Box::new(move |array: &ArrayRef| { + let values = array.as_boolean().values(); + + ordered(values.len(), |row| values.value(row), &intervals) + }) + } + DataType::Decimal128(_, scale) if i32::from(*scale) == column.type_scale() => primitive::( + typed(intervals, |scalar| match scalar { + Scalar::Int(value) => Some(i128::from(*value)), + Scalar::Bytes(bytes) => be_i128(bytes), + _ => None, + }) + .ok_or_else(malformed_bounds)?, + ), + DataType::Decimal256(_, scale) if i32::from(*scale) == column.type_scale() => primitive::( + typed(intervals, |scalar| match scalar { + Scalar::Int(value) => Some(i256::from_i128(i128::from(*value))), + Scalar::Bytes(bytes) => be_i256(bytes), + _ => None, + }) + .ok_or_else(malformed_bounds)?, + ), + other => { + let byte_array = ByteArray::of(other).ok_or_else(|| unsupported(path, other))?; + let intervals = typed(intervals, |scalar| match scalar { + Scalar::Bytes(bytes) => Some(bytes.clone()), + _ => None, + }) + .ok_or_else(malformed_bounds)?; + + by_bytes(byte_array, move |value| { + holds(&intervals, |bound| Some(value.cmp(bound.as_slice()))) + }) + } + }) +} + +/// The arrow types whose rows are bytes. +#[derive(Clone, Copy)] +enum ByteArray { + Utf8, + LargeUtf8, + Utf8View, + Binary, + LargeBinary, + BinaryView, + FixedSizeBinary, +} + +impl ByteArray { + fn of(data_type: &DataType) -> Option { + Some(match data_type { + DataType::Utf8 => Self::Utf8, + DataType::LargeUtf8 => Self::LargeUtf8, + DataType::Utf8View => Self::Utf8View, + DataType::Binary => Self::Binary, + DataType::LargeBinary => Self::LargeBinary, + DataType::BinaryView => Self::BinaryView, + DataType::FixedSizeBinary(_) => Self::FixedSizeBinary, + _ => return None, + }) + } +} + +/// A matcher testing each row's bytes. +fn by_bytes(byte_array: ByteArray, test: impl Fn(&[u8]) -> bool + Send + 'static) -> Matcher { + macro_rules! rows { + ($cast:ident $(::<$offset:ty>)?) => { + Box::new(move |array: &ArrayRef| { + let array = array.$cast$(::<$offset>)?(); + + BooleanBuffer::collect_bool(array.len(), |row| { + let value: &[u8] = array.value(row).as_ref(); + + test(value) + }) + }) + }; + } + + match byte_array { + ByteArray::Utf8 => rows!(as_string::), + ByteArray::LargeUtf8 => rows!(as_string::), + ByteArray::Utf8View => rows!(as_string_view), + ByteArray::Binary => rows!(as_binary::), + ByteArray::LargeBinary => rows!(as_binary::), + ByteArray::BinaryView => rows!(as_binary_view), + ByteArray::FixedSizeBinary => rows!(as_fixed_size_binary), + } +} + +impl Interval { + /// `order(bound)` orders the value against a bound; `None` (NaN) is in no bounded interval. + fn contains(&self, order: impl Fn(&T) -> Option) -> bool { + let lower = self.lower.as_ref().is_none_or(|bound| match order(bound) { + Some(Ordering::Greater) => true, + Some(Ordering::Equal) => self.lower_inclusive, + _ => false, + }); + let upper = self.upper.as_ref().is_none_or(|bound| match order(bound) { + Some(Ordering::Less) => true, + Some(Ordering::Equal) => self.upper_inclusive, + _ => false, + }); + + lower && upper + } +} + +/// Whether the value lies in the sorted, disjoint `intervals`: only the last one starting at or below it can hold it. +fn holds(intervals: &[Interval], order: impl Fn(&T) -> Option) -> bool { + let candidate = intervals.partition_point(|interval| { + interval + .lower + .as_ref() + .is_none_or(|bound| matches!(order(bound), Some(Ordering::Greater | Ordering::Equal))) + }); + + candidate > 0 && intervals[candidate - 1].contains(order) +} + +/// The values of an `A` array, widened to the bounds' type. +fn primitive(intervals: Vec>) -> Matcher +where + A: ArrowPrimitiveType, + T: PartialOrd + From + Send + 'static, +{ + Box::new(move |array: &ArrayRef| { + let values = array.as_primitive::().values(); + + ordered(values.len(), |row| T::from(values[row]), &intervals) + }) +} + +/// Floats compare as IEEE 754 through `partial_cmp`: NaN orders with nothing, itself included, so it is in no interval, +/// not even an unbounded one, and -0.0 == +0.0. +fn ordered(rows: usize, value: impl Fn(usize) -> T, intervals: &[Interval]) -> BooleanBuffer { + BooleanBuffer::collect_bool(rows, |row| { + let value = value(row); + + value.partial_cmp(&value).is_some() && holds(intervals, |bound| value.partial_cmp(bound)) + }) +} + +/// The leaf at `path` in a predicate batch, and its rows that are not null, a struct above it included. +fn column(batch: &RecordBatch, path: &[String]) -> Result<(ArrayRef, BooleanBuffer), ArrowError> { + let missing = || ArrowError::SchemaError(format!("The row filter batch has no leaf \"{}\"", path.join("."))); + let (array, nulls) = + struct_child(batch.column_by_name(&path[0]).ok_or_else(missing)?, &path[1..]).map_err(|_| missing())?; + let valid = nulls.map_or_else(|| BooleanBuffer::new_set(array.len()), |nulls| nulls.into_inner()); + + Ok((array, valid)) +} + +/// The parquet leaf at `path`, added to `leaves` once. +fn leaf(descr: &SchemaDescriptor, path: &[String], leaves: &mut Vec) -> Result { + let leaf = (0..descr.num_columns()) + .find(|leaf| descr.column(*leaf).path().parts() == path) + .ok_or_else(|| not_a_leaf(path))?; + + if !leaves.contains(&leaf) { + leaves.push(leaf); + } + + Ok(leaf) +} + +fn not_a_leaf(path: &[String]) -> Error { + Error::Plan(format!( + "Row filter column \"{}\" is not a leaf of the file", + path.join(".") + )) +} + +/// The arrow type the leaf at `path` is read as. +fn data_type(schema: &Schema, path: &[String]) -> Result { + Ok(struct_field(schema, path) + .map_err(|_| not_a_leaf(path))? + .0 + .data_type() + .clone()) +} + +fn timestamp_unit(logical: Option<&LogicalType>, converted: ConvertedType) -> Option { + match (logical, converted) { + (Some(LogicalType::Timestamp(timestamp)), _) => Some(match timestamp.unit { + parquet::basic::TimeUnit::MILLIS => TimeUnit::Millisecond, + parquet::basic::TimeUnit::MICROS => TimeUnit::Microsecond, + parquet::basic::TimeUnit::NANOS => TimeUnit::Nanosecond, + }), + (_, ConvertedType::TIMESTAMP_MILLIS) => Some(TimeUnit::Millisecond), + (_, ConvertedType::TIMESTAMP_MICROS) => Some(TimeUnit::Microsecond), + _ => None, + } +} + +/// Every bound converted; `None` when one has another kind. +fn typed(intervals: &[Interval], convert: impl Fn(&Scalar) -> Option) -> Option>> { + intervals + .iter() + .map(|interval| { + Some(Interval { + lower: match &interval.lower { + Some(bound) => Some(convert(bound)?), + None => None, + }, + lower_inclusive: interval.lower_inclusive, + upper: match &interval.upper { + Some(bound) => Some(convert(bound)?), + None => None, + }, + upper_inclusive: interval.upper_inclusive, + }) + }) + .collect() +} + +/// Big-endian two's complement of any width, `None` when it does not fit. +fn be_i128(bytes: &[u8]) -> Option { + sign_extended::<16>(bytes).map(i128::from_be_bytes) +} + +fn be_i256(bytes: &[u8]) -> Option { + sign_extended::<32>(bytes).map(i256::from_be_bytes) +} + +fn sign_extended(bytes: &[u8]) -> Option<[u8; N]> { + let extension = if bytes.first().is_some_and(|byte| *byte >= 0x80) { + 0xFF + } else { + 0x00 + }; + let mut bytes = bytes; + + while bytes.len() > N { + let (first, rest) = bytes.split_first()?; + + // dropping a sign-extension byte keeps the value only when the next byte carries the same sign + if *first != extension || (rest[0] >= 0x80) != (extension == 0xFF) { + return None; + } + + bytes = rest; + } + + let mut extended = [extension; N]; + extended[N - bytes.len()..].copy_from_slice(bytes); + + Some(extended) +} + +fn unsupported(path: &[String], data_type: &DataType) -> Error { + Error::Unsupported { + column: path.join("."), + parquet: format!("{data_type} in a row filter"), + } +} + +fn malformed(message: String) -> Error { + Error::Plan(format!("arrow received a malformed row filter: {message}")) +} + +fn map<'a>(raw: &'a Raw, what: &str) -> Result<&'a [(Vec, Raw)], Error> { + match raw { + Raw::Map(entries) => Ok(entries), + other => Err(malformed(format!( + "{what} must be an array with string keys, got {other:?}" + ))), + } +} + +fn list<'a>(raw: &'a Raw, what: &str) -> Result<&'a [Raw], Error> { + match raw { + Raw::List(entries) => Ok(entries), + other => Err(malformed(format!("{what} must be a list, got {other:?}"))), + } +} + +fn field<'a>(entries: &'a [(Vec, Raw)], key: &[u8]) -> Result<&'a Raw, Error> { + entries + .iter() + .find(|(name, _)| name == key) + .map(|(_, value)| value) + .ok_or_else(|| malformed(format!("missing key \"{}\"", String::from_utf8_lossy(key)))) +} + +fn path(raw: &Raw) -> Result, Error> { + let segments = list(raw, "a path")? + .iter() + .map(|segment| match segment { + Raw::Bytes(segment) => { + String::from_utf8(segment.clone()).map_err(|_| malformed("a path segment must be UTF-8".to_string())) + } + other => Err(malformed(format!("a path segment must be a string, got {other:?}"))), + }) + .collect::, _>>()?; + + if segments.is_empty() { + return Err(malformed("a path must not be empty".to_string())); + } + + Ok(segments) +} + +fn interval(raw: &Raw) -> Result, Error> { + let parts = list(raw, "an interval")?; + let [lower, lower_inclusive, upper, upper_inclusive] = parts else { + return Err(malformed(format!("an interval has 4 parts, got {}", parts.len()))); + }; + let inclusive = |raw: &Raw| match raw { + Raw::Bool(inclusive) => Ok(*inclusive), + other => Err(malformed(format!( + "an interval inclusiveness must be a bool, got {other:?}" + ))), + }; + + Ok(Interval { + lower: scalar(lower)?, + lower_inclusive: inclusive(lower_inclusive)?, + upper: scalar(upper)?, + upper_inclusive: inclusive(upper_inclusive)?, + }) +} + +fn scalar(raw: &Raw) -> Result, Error> { + Ok(match raw { + Raw::Null => None, + Raw::Bool(value) => Some(Scalar::Bool(*value)), + Raw::Int(value) => Some(Scalar::Int(*value)), + Raw::Float(value) if value.is_nan() => return Err(malformed("a bound must not be NaN".to_string())), + Raw::Float(value) => Some(Scalar::Float(*value)), + Raw::Bytes(value) => Some(Scalar::Bytes(value.clone())), + other => return Err(malformed(format!("a bound must be a scalar, got {other:?}"))), + }) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::{ + Array, ArrayRef, BooleanArray, Date32Array, Date64Array, Decimal128Array, FixedSizeBinaryArray, Float64Array, + Int64Array, RecordBatch, StringArray, StructArray, TimestampSecondArray, UInt64Array, + }; + use arrow_array::{Decimal256Array, Float32Array, Time32MillisecondArray, UInt32Array}; + use arrow_buffer::i256; + use arrow_buffer::NullBuffer; + use arrow_schema::{DataType, Field, Fields, TimeUnit}; + use parquet::arrow::arrow_reader::ArrowPredicate; + use parquet::arrow::parquet_to_arrow_schema; + use parquet::schema::parser::parse_message_type; + use parquet::schema::types::SchemaDescriptor; + + use super::{Interval, PlanPredicate, Raw, Scalar}; + use crate::parquet::error::Error; + + const SCHEMA: &str = "message schema { + optional int64 i; + optional double f; + optional binary s (STRING); + optional int32 day (DATE); + optional fixed_len_byte_array(2) d (DECIMAL(4, 2)); + optional int64 u (INTEGER(64, false)); + optional fixed_len_byte_array(16) id (UUID); + optional group address { optional binary city (STRING); } + optional group tags (LIST) { repeated group list { optional binary element (STRING); } } + }"; + + fn descr() -> SchemaDescriptor { + SchemaDescriptor::new(Arc::new(parse_message_type(SCHEMA).unwrap())) + } + + fn rows(filter: &Raw, column: &str, array: ArrayRef) -> Vec { + let descr = descr(); + let schema = parquet_to_arrow_schema(&descr, None).unwrap(); + let field = schema.field_with_name(column).unwrap().clone(); + + kept(filter, &arrow_schema::Schema::new(vec![field]), &descr, array) + } + + /// The rows every conjunct's predicate keeps. + fn kept(filter: &Raw, schema: &arrow_schema::Schema, descr: &SchemaDescriptor, array: ArrayRef) -> Vec { + let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![array]).unwrap(); + let mut kept = vec![true; batch.num_rows()]; + + for mut predicate in PlanPredicate::conjuncts(filter, schema, descr).unwrap() { + let truth = predicate.evaluate(batch.clone()).unwrap(); + + assert_eq!(truth.null_count(), 0); + + kept = kept + .iter() + .zip(truth.iter()) + .map(|(kept, truth)| *kept && truth.unwrap()) + .collect(); + } + + kept + } + + fn node(entries: Vec<(&str, Raw)>) -> Raw { + map(entries) + } + + fn path(path: &[&str]) -> Raw { + Raw::List(path.iter().map(|segment| bytes(segment)).collect()) + } + + fn bound(bound: Option) -> Raw { + match bound { + None => Raw::Null, + Some(Scalar::Int(value)) => Raw::Int(value), + Some(Scalar::Float(value)) => Raw::Float(value), + Some(Scalar::Bool(value)) => Raw::Bool(value), + Some(Scalar::Bytes(value)) => Raw::Bytes(value), + } + } + + fn intervals_at(at: &[&str], intervals: Vec>) -> Raw { + node(vec![ + ("leaf", bytes("intervals")), + ("path", path(at)), + ( + "intervals", + Raw::List( + intervals + .into_iter() + .map(|interval| { + Raw::List(vec![ + bound(interval.lower), + Raw::Bool(interval.lower_inclusive), + bound(interval.upper), + Raw::Bool(interval.upper_inclusive), + ]) + }) + .collect(), + ), + ), + ]) + } + + fn intervals(column: &str, intervals: Vec>) -> Raw { + intervals_at(&[column], intervals) + } + + fn is_null(at: &[&str]) -> Raw { + node(vec![("leaf", bytes("is_null")), ("path", path(at))]) + } + + fn starts_with(at: &[&str], prefix: &[u8]) -> Raw { + node(vec![ + ("leaf", bytes("starts_with")), + ("path", path(at)), + ("prefix", Raw::Bytes(prefix.to_vec())), + ]) + } + + fn absent(is_null: bool) -> Raw { + node(vec![("leaf", bytes("absent")), ("is_null", Raw::Bool(is_null))]) + } + + fn and(nodes: Vec) -> Raw { + node(vec![("and", Raw::List(nodes))]) + } + + fn or(nodes: Vec) -> Raw { + node(vec![("or", Raw::List(nodes))]) + } + + fn below(bound: T, inclusive: bool) -> Interval { + Interval { + lower: None, + lower_inclusive: false, + upper: Some(bound), + upper_inclusive: inclusive, + } + } + + fn point(bound: T) -> Interval { + Interval { + lower: Some(bound.clone()), + lower_inclusive: true, + upper: Some(bound), + upper_inclusive: true, + } + } + + fn not(negated: Raw) -> Raw { + node(vec![("not", negated)]) + } + + fn ints() -> ArrayRef { + Arc::new(Int64Array::from(vec![None, Some(3), Some(7)])) + } + + #[test] + fn a_null_stays_null_under_not_and_is_never_kept() { + let lt = intervals("i", vec![below(Scalar::Int(5), false)]); + + assert_eq!(rows(<, "i", ints()), [false, true, false]); + assert_eq!(rows(¬(lt.clone()), "i", ints()), [false, false, true]); + assert_eq!(rows(¬(not(lt)), "i", ints()), [false, true, false]); + } + + #[test] + fn and_and_or_follow_kleene_logic_and_swap_under_not() { + let lt = intervals("i", vec![below(Scalar::Int(5), false)]); + let null = is_null(&["i"]); + + assert_eq!( + rows(&or(vec![lt.clone(), null.clone()]), "i", ints()), + [true, true, false] + ); + assert_eq!( + rows(¬(and(vec![lt.clone(), not(null.clone())])), "i", ints()), + [true, false, true] + ); + assert_eq!(rows(¬(or(vec![lt, null])), "i", ints()), [false, false, true]); + assert_eq!(rows(&and(vec![]), "i", ints()), [true, true, true]); + assert_eq!(rows(¬(and(vec![])), "i", ints()), [false, false, false]); + assert_eq!(rows(&or(vec![]), "i", ints()), [false, false, false]); + } + + #[test] + fn nan_is_in_no_interval_so_every_negation_keeps_it() { + let floats: ArrayRef = Arc::new(Float64Array::from(vec![Some(f64::NAN), Some(3.0), Some(5.0), None])); + let lt = intervals("f", vec![below(Scalar::Float(5.0), false)]); + let unbounded = intervals( + "f", + vec![Interval { + lower: None, + lower_inclusive: false, + upper: None, + upper_inclusive: false, + }], + ); + + assert_eq!(rows(<, "f", Arc::clone(&floats)), [false, true, false, false]); + assert_eq!(rows(¬(lt), "f", Arc::clone(&floats)), [true, false, true, false]); + assert_eq!( + rows( + ¬(intervals("f", vec![point(Scalar::Float(5.0))])), + "f", + Arc::clone(&floats) + ), + [true, true, false, false] + ); + assert_eq!(rows(&unbounded, "f", floats), [false, true, true, false]); + } + + #[test] + fn negative_zero_equals_positive_zero() { + let zeros: ArrayRef = Arc::new(Float64Array::from(vec![-0.0, 0.0])); + + assert_eq!( + rows( + &intervals("f", vec![point(Scalar::Float(0.0))]), + "f", + Arc::clone(&zeros) + ), + [true, true] + ); + assert_eq!( + rows(&intervals("f", vec![below(Scalar::Float(0.0), false)]), "f", zeros), + [false, false] + ); + } + + #[test] + fn a_decimal_bound_of_any_width_compares_with_the_unscaled_value() { + let decimals: ArrayRef = Arc::new( + Decimal128Array::from(vec![Some(1234), Some(1233), Some(-1234), None]) + .with_precision_and_scale(4, 2) + .unwrap(), + ); + let wide = Scalar::Bytes(vec![0, 0, 0, 0, 0, 0, 0x04, 0xD2]); + + assert_eq!( + rows(&intervals("d", vec![point(wide.clone())]), "d", Arc::clone(&decimals)), + [true, false, false, false] + ); + assert_eq!( + rows(&intervals("d", vec![below(wide, false)]), "d", Arc::clone(&decimals)), + [false, true, true, false] + ); + assert_eq!( + rows( + &intervals("d", vec![below(Scalar::Bytes(vec![0xFF, 0x00]), true)]), + "d", + decimals + ), + [false, false, true, false] + ); + } + + #[test] + fn a_date_bound_is_its_day_number() { + let days: ArrayRef = Arc::new(Date32Array::from(vec![1, 2, 3])); + + assert_eq!( + rows(&intervals("day", vec![point(Scalar::Int(2))]), "day", days), + [false, true, false] + ); + } + + #[test] + fn an_unsigned_64_bit_pattern_compares_unsigned() { + let big: ArrayRef = Arc::new(UInt64Array::from(vec![5, u64::MAX])); + + assert_eq!( + rows(&intervals("u", vec![below(Scalar::Int(-1), false)]), "u", big), + [true, false] + ); + } + + #[test] + fn bytes_compare_unsigned_and_prefixes_match_raw_bytes() { + let strings: ArrayRef = Arc::new(StringArray::from(vec![Some("a"), Some("ź"), Some(""), None])); + let prefix = |prefix: &str| starts_with(&["s"], prefix.as_bytes()); + + assert_eq!( + rows( + &intervals("s", vec![below(Scalar::Bytes(b"z".to_vec()), false)]), + "s", + Arc::clone(&strings) + ), + [true, false, true, false] + ); + assert_eq!(rows(&prefix(""), "s", Arc::clone(&strings)), [true, true, true, false]); + assert_eq!( + rows(¬(prefix("")), "s", Arc::clone(&strings)), + [false, false, false, false] + ); + assert_eq!( + rows(&prefix("\u{c5}"), "s", Arc::clone(&strings)), + [false, false, false, false] + ); + assert_eq!( + rows(&starts_with(&["s"], &[0xC5]), "s", strings), + [false, true, false, false] + ); + } + + #[test] + fn a_uuid_compares_its_sixteen_bytes() { + let ids: ArrayRef = + Arc::new(FixedSizeBinaryArray::try_from_iter(vec![[1u8; 16], [2u8; 16]].into_iter()).unwrap()); + + assert_eq!( + rows(&intervals("id", vec![point(Scalar::Bytes(vec![2; 16]))]), "id", ids), + [false, true] + ); + } + + #[test] + fn an_absent_column_is_null_in_every_row() { + assert_eq!(rows(&absent(false), "i", ints()), [false, false, false]); + assert_eq!(rows(¬(absent(false)), "i", ints()), [false, false, false]); + assert_eq!(rows(&absent(true), "i", ints()), [true, true, true]); + assert_eq!(rows(¬(absent(true)), "i", ints()), [false, false, false]); + } + + #[test] + fn a_value_is_found_among_sorted_intervals() { + let values: ArrayRef = Arc::new(Int64Array::from(vec![1, 2, 5, 6, 8, 9])); + let above = Interval { + lower: Some(Scalar::Int(8)), + lower_inclusive: false, + upper: None, + upper_inclusive: false, + }; + + assert_eq!( + rows( + &intervals("i", vec![below(Scalar::Int(2), false), point(Scalar::Int(5)), above]), + "i", + values + ), + [true, false, true, false, false, true] + ); + } + + #[test] + fn each_root_conjunct_is_its_own_predicate_reading_its_own_leaves() { + let descr = descr(); + let schema = parquet_to_arrow_schema(&descr, None).unwrap(); + let null = |column: &str| is_null(&[column]); + let predicates = PlanPredicate::conjuncts( + &and(vec![null("s"), and(vec![null("i"), or(vec![null("f"), null("i")])])]), + &schema, + &descr, + ) + .unwrap(); + + assert_eq!( + predicates.iter().map(PlanPredicate::leaves).collect::>(), + [&[2][..], &[0], &[1, 0]] + ); + assert!(PlanPredicate::conjuncts(&and(vec![]), &schema, &descr) + .unwrap() + .is_empty()); + } + + #[test] + fn every_primitive_kind_compares_in_its_own_type() { + let descr = SchemaDescriptor::new(Arc::new( + parse_message_type( + "message schema { + optional int32 u32 (INTEGER(32, false)); + optional float f32; + optional int32 t (TIME(MILLIS, true)); + optional fixed_len_byte_array(17) d (DECIMAL(40, 2)); + }", + ) + .unwrap(), + )); + let schema = parquet_to_arrow_schema(&descr, None).unwrap(); + let single = |name: &str| arrow_schema::Schema::new(vec![schema.field_with_name(name).unwrap().clone()]); + let cases: Vec<(Raw, &str, ArrayRef, Vec)> = vec![ + ( + intervals("u32", vec![below(Scalar::Int(-1), false)]), + "u32", + Arc::new(UInt32Array::from(vec![5, u32::MAX])), + vec![true, false], + ), + ( + not(intervals("f32", vec![below(Scalar::Float(5.0), false)])), + "f32", + Arc::new(Float32Array::from(vec![f32::NAN, 3.0, 5.0])), + vec![true, false, true], + ), + ( + intervals("t", vec![point(Scalar::Int(2_000))]), + "t", + Arc::new(Time32MillisecondArray::from(vec![1_000, 2_000])), + vec![false, true], + ), + ( + intervals("d", vec![below(Scalar::Bytes(vec![0x04, 0xD2]), false)]), + "d", + Arc::new( + Decimal256Array::from(vec![i256::from_i128(1234), i256::from_i128(-1234)]) + .with_precision_and_scale(40, 2) + .unwrap(), + ), + vec![false, true], + ), + ]; + + for (filter, column, array, expected) in cases { + assert_eq!(kept(&filter, &single(column), &descr, array), expected, "{column}"); + } + } + + #[test] + fn a_date64_hint_turns_day_bounds_into_milliseconds_and_a_timestamp_hint_keeps_plain_integers() { + let descr = SchemaDescriptor::new(Arc::new( + parse_message_type("message schema { optional int32 day (DATE); optional int64 seconds; }").unwrap(), + )); + let day = Field::new("day", DataType::Date64, true); + let seconds = Field::new("seconds", DataType::Timestamp(TimeUnit::Second, None), true); + + assert_eq!( + kept( + &intervals("day", vec![point(Scalar::Int(2))]), + &arrow_schema::Schema::new(vec![day]), + &descr, + Arc::new(Date64Array::from(vec![86_400_000, 172_800_000, 172_800_001])) + ), + [false, true, false] + ); + assert_eq!( + kept( + &intervals("seconds", vec![below(Scalar::Int(2), true)]), + &arrow_schema::Schema::new(vec![seconds]), + &descr, + Arc::new(TimestampSecondArray::from(vec![2, 3])) + ), + [true, false] + ); + } + + #[test] + fn a_struct_leaf_is_null_where_its_struct_is() { + let cities = Arc::new(StringArray::from(vec![Some("a"), Some("a")])) as ArrayRef; + let address: ArrayRef = Arc::new(StructArray::new( + Fields::from(vec![Field::new("city", DataType::Utf8, true)]), + vec![cities], + Some(NullBuffer::from(vec![true, false])), + )); + let city = intervals_at(&["address", "city"], vec![point(Scalar::Bytes(b"a".to_vec()))]); + let null = is_null(&["address", "city"]); + + assert_eq!(rows(&city, "address", Arc::clone(&address)), [true, false]); + assert_eq!(rows(¬(city), "address", Arc::clone(&address)), [false, false]); + assert_eq!(rows(&null, "address", address), [false, true]); + } + + #[test] + fn booleans_order_false_before_true() { + let flags: ArrayRef = Arc::new(BooleanArray::from(vec![false, true])); + let descr = SchemaDescriptor::new(Arc::new( + parse_message_type("message schema { optional boolean b; }").unwrap(), + )); + let schema = parquet_to_arrow_schema(&descr, None).unwrap(); + + assert_eq!( + kept( + &intervals("b", vec![below(Scalar::Bool(true), false)]), + &schema, + &descr, + flags + ), + [true, false] + ); + } + + #[test] + fn a_bound_of_another_kind_than_the_column_or_a_path_that_is_no_leaf_is_refused() { + let descr = descr(); + let schema = parquet_to_arrow_schema(&descr, None).unwrap(); + let refused = |filter: Raw| match PlanPredicate::conjuncts(&filter, &schema, &descr) { + Err(Error::Plan(message)) => message, + _ => panic!("the filter was not refused"), + }; + + assert_eq!( + refused(intervals("i", vec![point(Scalar::Bytes(b"1".to_vec()))])), + "arrow received a malformed row filter: bounds of \"i\" are not Int64 values" + ); + assert_eq!( + refused(is_null(&["missing"])), + "Row filter column \"missing\" is not a leaf of the file" + ); + assert_eq!( + refused(is_null(&["i", "deeper"])), + "Row filter column \"i.deeper\" is not a leaf of the file" + ); + assert_eq!( + refused(is_null(&["tags", "list", "element"])), + "Row filter column \"tags.list.element\" is not a leaf of the file" + ); + } + + fn map(entries: Vec<(&str, Raw)>) -> Raw { + Raw::Map( + entries + .into_iter() + .map(|(key, value)| (key.as_bytes().to_vec(), value)) + .collect(), + ) + } + + fn bytes(value: &str) -> Raw { + Raw::Bytes(value.as_bytes().to_vec()) + } + + #[test] + fn the_converter_shape_compiles_into_one_predicate_per_root_conjunct() { + let descr = descr(); + let schema = parquet_to_arrow_schema(&descr, None).unwrap(); + let predicates = PlanPredicate::conjuncts( + &and(vec![ + not(absent(true)), + or(vec![intervals_at( + &["address", "city"], + vec![Interval { + lower: Some(Scalar::Bytes(b"a".to_vec())), + lower_inclusive: true, + upper: None, + upper_inclusive: false, + }], + )]), + is_null(&["i"]), + starts_with(&["s"], b"p"), + and(vec![]), + ]), + &schema, + &descr, + ) + .unwrap(); + + assert_eq!( + predicates.iter().map(PlanPredicate::leaves).collect::>(), + [&[][..], &[7], &[0], &[2]] + ); + } + + #[test] + fn an_unknown_leaf_or_a_missing_key_is_refused() { + let descr = descr(); + let schema = parquet_to_arrow_schema(&descr, None).unwrap(); + let refused = |filter: Raw| match PlanPredicate::conjuncts(&filter, &schema, &descr) { + Err(Error::Plan(message)) => message.replace("arrow received a malformed row filter: ", ""), + _ => panic!("the filter was not refused"), + }; + + assert_eq!(refused(map(vec![("xor", Raw::List(vec![]))])), "missing key \"leaf\""); + assert!(refused(map(vec![("leaf", bytes("between"))])).starts_with("unknown leaf")); + assert_eq!(refused(map(vec![("leaf", bytes("is_null"))])), "missing key \"path\""); + assert_eq!(refused(map(vec![("leaf", bytes("absent"))])), "missing key \"is_null\""); + assert_eq!( + refused(map(vec![ + ("leaf", bytes("intervals")), + ("path", Raw::List(vec![bytes("f")])), + ( + "intervals", + Raw::List(vec![Raw::List(vec![ + Raw::Float(f64::NAN), + Raw::Bool(true), + Raw::Null, + Raw::Bool(false) + ])]) + ), + ])), + "a bound must not be NaN" + ); + } +} diff --git a/src/extension/arrow-ext/src/parquet/footer.rs b/src/extension/arrow-ext/src/parquet/footer.rs index eab46a8ae6..d12275889b 100644 --- a/src/extension/arrow-ext/src/parquet/footer.rs +++ b/src/extension/arrow-ext/src/parquet/footer.rs @@ -7,6 +7,7 @@ use bytes::{Bytes, BytesMut}; use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader}; use parquet::file::reader::ChunkReader; +use crate::parquet::chunks::refuse_encrypted; use crate::parquet::error::Error; /// What the first read takes from the end of the file: the whole footer of every file it is shorter than. @@ -14,6 +15,9 @@ const TAIL: u64 = 64 * 1024; const MAGIC: &[u8; 4] = b"PAR1"; +/// The magic of a file whose footer is encrypted (Parquet modular encryption). +const ENCRYPTED_MAGIC: &[u8; 4] = b"PARE"; + pub struct Footer { /// The thrift `FileMetaData`. bytes: Bytes, @@ -35,6 +39,10 @@ impl Footer { let tail = source.get_bytes(size - read, read as usize)?; let trailer = &tail[tail.len() - 8..]; + if &trailer[4..] == ENCRYPTED_MAGIC { + return Err(Error::Encrypted); + } + if &trailer[4..] != MAGIC { return Err(corrupt("Invalid Parquet file. Corrupt footer".to_string())); } @@ -58,6 +66,7 @@ impl Footer { bytes.freeze() }; let meta = ParquetMetaDataReader::decode_metadata(&bytes)?; + refuse_encrypted(&bytes)?; Ok(Self { bytes, @@ -176,4 +185,16 @@ mod tests { Err(Error::NotParquet(_)) )); } + + #[test] + fn a_file_with_an_encrypted_footer_is_refused() { + let mut data = file(10); + let end = data.len(); + data[end - 1] = b'E'; + + assert!(matches!( + Footer::read(&Source(Arc::new(Counting::new(data)))), + Err(Error::Encrypted) + )); + } } diff --git a/src/extension/arrow-ext/src/parquet/lib_calls.rs b/src/extension/arrow-ext/src/parquet/lib_calls.rs index 1cd3227ecd..da1034f92b 100644 --- a/src/extension/arrow-ext/src/parquet/lib_calls.rs +++ b/src/extension/arrow-ext/src/parquet/lib_calls.rs @@ -11,6 +11,8 @@ pub const SCHEMA_FROM_THRIFT: &str = "Flow\\Parquet\\ParquetFile\\Schema::fromTh pub const SCHEMA_TO_EXTENSION: &str = "Flow\\Parquet\\Engine\\Arrow\\SchemaConverter::toExtension"; pub const OPTIONS_TO_EXTENSION: &str = "Flow\\Parquet\\Engine\\Arrow\\OptionsConverter::toExtension"; pub const OPTIONS_TO_ENGINE: &str = "Flow\\Parquet\\Engine\\Arrow\\OptionsConverter::toEngine"; +pub const READ_PLAN_TO_EXTENSION: &str = "Flow\\Parquet\\Engine\\Arrow\\ReadPlanConverter::toExtension"; +pub const ROW_GROUP_CHUNKS_FROM_THRIFT: &str = "Flow\\Parquet\\Engine\\Arrow\\RustRowGroupChunks::fromThrift"; /// `Class::method(...$args)`; an exception the method throws surfaces as itself. pub fn call_static(name: &str, args: &mut [Zval]) -> PhpResult { diff --git a/src/extension/arrow-ext/src/parquet/library.rs b/src/extension/arrow-ext/src/parquet/library.rs index b89418385f..facb8caa35 100644 --- a/src/extension/arrow-ext/src/parquet/library.rs +++ b/src/extension/arrow-ext/src/parquet/library.rs @@ -1,8 +1,9 @@ -//! `Flow\Parquet\Engine\{RustParquetFileReader, RustParquetFileWriter}` and `Flow\Arrow\Parquet\RustColumnsReader`: the +//! `Flow\Parquet\Engine\{RustParquetFileReader, RustParquetFooter, RustParquetFileWriter}` and `Flow\Arrow\Parquet\RustColumnsReader`: the //! `flow-php/parquet` lib's values over the core - //! one footer read per file, columns as lists of the values `PhpParquetEngine` returns, rows written as it accepts //! them, and struct batches written as another extension exports them. +use std::collections::HashMap; use std::sync::Arc; use arrow_array::{new_null_array, Array, ArrayRef}; @@ -10,25 +11,29 @@ use ext_php_rs::convert::IntoZval; use ext_php_rs::error::Result as ZvalResult; use ext_php_rs::flags::{ClassFlags, DataType}; use ext_php_rs::prelude::*; -use ext_php_rs::types::{ZendHashTable, Zval}; +use ext_php_rs::types::{ZendClassObject, ZendHashTable, Zval}; use ext_php_rs::zend::ce; use parquet::basic::Type as PhysicalType; use parquet::file::metadata::ParquetMetaData; -use parquet::file::reader::Length; -use crate::interfaces::{parquet_file_reader_ce, parquet_file_writer_ce}; +use crate::interfaces::{footer_ce, parquet_file_reader_ce, parquet_file_writer_ce}; use crate::parquet::batch; use crate::parquet::cells::{Checked, Classes, ReadCell, Refusal, Scalar, Times, Validated, WriteColumn}; +use crate::parquet::chunks::chunk_spans; use crate::parquet::error::Error; +use crate::parquet::filter::Raw; use crate::parquet::footer::Footer; -use crate::parquet::lib_calls::{call_static, METADATA_FROM_THRIFT, SCHEMA_FROM_THRIFT}; +use crate::parquet::lib_calls::{ + call_static, METADATA_FROM_THRIFT, READ_PLAN_TO_EXTENSION, ROW_GROUP_CHUNKS_FROM_THRIFT, SCHEMA_FROM_THRIFT, +}; use crate::parquet::options; use crate::parquet::read::{ParquetReader, ReadPlan}; use crate::parquet::sink::PhpSink; use crate::parquet::source::{PhpSource, PhpStream}; use crate::parquet::write::ParquetWriter; use crate::php::{ - call_method, expect_object, for_each_value, ht_for_each, ht_get, ht_insert, read_property, IterableArg, + call_method, expect_object, for_each_value, ht_for_each, ht_get, ht_insert, ht_insert_long, read_property, + zval_array, zval_long, zval_str, IterableArg, }; use crate::render::{exception, parquet_exception, Side}; use crate::thrift; @@ -71,17 +76,116 @@ impl IntoZval for SchemaObject { } } -pub fn rows(value: Option, name: &str) -> PhpResult> { - value - .map(|value| { - u64::try_from(value).map_err(|_| { - exception( - INVALID_ARGUMENT, - format!("arrow Parquet {name} must be greater or equal to 0"), - ) - }) - }) - .transpose() +/// `ParquetFileReader::rowGroupChunks()`'s return, a `Flow\Parquet\Engine\Arrow\RustRowGroupChunks` typed as the +/// interface: a covariant userland return type cannot be checked when the class registers. +pub struct RowGroupChunksObject(Zval); + +impl IntoZval for RowGroupChunksObject { + const TYPE: DataType = DataType::object("Flow\\Parquet\\ParquetFile\\RowGroupChunks"); + const NULLABLE: bool = false; + + fn set_zval(self, zv: &mut Zval, _persistent: bool) -> ZvalResult<()> { + *zv = self.0; + + Ok(()) + } +} + +/// `$plan` (a `Flow\Parquet\Reader\ReadPlan`) through `ReadPlanConverter::toExtension()`, checked against `meta`. +pub fn read_plan(meta: &ParquetMetaData, plan: &Zval, stream: &PhpStream) -> PhpResult { + let converted = call_static(READ_PLAN_TO_EXTENSION, &mut [plan.shallow_clone()])?; + + converted_plan(meta, &converted).map_err(|error| parquet_exception(error, stream, Side::Read)) +} + +fn converted_plan(meta: &ParquetMetaData, converted: &Zval) -> Result { + let malformed = |what: &str| Error::Plan(format!("arrow received a malformed read plan: {what}")); + let converted = converted.array().ok_or_else(|| malformed("not an array"))?; + let selectors = ht_get(converted, b"selectors") + .and_then(Zval::array) + .ok_or_else(|| malformed("no 'selectors' list"))?; + let mut planned = Vec::with_capacity(selectors.len()); + + ht_for_each(selectors, |_, _, selector| { + let selector = selector + .array() + .ok_or_else(|| malformed("a selector is not an array"))?; + let count = selector + .get_index(0) + .and_then(Zval::long) + .and_then(|count| usize::try_from(count).ok()) + .ok_or_else(|| malformed("a selector's row count is not an int of 0 or more"))?; + let skip = selector + .get_index(1) + .and_then(Zval::bool) + .ok_or_else(|| malformed("a selector's skip is not a bool"))?; + + planned.push((count, skip)); + + Ok::<(), Error>(()) + })?; + + let limit = match ht_get(converted, b"limit") { + Some(limit) if limit.is_null() => None, + Some(limit) => Some( + limit + .long() + .and_then(|limit| usize::try_from(limit).ok()) + .ok_or_else(|| malformed("'limit' is not an int of 0 or more"))?, + ), + None => return Err(malformed("no 'limit'")), + }; + let filter = ht_get(converted, b"filter").map(raw).transpose()?; + + ReadPlan::from_selection(meta, planned, filter, limit) +} + +/// A converter array as data: a map keyed by its keys' bytes when a key is a string, else a list of its values. +fn raw(value: &Zval) -> Result { + if value.is_null() { + return Ok(Raw::Null); + } + + if let Some(value) = value.bool() { + return Ok(Raw::Bool(value)); + } + + if let Some(value) = value.long() { + return Ok(Raw::Int(value)); + } + + if let Some(value) = value.double() { + return Ok(Raw::Float(value)); + } + + if let Some(value) = value.zend_str() { + return Ok(Raw::Bytes(value.as_bytes().to_vec())); + } + + let Some(table) = value.array() else { + return Err(Error::Plan(format!( + "arrow received a malformed row filter: a {} is no row filter value", + value.get_type() + ))); + }; + let mut entries = Vec::new(); + let mut keyed = false; + + ht_for_each(table, |key, index, value| { + keyed |= key.is_some(); + entries.push(( + key.map_or_else(|| index.to_string().into_bytes(), |key| key.as_bytes().to_vec()), + raw(value)?, + )); + + Ok::<(), Error>(()) + })?; + + Ok(if keyed { + Raw::Map(entries) + } else { + Raw::List(entries.into_iter().map(|(_, value)| value).collect()) + }) } pub fn batch_size(value: i64) -> PhpResult { @@ -93,6 +197,21 @@ pub fn batch_size(value: i64) -> PhpResult { }) } +/// `ParquetFileReader::footer()`'s return, typed `Flow\Parquet\Engine\RustParquetFooter`: the class registers before +/// the reader, so the covariant return type is checked. +pub struct FooterObject(Zval); + +impl IntoZval for FooterObject { + const TYPE: DataType = DataType::object("Flow\\Parquet\\Engine\\RustParquetFooter"); + const NULLABLE: bool = false; + + fn set_zval(self, zv: &mut Zval, _persistent: bool) -> ZvalResult<()> { + *zv = self.0; + + Ok(()) + } +} + #[php_class] #[php( name = "Flow\\Parquet\\Engine\\RustParquetFileReader", @@ -101,12 +220,11 @@ pub fn batch_size(value: i64) -> PhpResult { )] pub struct RustParquetFileReader { stream: Arc, - source: PhpSource, - footer: Footer, + footer: Arc