diff --git a/Justfile b/Justfile
index ef884f47a1..2d4e741659 100644
--- a/Justfile
+++ b/Justfile
@@ -138,9 +138,7 @@ docs-api:
# Regenerate Parquet Thrift PHP classes.
gen-thrift: && fix
- grep -q 'namespace php Flow.Parquet.ThriftModel' src/lib/parquet/src/Flow/Parquet/Resources/Thrift/parquet.thrift || { echo "Flow php namespace not found in thrift definition!"; exit 1; }
- rm src/lib/parquet/src/Flow/Parquet/ThriftModel/*.php
- thrift --gen php --out src/lib/parquet/src src/lib/parquet/src/Flow/Parquet/Resources/Thrift/parquet.thrift
+ php bin/thrift.php parquet
# Regenerate PostgreSQL pg_query Protobuf PHP classes.
gen-protobuf-pg: && fix
diff --git a/bin/docs.php b/bin/docs.php
index 833a0db5f7..0ac9d5ff8e 100755
--- a/bin/docs.php
+++ b/bin/docs.php
@@ -73,6 +73,7 @@ public function execute(InputInterface $input, OutputInterface $output): int
__DIR__ . '/../src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/functions.php',
__DIR__ . '/../src/lib/filesystem/src/Flow/Filesystem/DSL/functions.php',
__DIR__ . '/../src/lib/types/src/Flow/Types/DSL/functions.php',
+ __DIR__ . '/../src/lib/parquet/src/Flow/Parquet/DSL/functions.php',
__DIR__ . '/../src/lib/postgresql/src/Flow/PostgreSql/DSL/schema.php',
__DIR__ . '/../src/lib/postgresql/src/Flow/PostgreSql/DSL/query.php',
__DIR__ . '/../src/lib/postgresql/src/Flow/PostgreSql/DSL/condition.php',
diff --git a/bin/thrift.php b/bin/thrift.php
new file mode 100755
index 0000000000..241dff0b5b
--- /dev/null
+++ b/bin/thrift.php
@@ -0,0 +1,100 @@
+#!/usr/bin/env php
+addCommand(new class extends Command {
+ public function configure(): void
+ {
+ $this
+ ->setName('parquet')
+ ->setDescription('Regenerates the Parquet ThriftModel classes from parquet.thrift.')
+ ->addOption(
+ 'thrift',
+ null,
+ InputOption::VALUE_REQUIRED,
+ 'Thrift compiler binary, found in PATH when omitted',
+ );
+ }
+
+ public function execute(InputInterface $input, OutputInterface $output): int
+ {
+ $source = __DIR__ . '/../src/lib/parquet/src';
+ $idl = $source . '/Flow/Parquet/Resources/Thrift/parquet.thrift';
+ $models = $source . '/Flow/Parquet/ThriftModel';
+
+ $thrift = $input->getOption('thrift') === null
+ ? (new ExecutableFinder())->find('thrift')
+ : type_string()->assert($input->getOption('thrift'));
+
+ if (!is_string($thrift) || !is_executable($thrift)) {
+ $output->writeln('Thrift compiler not found, pass an executable with --thrift');
+
+ return Command::FAILURE;
+ }
+
+ if (!str_contains((string) file_get_contents($idl), 'namespace php Flow.Parquet.ThriftModel')) {
+ $output->writeln('Flow php namespace not found in thrift definition!');
+
+ return Command::FAILURE;
+ }
+
+ foreach (glob($models . '/*.php') ?: [] as $file) {
+ unlink($file);
+ }
+
+ (new Process([$thrift, '--gen', 'php', '--out', $source, $idl]))->mustRun(
+ static fn(string $type, string $buffer) => $output->write($buffer),
+ );
+
+ // the Thrift compiler types every field non-null; a field the IDL does not require is null when a file leaves it out
+ foreach (glob($models . '/*.php') ?: [] as $file) {
+ $class = 'Flow\\Parquet\\ThriftModel\\' . basename($file, '.php');
+
+ if (!class_exists($class) || !property_exists($class, '_TSPEC')) {
+ continue;
+ }
+
+ /** @var array $spec */
+ $spec = (new ReflectionProperty($class, '_TSPEC'))->getValue();
+ $model = (string) file_get_contents($file);
+ $rewritten = $model;
+
+ foreach ($spec as $field) {
+ if ($field['isRequired']) {
+ continue;
+ }
+
+ $rewritten = (string) preg_replace(
+ '/(\* @var )(?!null\|)([^\n]+)(\n\s*\*\/\n\s*public \$'
+ . preg_quote($field['var'], '/')
+ . ' = null;)/',
+ '$1null|$2$3',
+ $rewritten,
+ );
+ }
+
+ if ($rewritten !== $model) {
+ file_put_contents($file, $rewritten);
+ }
+ }
+
+ return Command::SUCCESS;
+ }
+});
+
+$application->run();
diff --git a/composer.json b/composer.json
index 48de3ffb0d..1bca0798ee 100644
--- a/composer.json
+++ b/composer.json
@@ -260,6 +260,7 @@
"src/lib/filesystem/src/Flow/Filesystem/DSL/functions.php",
"src/lib/parquet/src/Flow/Parquet/functions.php",
"src/lib/parquet/src/Flow/Parquet/Binary/functions.php",
+ "src/lib/parquet/src/Flow/Parquet/DSL/functions.php",
"src/lib/parquet/src/stubs.php",
"src/lib/postgresql/src/Flow/PostgreSql/DSL/client.php",
"src/lib/postgresql/src/Flow/PostgreSql/DSL/condition.php",
diff --git a/documentation/components/adapters/parquet.md b/documentation/components/adapters/parquet.md
index 862480ec79..06d5153c8b 100644
--- a/documentation/components/adapters/parquet.md
+++ b/documentation/components/adapters/parquet.md
@@ -73,3 +73,72 @@ data_frame()
->write(to_output())
->run();
```
+
+A `UINT_64` column reads as `numeric_string` (`'18446744073709551615'` is past `PHP_INT_MAX`). A `numeric_string`
+column is written as a parquet `STRING`; write `UINT_64` through the lib with an explicit schema:
+
+```php
+write(
+ __DIR__ . '/ids.parquet',
+ Schema::with(new FlatColumn('id', PhysicalType::INT64, logicalType: LogicalType::integer(64, false))),
+ [['id' => 7], ['id' => '18446744073709551615']],
+);
+```
+
+## Filter pushdown
+
+A `filter()` directly over `from_parquet()` moves into the reader, which returns only matching rows: row groups whose
+statistics prove that no row matches are skipped, and a `limit()` after the filter stops the read at that many
+matching rows:
+
+```php
+read(from_parquet(__DIR__ . '/orders/*.parquet'))
+ ->filter(ref('id')->greaterThan(lit(1000))->and(ref('status')->equals(lit('paid'))))
+ ->limit(10)
+ ->write(to_output())
+ ->run();
+```
+
+Each part of an `and()` is pushed on its own when it compares a file column with a literal it can compare exactly.
+
+Pushed:
+
+- `equals`, `notEquals`, `same`, `notSame`, `lessThan`, `lessThanEqual`, `greaterThan`, `greaterThanEqual`,
+ `between`, `isIn` (literal list), `startsWith`, `isNull`, `isNotNull`, and `not()`/`and()`/`or()` of those
+- an int literal on a float column up to 2^53, an integral float on an int column below 2^53
+- a `date` literal on a `datetime` column and the reverse
+- a digit string (`lit('1000')`) on a `numeric_string` (`UINT_64`) column
+
+Evaluated on the rows read:
+
+- other functions, two columns compared, a literal of another type
+- `json` columns (except `isNull`/`isNotNull`), ordering on `uuid`
+- a column the file stores as another type than the read
+
+Partition columns list fewer files instead ([partitioning](/documentation/components/core/partitioning.md)).
+
+`explain()` shows the filter and the limit under `Read`, not what the reader skipped:
+
+```
+#3 Result
+│ Rows this plan hands out: to the trigger, or to the node reading it
+└─ #2 Limit
+ │ Limit: 10
+ └─ #1 Read
+ Extractor: ParquetExtractor
+ Source: file:///orders/*.parquet
+ Limit: 10
+ Filter: All
+```
diff --git a/documentation/components/core/core.md b/documentation/components/core/core.md
index 02254f9456..01d6a5f377 100644
--- a/documentation/components/core/core.md
+++ b/documentation/components/core/core.md
@@ -67,7 +67,7 @@ Outputs
├─ #4 Result
│ │ Rows this plan hands out: to the trigger, or to the node reading it
│ └─ #3 Select
-│ │ Columns: id, email
+│ │ Keeps only: id, email
│ └─ #2 Filter
│ │ Condition: IsNotNull
│ └─ #1 Read
@@ -129,7 +129,7 @@ echo $dataFrame->explain()->toString(format: Format::flow);
└─ #2 Filter
│ Condition: IsNotNull
├─ #3 Select
- │ │ Columns: id, email
+ │ │ Keeps only: id, email
│ └─ #4 Result
│ Rows this plan hands out: to the trigger, or to the node reading it
└─ #5 Write
@@ -154,7 +154,7 @@ echo $dataFrame->explain()->toString(format: Format::boxes);
┌─────────────┴─────────────┐┌─────────────┴─────────────┐
│ #3 Select ││ #2 Filter │
│ ──────────────────── ││ (shared) │
-│ Columns: id, email ││ │
+│ Keeps only: id, email ││ │
└─────────────┬─────────────┘└───────────────────────────┘
┌─────────────┴─────────────┐
│ #2 Filter │
diff --git a/documentation/components/core/partitioning.md b/documentation/components/core/partitioning.md
index 186cc68419..feb42eb533 100644
--- a/documentation/components/core/partitioning.md
+++ b/documentation/components/core/partitioning.md
@@ -268,8 +268,10 @@ it from the file schema (Parquet, Floe), so `filter(ref('date')->between(lit($fr
### Partition Pruning
-The optimizer pushes a `filter()` that reads only partition columns into the source, so whole directories
-are skipped; a filter on a body column still reads everything and then discards. A partition column renamed
+The optimizer moves a `filter()` into the file source, which returns only matching rows. A filter that reads only
+partition columns skips whole directories; a filter on a body column reads every file and drops the rows it does not
+match (Parquet skips row groups, see [filter pushdown](/documentation/components/adapters/parquet.md#filter-pushdown)).
+A `limit()` after the filter reaches the source with it ([filter pushdown](/documentation/components/adapters/parquet.md#filter-pushdown)). A partition column renamed
with `rename()` or copied with `withEntry('day', ref('date'))` is still pruned; any other redefinition of it
stops the push. A filter whose answer can change between two evaluations - `call()`, `now()`, `uuid()`,
`to_date_time()` with a format that leaves fields out (`'Y-m-d'`) - still filters rows but prunes nothing.
@@ -283,6 +285,7 @@ use function Flow\ETL\DSL\{data_frame, lit, ref, to_output};
data_frame()
->read(from_csv(__DIR__ . '/output/date=*/department=*/*.csv'))
->filter(ref('date')->greaterThanEqual(lit('2024-01-01')))
+ ->limit(10)
->write(to_output())
->run();
```
diff --git a/documentation/components/extensions/arrow-ext.md b/documentation/components/extensions/arrow-ext.md
index f79d40880f..1c2dcdc35d 100644
--- a/documentation/components/extensions/arrow-ext.md
+++ b/documentation/components/extensions/arrow-ext.md
@@ -92,18 +92,23 @@ use Flow\Parquet\Engine\{RustParquetFileReader, RustParquetFileWriter};
use Flow\Parquet\Engine\Arrow\{OptionsConverter, SchemaConverter};
use Flow\Parquet\Options;
use Flow\Parquet\ParquetFile\Compressions;
+use Flow\Parquet\ParquetFile\Index\StreamIndexes;
+use Flow\Parquet\Reader\ReadPlan;
use function Flow\Filesystem\DSL\{native_local_filesystem, path};
$filesystem = native_local_filesystem();
$file = new RustParquetFileReader($filesystem->readFrom(path('data.parquet'))); // Flow\Parquet\ParquetFileReader, one footer read
-$file->rowsNumber();
-$file->schema(); // Flow\Parquet\ParquetFile\Schema
-$file->metadata(); // Flow\Parquet\ParquetFile\Metadata
-$file->thrift(); // Flow\Parquet\ThriftModel\FileMetaData
-
-foreach ($file->readColumns(['id', 'name'], 1_000, null, null) as $chunk) { // RustColumnsReader, an Iterator that streams once
+$footer = $file->footer(); // Flow\Parquet\Engine\RustParquetFooter, readable after close()
+$footer->rowsNumber();
+$footer->schema(); // Flow\Parquet\ParquetFile\Schema
+$footer->metadata(); // Flow\Parquet\ParquetFile\Metadata
+$footer->thrift(); // Flow\Parquet\ThriftModel\FileMetaData
+$indexes = new StreamIndexes($footer, $filesystem->readFrom(path('data.parquet'))); // page indexes, read on demand
+$plan = ReadPlan::window($footer->rowsNumber(), $indexes, offset: 10, limit: 100); // rows 10-109
+
+foreach ($file->readColumns(['id', 'name'], 1_000, $plan) as $chunk) { // RustColumnsReader, an Iterator that streams once
// ['id' => [1, 2, ...], 'name' => ['a', 'b', ...]]
}
@@ -118,7 +123,7 @@ $writer->writeBatch([['id' => 1, 'name' => 'a']]); // an array or any Traversa
$writer->writeColumns(['id' => [2, 3], 'name' => ['b', null]]);
$writer->close(); // footer, then the stream closed
-$batches = new RustBatchReader($file, ['id', 'name'], 1_000, null, null); // batch size, offset, limit
+$batches = new RustBatchReader($file, ['id', 'name'], 1_000, $plan); // batch size, rows to read
$batches->schema(); // Flow\Arrow\RustArrowSchema
$batches->next(); // Flow\Arrow\RustParquetBatch, null after the last
```
diff --git a/documentation/components/libs/parquet.md b/documentation/components/libs/parquet.md
index 37b52280a7..49b89484c2 100644
--- a/documentation/components/libs/parquet.md
+++ b/documentation/components/libs/parquet.md
@@ -148,10 +148,10 @@ invalid file throws `InvalidArgumentException` `Given file is not valid Parquet
There are several things we can read from parquet file:
-- `ParquetFile::values(array $columns = [], ?int $limit = null, ?int $offset = null) : \Generator`
-- `ParquetFile::columns(int $batchSize, array $columns = [], ?int $limit = null, ?int $offset = null) : \Generator`
+- `ParquetFile::values(Query $query = new Query()) : \Generator` - rows, one by one
+- `ParquetFile::columns(Query $query = new Query(), int $batchSize = 1024) : \Generator` - columns, in batches
- `ParquetFile::metadata() : Metadata`
-- `ParquetFile::schema() : Schema` - shortcut for `ParquetFile::metadata()->schema()`
+- `ParquetFile::schema() : Schema` - the file schema, without building row-group metadata
- `ParquetFile::close() : void` - closes the reader and its stream
### Reading the whole file:
@@ -171,12 +171,12 @@ foreach ($file->values() as $row) {
```php
use Flow\Parquet\Reader;
+use function Flow\Parquet\DSL\parquet_query;
-$reader = new Reader();
+$file = (new Reader())->read('path/to/file.parquet');
-$file = $reader->read('path/to/file.parquet');
-foreach ($file->values(["column_1", "column_2"]) as $row) {
- // do something with $row
+foreach ($file->values(parquet_query()->select('column_1', 'column_2')) as $row) {
+ // ['column_1' => ..., 'column_2' => ...]
}
```
@@ -187,29 +187,88 @@ foreach ($file->values(["column_1", "column_2"]) as $row) {
> To achieve the best results, we will need to play a bit with Writer options (covered later).
```php
-use Flow\Parquet\Reader;
+foreach ($file->values(parquet_query()->select('column_1')->offset(1000)->limit(100)) as $row) {
+ // rows 1000-1099
+}
+```
-$reader = new Reader();
+### Reading columns in batches
-$file = $reader->read('path/to/file.parquet');
-foreach ($file->values(["column_1", "column_2"], limit: 100, offset: 1000) as $row) {
- // do something with $row
+```php
+foreach ($file->columns(parquet_query()->select('column_1', 'column_2'), batchSize: 1000) as $chunk) {
+ // ['column_1' => [...up to 1000 values], 'column_2' => [...]]
}
```
-### Reading columns in batches
+### Filtering reads
```php
use Flow\Parquet\Reader;
+use function Flow\Parquet\DSL\{parquet_column, parquet_query};
$file = (new Reader())->read('path/to/file.parquet');
-foreach ($file->columns(batchSize: 1000, columns: ["column_1", "column_2"]) as $chunk) {
- // $chunk = ['column_1' => [...up to 1000 values], 'column_2' => [...]]
+$query = parquet_query()
+ ->select('id', 'country')
+ ->where(
+ parquet_column('country')->equals('PL')
+ ->and(parquet_column('user.age')->lessThan(18)->negate())
+ ->and(parquet_column('status')->in(['new', 'paid'])->or(parquet_column('note')->isNull())),
+ )
+ ->offset(20)
+ ->limit(10);
+
+foreach ($file->values($query) as $row) {
+ // only matching rows: the 21st to the 30th match
}
```
-`limit` and `offset` work as in `values()`.
+The rows returned are exactly the matching ones; offset and limit count matching rows. Row groups and pages whose
+statistics (column chunk statistics, the [page index](#page-index)) prove no row matches are never read - both engines
+skip the same ones and read a kept page's bytes alone. A second `where()` narrows the first.
+
+| Method on `parquet_column($path)` | Filter |
+|------------------------------------------------------------------------------|--------------------------|
+| `equals`, `lessThan`, `lessThanOrEqual`, `greaterThan`, `greaterThanOrEqual` | `Comparison` |
+| `notEquals` | `Not` of `equals` |
+| `in(array)`, `notIn(array)` | `In`, `Not` of `In` |
+| `isNull()`, `isNotNull()` | `IsNull`, its `Not` |
+| `startsWith(string)` | `StartsWith` |
+
+- `->and()`, `->or()`, `->negate()` on any filter build `All`, `Any`, `Not`; `new All(...$filters)` and
+ `new Any(...$filters)` combine a list (namespace `Flow\Parquet\Filter`).
+- `$path` names a leaf through structs (`'user.address.city'`); a column the file lacks reads as null. A leaf inside a
+ list or a map is refused (`InvalidArgumentException`): a row holds no single value there.
+- Semantics are SQL's: a comparison on null is null, `NOT` null is null, only true keeps a row. Floats follow IEEE 754
+ (`NaN` matches nothing, `-0.0` equals `0.0`), strings compare in byte order (`'10' < '9'`), `UINT_64` values
+ numerically, date-times by instant. A literal of another kind than the column's values (a string on an `INT64`
+ column) matches nothing, under `->negate()` too.
+- Literals: scalars, `DateTimeInterface`, `Stringable` (uuids); `null` (use `isNull()`) and `NaN` are refused.
+- Files written without `column_orders` (older flow-php versions) are never pruned, only filtered. The statistics a file
+ holds are set by the writer options in [Statistics](#statistics).
+
+#### Reading with a ReadPlan
+
+```php
+use Flow\Parquet\Filter\Binding\BoundFilter;
+use Flow\Parquet\Reader\ReadPlanner;
+use function Flow\Parquet\DSL\parquet_column;
+
+$filter = parquet_column('id')->greaterThan(100);
+$plan = (new ReadPlanner())->plan(
+ $file->reader()->footer(),
+ $file->indexes(),
+ BoundFilter::bind($filter, $file->schema(), $file->options()),
+ limit: 10, // rows kept by $plan->rowFilter, refused unless it is the whole filter
+);
+
+foreach ($file->reader()->readColumns(['id'], 1024, $plan) as $batch) {
+ // only rows matching $plan->rowFilter
+}
+```
+
+`ReadPlan::$rowFilter` holds the conjuncts the file compares exactly; rows come back matching them, a superset of the
+query's matches when they are not the whole filter.
## Writing Parquet Files
@@ -331,15 +390,24 @@ $writer->close();
### Writer Options
-- `DICTIONARY_PAGE_MIN_CARDINALITY_RATION` - default '0.4' - minimum ratio of unique values to total values for a column
- to have dictionary encoding.
-- `DICTIONARY_PAGE_SIZE` - default: `1Mb` - maximum size of dictionary page.
+- `COLUMN_INDEX_TRUNCATE_LENGTH` - default: `64` - bytes a string/binary page's min/max in the column index are cut to.
+- `DICTIONARY_PAGE_SIZE` - default: `1Mb` - maximum size of dictionary page; the pages of a column chunk after it is
+ reached are written `PLAIN`.
- `GZIP_COMPRESSION_LEVEL` - default: `9` - compression level for GZIP compression (applied only when GZIP compression
is enabled).
+- `COLUMNS_BLOOM_FILTERS` - default: `null` - `BloomFilterOptions` by column flat path: those columns get a bloom filter
+ per row group ([Bloom filters](#bloom-filters)).
+- `PAGE_HEADER_STATISTICS` - default: `false` - data page headers carry the page's statistics (at `StatisticsLevel::PAGE`).
+- `PAGE_MAXIMUM_ROWS_COUNT` - default: `1000` - rows per data page: the PHP writer closes a page at exactly that many;
+ the arrow writer treats it as a target, checked between write batches, so its pages can hold more. Smaller pages let
+ the page index skip more of a filtered read.
- `PAGE_SIZE_BYTES` - default: `128Kb` - maximum size of data page.
- `ROW_GROUP_SIZE_BYTES` - default: `32Mb` - maximum size of row group.
- `ROW_GROUP_SIZE_CHECK_INTERVAL` default: `1000` - number of rows to write before checking if row group size limit is
reached.
+- `STATISTICS_LEVEL` - default: `StatisticsLevel::PAGE` - `NONE`, `CHUNK` (column chunk statistics) or `PAGE` (also a
+ column index); the offset index is written at every level.
+- `STATISTICS_TRUNCATE_LENGTH` - default: `64` - bytes a string/binary column chunk's min/max statistics are cut to.
- `VALIDATE_DATA` - default: `true` - if set to `true` then writer will validate data against schema.
- `WRITER_VERSION` - default `1` - tells writer which version of parquet format should be used.
@@ -369,6 +437,89 @@ more frequently.
Unfortunately, there is no one size fits all solution here.
You will need to play a bit with those values to find the best one for your use case.
+## Statistics
+
+Both engines write the column chunk statistics arrow-rs writes: `min_value`/`max_value` with exactness flags,
+`nan_count` for floats and a column order per leaf.
+
+```php
+set(Option::STATISTICS_TRUNCATE_LENGTH, 16)))
+ ->write(__DIR__ . '/orders.parquet', $schema, $rows);
+
+$metadata = (new Reader())->read(__DIR__ . '/orders.parquet')->metadata();
+$column = $metadata->schema()->getFlat('customer');
+
+foreach ($metadata->rowGroups()->all() as $rowGroup) {
+ $statistics = $rowGroup->getColumnChunk($column)->statistics();
+
+ $statistics?->minValue($column); // physical value: bytes for strings, unscaled int for decimal(10, 2)
+ $statistics?->maxValue($column);
+ $statistics?->isMaxValueExact(); // false when the max was truncated and incremented
+ $statistics?->nullCount();
+ $statistics?->nanCount(); // FLOAT, DOUBLE and FLOAT16 only
+}
+
+$metadata->columnOrder($column); // ColumnOrder::TypeDefined; ColumnOrder::Absent in files written before it
+```
+
+| Physical type | Order of `min_value`/`max_value` |
+|----------------------------------|----------------------------------------------------------------------------------------|
+| BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY | unsigned bytes (`'10' < '9'`) |
+| DECIMAL | signed (two's complement bytes, or the unscaled INT32/INT64) |
+| INT32/INT64 `UINT_*` | unsigned |
+| FLOAT, DOUBLE, FLOAT16 | IEEE 754 total order (`-0.0 < 0.0`); NaN counted, a bound only when every value is NaN |
+| INT96, INTERVAL | none: no `min_value`/`max_value` is written |
+
+Strings and binaries longer than `STATISTICS_TRUNCATE_LENGTH` keep a prefix as the min and an incremented prefix as the
+max. Decimals and FLOAT16 are never truncated.
+
+### Page index
+
+Both writers write a column index (each page's min, max, null and NaN counts) and an offset index (where each page
+starts, and its first row) by default. Pages start on row boundaries, lists included.
+
+```php
+set(Option::STATISTICS_LEVEL, StatisticsLevel::CHUNK)))
+ ->write(__DIR__ . '/orders.parquet', $schema, $rows); // no column index; NONE also drops chunk statistics
+
+$file = (new Reader())->read(__DIR__ . '/orders.parquet');
+$column = $file->schema()->getFlat('customer');
+
+$file->indexes()->columnIndex(0, $column); // ?ColumnIndex of row group 0: nullPages, minValues, maxValues, ...
+$file->indexes()->offsetIndex(0, $column); // ?OffsetIndex: pages[i]->offset, compressedPageSize, firstRowIndex
+```
+
+### Bloom filters
+
+A bloom filter per column chunk lets a read skip the row groups that cannot hold a value, where min/max cannot (an id
+inside every group's range). Only `equals()` and `in()` use them. Both writers write them after each row group, for the
+columns asked only.
+
+```php
+set(Option::COLUMNS_BLOOM_FILTERS, [
+ 'id' => new BloomFilterOptions(), // fpp 0.05, ndv 1 048 576 (the arrow-rs defaults)
+ 'user.email' => new BloomFilterOptions(0.01, 10_000),
+])))->write(__DIR__ . '/users.parquet', $schema, $rows);
+
+(new Reader())->read(__DIR__ . '/users.parquet')
+ ->values(parquet_query()->select('id')->where(parquet_column('id')->in([42, 1337])));
+```
+
## Compressions
> [!NOTE]
diff --git a/documentation/contributing/rust.md b/documentation/contributing/rust.md
index 79c6a7cf77..cdd6bfeac5 100644
--- a/documentation/contributing/rust.md
+++ b/documentation/contributing/rust.md
@@ -169,9 +169,10 @@ holds the same number:
| flow_php | `FLOW_PHP_ABI` | `Flow\ETL\FlowPhpExtension::ABI` |
| arrow | `FLOW_ARROW_ABI` | `Flow\Parquet\Engine\ArrowExtension::ABI` |
-Bump both, in one change, whenever a class or interface the extension registers changes. Every `Adaptive*` pick calls
-`available()`: an extension that is loaded with another ABI, or with none, is refused with a `RuntimeException`
-naming the extension and its version. phpt `099` (flow_php) and `054` (arrow) pin that the two numbers are equal.
+Bump both, in one change, whenever the contract changes: a class or interface the extension registers, or the
+schema, options or values that cross it. Every `Adaptive*` pick calls `available()`: an extension that is loaded with
+another ABI, or with none, is refused with a `RuntimeException` naming the extension and its version. phpt `099`
+(flow_php) and `054` (arrow) pin that the two numbers are equal.
The PHP interfaces an extension also registers are guarded by `interface_exists(X::class, false)`, so an extension
that does not register one still gets the PHP declaration and fails at the ABI check, not with "Class not found".
diff --git a/documentation/upgrading.md b/documentation/upgrading.md
index 6a715ea207..ccbe728323 100644
--- a/documentation/upgrading.md
+++ b/documentation/upgrading.md
@@ -19,6 +19,117 @@ Upgrading from older versions: [0.35.x to 0.40.x](/documentation/upgrading-0.35-
| `ListType>`, `MapType>` | `ListType`, `MapType` |
| Mago `plugins = ["flow-php"]` infers `type_structure()` shapes | Built-in plugin no longer infers them - use `flow-php/mago-types-bridge` (`plugins = ["flow/types"]`) |
+### 2) `flow-php/parquet` - statistics are written and read as arrow-rs does; one physical schema for one logical schema on both engines
+
+| Before | After |
+|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| PHP writer string bounds by PHP `<` (`'9'` below `'10'`), whole strings, no exactness flags, no `nan_count`, no `column_orders`, deprecated `min`/`max` for every type | unsigned-byte order, truncated to `Option::STATISTICS_TRUNCATE_LENGTH` (64) bytes, exactness flags, `nan_count` for floats, `column_orders` per leaf, deprecated `min`/`max` only for signed sort orders |
+| statistics of files written by older flow versions read as written | `ColumnOrder::Absent`: no bounds are trusted, so such files are never pruned; a statistic of the wrong width (older INT64 bounds are 16 bytes) reads as `null` |
+| `StatisticsReader` read BYTE_ARRAY statistics with a length prefix, a FLBA decimal as a float, an INT64 decimal as the unscaled int | raw bytes for BYTE_ARRAY/FLBA (decimals as unscaled two's complement bytes, uuid as 16 bytes), the unscaled int for INT32/INT64 decimals |
+| `FlatColumn::decimal()` of precision up to 18 written as FIXED_LEN_BYTE_ARRAY | INT32 up to precision 9, INT64 up to 18, FIXED_LEN_BYTE_ARRAY above; older files still read |
+| the packer/unpacker encoded FLBA/BYTE_ARRAY decimals and UUIDs; `DecimalConverter` took INT-backed decimals only | `new DecimalConverter($physicalType, $precision, $scale, $byteLength)` and `UuidConverter` convert to the physical bytes; the packer/unpacker take them as-is |
+| `FlatColumn::int32()` / `int64()` wrote converted `INT_32` / `INT_64` | plain `INT32` / `INT64`, as arrow-rs |
+| a column with a logical type and no converted type written without one (DuckDB read a logical `UINT_64` as signed) | the converted type derived from the logical type, as arrow-rs; a `FlatColumn` whose converted and logical types disagree is refused, on read and write |
+| the arrow writer named map entries `m.entries.key` | `m.key_value.key` (LogicalTypes.md): `COLUMNS_*` options keyed by `…key_value.key` address the same leaf on both engines |
+| the arrow writer wrote `FlatColumn::enum()` as `STRING` | `ENUM` |
+| `Option::ARROW_BATCH_SIZE`, read by nothing | removed |
+| `Flow\Parquet\Dremel\Statistics\Comparator` | removed: `Flow\Parquet\Data\PhysicalOrder` |
+| `new StatisticsCounter($column)` | `new StatisticsCounter($column, new StatisticsTruncation($length))` |
+| `new UnsignedIntegerConverter($path, $bitWidth)` | `new UnsignedInt32Converter()`; UINT_64 is `UnsignedInt64Converter` |
+
+### 3) `flow-php/parquet`, `flow-php/etl-adapter-parquet` - `UINT_64` reads as a decimal string on both engines (reverses rows 84 and 85 of 0.44→0.45 for `UINT_64`)
+
+| Before | After |
+|-----------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------|
+| `UINT_64` read as `int`, refused above `PHP_INT_MAX` on both engines | read as the decimal string (`'18446744073709551615'`) on both engines; `UINT_8/16/32` stay `int` |
+| `from_parquet()` typed a `UINT_64` column `integer` | `numeric_string` |
+| writes accepted an int from 0 to `PHP_INT_MAX` into `UINT_64` | an int >= 0 or a canonical digit string up to `18446744073709551615` |
+| `to_parquet()` refused a `numeric_string` column (`NumericStringType is not supported`) | written as a parquet `STRING` |
+
+### 4) `flow-php/parquet` - the `arrow` extension must be rebuilt (arrow-rs 60, ABI 2)
+
+| Before | After |
+|---------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| arrow-ext built against arrow-rs 59, `FLOW_ARROW_ABI` 1 | arrow-rs 60, `FLOW_ARROW_ABI` / `ArrowExtension::ABI` 2 (covering every contract change below): an older build is refused by `ArrowExtension::available()` |
+
+### 5) `flow-php/parquet`, `flow-php/etl-adapter-parquet` - reads take a `Query` (or, below the file, a `ReadPlan`) instead of limit/offset, the PHP writer writes the page index, the reader exposes its `Footer`; the `arrow` and `flow_php` extensions must be rebuilt
+
+| Before | After |
+|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `ParquetFile::values(array $columns, ?int $limit, ?int $offset)` | `values(Query $query)`: `$file->values(parquet_query()->select(...$columns)->offset($offset)->limit($limit))` |
+| `ParquetFile::columns(int $batchSize, array $columns, ?int $limit, ?int $offset)` | `columns(Query $query, int $batchSize = 1024)` |
+| a limit of `0` refused (`Limit must be greater than 0`); a negative offset or limit refused by `columns()` | `limit(0)` reads no row; a negative offset or limit is refused by `Query` |
+| `ParquetFileReader::metadata()`, `schema()`, `rowsNumber()`, `totalByteSize()` | `ParquetFileReader::footer(): Footer` holds them (and `rowGroupChunks()`); a footer taken before `close()` stays readable |
+| `ParquetFileReader::readColumns(array $columns, int $batchSize, ?int $limit, ?int $offset)` | `readColumns(array $columns, int $batchSize, ReadPlan $plan)`, returning the rows `$plan->selection` selects that match `$plan->rowFilter`, at most `$plan->limit`; build the plan with `ReadPlanner::plan()` or `ReadPlan::window($rows, $file->indexes(), $offset, $limit)` |
+| `RustParquetFileReader::thrift()` | `RustParquetFileReader::footer()->thrift()` (`RustParquetFooter`) |
+| `ParquetOpenSource::batches(Schema $schema, int $batchSize, ?int $offset, ?int $limit, Backend $backend)` | `batches(Schema $schema, int $batchSize, ReadPlan $plan, Backend $backend)` |
+| `RustBatchReader` / `RustColumnsReader::__construct(..., int $batchSize, ?int $offset, ?int $limit)` | `__construct(..., int $batchSize, ReadPlan $plan)` |
+| `ColumnReader::read(Column, Metadata, SourceStream, ?int $limit, ?int $offset)` | `readRowGroup(Column, RowGroup, int $group, SourceStream, RowSelection, Indexes, array $decoded = [])`, one row group at a time; `$decoded` pages, keyed by their first row, are served instead of read |
+| `new ColumnReader(ColumnChunkReader $chunks, DremelAssembler $assembler)` | `new ColumnReader(new SelectedPages($chunks), $assembler)` |
+| `ColumnChunkReader::read(ColumnChunk, FlatColumn, SourceStream)` yielded a list of pages | pages keyed by their first row in the row group; `?OffsetIndex $offsetIndex` with `list $pages` reads only those pages of it, `?Dictionary $dictionary` replaces the dictionary page, the generator returns the dictionary |
+| `RowGroupContainer(string, RowGroup)` | `RowGroupContainer(string, RowGroup, list, list)` |
+| `new PlainFlatColumnChunkBuilder($column, $options, $compression)` (and the RLE_DICTIONARY, DELTA_BINARY_PACKED builders, `ColumnChunkBuilderFactory::createBuilder()`) | a 4th argument, `StatisticsOptions::fromOptions($options)` |
+| `ColumnChunk::dataPageOffset(): ?int` | `int`: a column chunk always records it |
+| the PHP writer wrote no column index or offset index | both, by default (`Option::STATISTICS_LEVEL`, `StatisticsLevel::PAGE`; `CHUNK` drops the column index, `NONE` all statistics) |
+| the PHP writer wrote page-header statistics on `DATA_PAGE_V2` | none, as arrow-rs: `Option::PAGE_HEADER_STATISTICS` (`true`) writes them, on v1 and v2 pages |
+| `Option::PAGE_MAXIMUM_ROWS_COUNT` (default `1000`) was read by nothing | the PHP writer closes a page at exactly that many rows; the arrow writer treats it as a target (arrow-rs `data_page_row_count_limit`, checked between write batches), so its pages can hold more; below 1 is refused (`InvalidArgumentException`) |
+| the arrow writer skipped a `COLUMNS_COMPRESSIONS` / `COLUMNS_ENCODINGS` entry whose column path is digits only (`'2024'`, an int array key) | applies it; a key that is not UTF-8 is refused (`InvalidArgumentException`) instead of crashing the process |
+| `Option::ROW_GROUP_SIZE_BYTES` below 1 was taken by both writers | refused (`InvalidArgumentException`) by both writers |
+| the arrow writer threw `Error` (a Rust panic) on `PLAIN_DICTIONARY` / `RLE_DICTIONARY` in `COLUMNS_ENCODINGS` | enables the column's dictionary: `RLE_DICTIONARY` data pages and a `PLAIN` dictionary page, as the PHP writer writes |
+| the arrow writer skipped a `COLUMNS_COMPRESSIONS` / `COLUMNS_ENCODINGS` value that is not a string | refused (`InvalidArgumentException`) |
+| files with an `RLE_DICTIONARY` column spanning more than one page, written by the PHP writer | were corrupt (later pages decoded to wrong values): rewrite them; one dictionary per chunk, `PLAIN` past `Option::DICTIONARY_PAGE_SIZE` |
+| `Option::DICTIONARY_PAGE_MIN_CARDINALITY_RATION` | removed: it was never read; the dictionary falls back to `PLAIN` on `Option::DICTIONARY_PAGE_SIZE` only, as arrow-rs |
+| `Options::set()` / `get()` took scalars and arrays | also a `UnitEnum`; `Options::getEnum(Option $option, string $class)` |
+| no column index truncation option | `Option::COLUMN_INDEX_TRUNCATE_LENGTH` (64): BYTE_ARRAY/FIXED_LEN_BYTE_ARRAY page bounds in the column index |
+| `DataPageHeaderV2::statistics(Options $options)` | `statistics()`, as `DataPageHeader::statistics()` |
+| `ParquetFile::pageHeaders()` repeated every chunk of a row group under each column | each column's own chunk pages, once |
+| `FLOW_PHP_ABI` / `FlowPhpExtension::ABI` 1 | 2: rebuild the `flow_php` extension (and `arrow`, item 4); an older build is refused by `available()` |
+
+### 6) `flow-php/etl`, `flow-php/types` - string comparisons in filters use byte order, as sort already does
+
+| Before | After |
+|------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `ref('a')->lessThan(lit('10'))` on `'9'`: `true` (PHP `<` compares numeric strings as numbers) | `false`: `lessThan`, `lessThanEqual`, `greaterThan`, `greaterThanEqual`, `between`, `greatest` and `least` order two strings by bytes (`strcmp`), as `sortBy()` does |
+| `greatest(lit('9'), lit('10'))`: `'10'` | `'9'` |
+| a `numeric_string` column was comparable only with another `numeric_string` | comparable with a `string` literal too (`ref('id')->greaterThan(lit('1000'))`), and ordered numerically; still not comparable with an `integer` |
+| `Between\Boundary::compareValues(mixed $left, mixed $right)` | removed: `(new ValueOrder(...$operandTypes))->compare($left, $right)`; `Boundary::compare($value, $lower, $upper, ValueOrder $order = new ValueOrder())` |
+
+### 7) `flow-php/etl` - a file extractor reads a `FileScan` and returns only rows matching its conjuncts; `Read` and `Executor\Pipeline` carry it
+
+| Before | After |
+|--------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `FileExtractor::extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles())` | `FileExtractor::scan(FlowContext $context, FileScan $scan)`, returning only rows matching every one of `$scan->conjuncts`; `extract(FlowContext $context, ?int $limit = null)` is `Extractor`'s, `use Scanning;` makes it `scan()` with a limit |
+| `$extractor->extract($context, pathFilter: $filter)` | `$extractor->scan($context, new FileScan(pathFilter: $filter))` |
+| `Read::limit()`, `Read::pathFilter()`, `new Read($extractor, $limit, $pathFilter)` | `Read::scan()->limit`, `->pathFilter`, `->conjuncts`; `new Read($extractor, new FileScan($limit, $pathFilter))` |
+| `Read::withPathFilter()` on any extractor | refused (`InvalidArgumentException`) on an extractor that is no `FileExtractor`, as `withFilter()` |
+| `Executor\Pipeline::limit()`, `pathFilter()`, `new Pipeline(..., $limit, $pathFilter, $sources)` | `Pipeline::scan()`; `new Pipeline(..., $scan, $sources)` |
+| a `filter()` over a file source stayed a `Filter` step | each deterministic part of it is pushed into the source (`FileScan::$conjuncts`) and leaves the step, an emptied step is removed; a `limit()` after it is pushed beside it, so the read stops after that many matching rows; `explain()` prints each part as `Filter: …` under `Read` |
+| `new FileReadLoop($fileColumns, $schema)`, `read($sources, $batches, $batchSize, Backend $backend, $window)` | `new FileReadLoop($fileColumns, $schema, $scan->conjuncts)`, `read($sources, $batches, $batchSize, FlowContext $context, $window)` |
+| `Optimizer::default()` ran `PushLimitIntoSource` before `PushFilterIntoSource` | after it |
+| `Rewrite::of(Node $node)` | `of(Node $original, Node $rebuilt)`: the node as it was and rebuilt over its rewritten children |
+| `new Flow\ETL\Plan\ReplaceLeaf($target, $replacement)` | `ReplaceNodes::one($target, $replacement)`, or `new ReplaceNodes($replacements, $removals)` for several in one rewrite (a removed node gives way to its child) |
+| `new ScalarFunctionFilterTransformer($function, ?ScalarFunction $resolved)` | the second argument is `?Function\Evaluation\Filter`; `bind()` sets it, as before |
+| `explain()`: `Select` printed `Columns: …`, other nodes `Defines columns: …` | `Keeps only: …` and `Changes columns: …` (a column defined, renamed or dropped) |
+| `Redefined::$unknown` | `Redefined::isUnknown()`; `Redefined::isAllBut()` tells an `allBut()` apart |
+
+A custom file extractor:
+
+```php
+// before
+public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator
+{
+ // reads with $limit and $pathFilter
+}
+
+// after
+use Scanning; // extract() is scan() with only a limit
+
+public function scan(FlowContext $context, FileScan $scan): Generator
+{
+ // reads with $scan->limit and $scan->pathFilter, returns only rows matching every one of $scan->conjuncts
+}
+```
+
---
## Upgrading from 0.45.x to 0.46.x
diff --git a/phpunit.xml.dist b/phpunit.xml.dist
index 29cbc06bf6..9a8fe2f9a8 100644
--- a/phpunit.xml.dist
+++ b/phpunit.xml.dist
@@ -58,6 +58,9 @@
src/lib/parquet/tests/Flow/Parquet/Tests/Integration
+
+ src/lib/parquet-viewer/tests/Flow/ParquetViewer/Tests/Unit
+
src/lib/parquet-viewer/tests/Flow/ParquetViewer/Tests/Integration
diff --git a/shell.nix b/shell.nix
index 1118ac4b3e..2c61d28ced 100644
--- a/shell.nix
+++ b/shell.nix
@@ -114,6 +114,7 @@ pkgs.mkShell {
pkgs.just
pkgs.oxipng
pkgs.jpegoptim
+ pkgs.thrift
]
++ pkgs.lib.optional with-blackfire pkgs.blackfire
++ pkgs.lib.optionals with-wasm [
diff --git a/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroExtractor.php b/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroExtractor.php
index 82bd2a81ce..8581487bee 100644
--- a/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroExtractor.php
+++ b/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroExtractor.php
@@ -8,7 +8,10 @@
use Flow\ETL\Exception\SchemaNotDerivableException;
use Flow\ETL\Extractor;
use Flow\ETL\Extractor\File\FileExtractor;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\File\PathFiltering;
+use Flow\ETL\Extractor\File\Scanning;
+use Flow\ETL\Extractor\Signal;
use Flow\ETL\Extractor\Statistics;
use Flow\ETL\FlowContext;
use Flow\ETL\Rows;
@@ -16,12 +19,12 @@
use Flow\Filesystem\Filesystem;
use Flow\Filesystem\Local\NativeLocalFilesystem;
use Flow\Filesystem\Path;
-use Flow\Filesystem\Path\Filter;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
use Generator;
final class AvroExtractor implements Extractor, FileExtractor
{
+ use Scanning;
+
private ?Schema $schema = null;
use PathFiltering;
@@ -35,7 +38,10 @@ public function __construct(
);
}
- public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator
+ /**
+ * @return Generator
+ */
+ public function scan(FlowContext $context, FileScan $scan): Generator
{
yield Rows::empty(new Schema(), $context->backend());
}
diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVExtractor.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVExtractor.php
index 608d094f26..c65fff7748 100644
--- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVExtractor.php
+++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVExtractor.php
@@ -12,10 +12,12 @@
use Flow\ETL\Extractor\File\FileExtractor;
use Flow\ETL\Extractor\File\FileReading;
use Flow\ETL\Extractor\File\FileReadLoop;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\File\InferredColumns;
use Flow\ETL\Extractor\File\ListedFiles;
use Flow\ETL\Extractor\File\MetadataColumnsExtractor;
use Flow\ETL\Extractor\File\ReadWindow;
+use Flow\ETL\Extractor\File\Scanning;
use Flow\ETL\Extractor\InfersSchema;
use Flow\ETL\Extractor\RewindableExtractor;
use Flow\ETL\Extractor\Signal;
@@ -29,8 +31,6 @@
use Flow\Filesystem\Filesystem;
use Flow\Filesystem\Local\NativeLocalFilesystem;
use Flow\Filesystem\Path;
-use Flow\Filesystem\Path\Filter;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
use Flow\Types\Type\Native\String\StringTypeNarrower;
use Generator;
@@ -47,6 +47,7 @@ final class CSVExtractor implements
{
use Batches;
use FileReading;
+ use Scanning;
private SchemaInference $inference;
@@ -97,13 +98,13 @@ public function isRepeatable(): bool
/**
* @return Generator
*/
- public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator
+ public function scan(FlowContext $context, FileScan $scan): Generator
{
$fileColumns = $this->fileColumns($this->filesystem, $this->path);
$schema = $this->schema();
- $sources = iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $pathFilter), false);
+ $sources = iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $scan->pathFilter), false);
- yield from (new FileReadLoop($fileColumns, $schema))->read(
+ yield from (new FileReadLoop($fileColumns, $schema, $scan->conjuncts))->read(
$sources,
new CSVFileBatches(
new CSVFileReader($this->filesystem, $this->readOptions, $sources),
@@ -117,8 +118,8 @@ public function extract(FlowContext $context, ?int $limit = null, Filter $pathFi
: null,
),
$this->batchSize(),
- $context->backend(),
- new ReadWindow(limit: $limit),
+ $context,
+ new ReadWindow(limit: $scan->limit),
);
}
diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php
index 501557b90d..0eb6bc9d0d 100644
--- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php
+++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php
@@ -13,6 +13,7 @@
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\Exception\SchemaDefinitionNotUniqueException;
use Flow\ETL\Exception\SchemaMismatchException;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\Signal;
use Flow\ETL\Row\Reference;
use Flow\ETL\Rows;
@@ -1005,7 +1006,7 @@ public function test_limit(): void
$extractor = from_csv(path_real(__DIR__ . '/../Fixtures/orders_flow.csv'));
$extractor->withBatchSize(1);
- self::assertExtractedRowsCount(2, $extractor, flow_context(config()), limit: 2);
+ self::assertExtractedRowsCount(2, $extractor, flow_context(config()), scan: new FileScan(2));
}
/**
diff --git a/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalKeySetExtractorTest.php b/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalKeySetExtractorTest.php
index 7ae0eed7ba..5c5b0633bb 100644
--- a/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalKeySetExtractorTest.php
+++ b/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalKeySetExtractorTest.php
@@ -15,6 +15,7 @@
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\Exception\RuntimeException;
use Flow\ETL\Exception\SchemaNotDerivableException;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Tests\Double\SpyBackend;
use Flow\ETL\Tests\FlowTestCase;
use Generator;
@@ -151,7 +152,7 @@ public function test_pushed_limit_issues_no_query_once_satisfied(): void
);
$counter->reset();
- self::assertExtractedRowsCount(1500, $extractor, limit: 1500);
+ self::assertExtractedRowsCount(1500, $extractor, scan: new FileScan(1500));
// the NULL check, 1000, then a page narrowed to the 500 still wanted - and no query for row 1501
static::assertSame(3, $counter->count);
}
diff --git a/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalLimitOffsetExtractorTest.php b/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalLimitOffsetExtractorTest.php
index 00746005eb..ac31f5a865 100644
--- a/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalLimitOffsetExtractorTest.php
+++ b/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalLimitOffsetExtractorTest.php
@@ -11,6 +11,7 @@
use Flow\ETL\Adapter\Doctrine\Tests\Double\NativeHandleStub;
use Flow\ETL\Cardinality;
use Flow\ETL\Exception\SchemaNotDerivableException;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Rows;
use Flow\ETL\Tests\Double\SpyBackend;
use Flow\ETL\Tests\FlowTestCase;
@@ -213,7 +214,7 @@ public function test_pushed_limit_skips_the_count_query(): void
))->withBatchSize(2);
$counter->reset();
- self::assertExtractedRowsCount(5, $extractor, limit: 5);
+ self::assertExtractedRowsCount(5, $extractor, scan: new FileScan(5));
// the three pages of the unlimited read, without the COUNT(*) in front of them
static::assertSame(3, $counter->count);
static::assertStringNotContainsString('COUNT(*)', implode(' ', $counter->queries));
diff --git a/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalQueryExtractorTest.php b/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalQueryExtractorTest.php
index 23f43c874d..6f4d1a1624 100644
--- a/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalQueryExtractorTest.php
+++ b/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Unit/DbalQueryExtractorTest.php
@@ -12,6 +12,7 @@
use Flow\ETL\Adapter\Doctrine\Tests\Double\NativeHandleStub;
use Flow\ETL\Cardinality;
use Flow\ETL\Exception\SchemaNotDerivableException;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Rows;
use Flow\ETL\Tests\Double\SpyBackend;
use Flow\ETL\Tests\FlowTestCase;
@@ -131,7 +132,7 @@ public function test_pushed_limit_stops_querying_further_parameter_sets(int $bat
$counter->reset();
// at 100 the first set is one batch that overshoots the limit - trimming it is the limit operator's job
- self::assertExtractedRowsCount($extractedRows, $extractor, limit: 5);
+ self::assertExtractedRowsCount($extractedRows, $extractor, scan: new FileScan(5));
static::assertSame(1, $counter->count);
}
diff --git a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/ExcelExtractor.php b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/ExcelExtractor.php
index e079d336de..bae5e9de48 100644
--- a/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/ExcelExtractor.php
+++ b/src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/ExcelExtractor.php
@@ -12,10 +12,12 @@
use Flow\ETL\Extractor\File\FileExtractor;
use Flow\ETL\Extractor\File\FileReading;
use Flow\ETL\Extractor\File\FileReadLoop;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\File\InferredColumns;
use Flow\ETL\Extractor\File\ListedFiles;
use Flow\ETL\Extractor\File\MetadataColumnsExtractor;
use Flow\ETL\Extractor\File\ReadWindow;
+use Flow\ETL\Extractor\File\Scanning;
use Flow\ETL\Extractor\InfersSchema;
use Flow\ETL\Extractor\RewindableExtractor;
use Flow\ETL\Extractor\Signal;
@@ -29,8 +31,6 @@
use Flow\Filesystem\Filesystem;
use Flow\Filesystem\Local\NativeLocalFilesystem;
use Flow\Filesystem\Path;
-use Flow\Filesystem\Path\Filter;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
use Generator;
use Throwable;
@@ -48,6 +48,7 @@ final class ExcelExtractor implements
{
use Batches;
use FileReading;
+ use Scanning;
private SchemaInference $inference;
@@ -121,7 +122,7 @@ public function isRepeatable(): bool
/**
* @return Generator
*/
- public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator
+ public function scan(FlowContext $context, FileScan $scan): Generator
{
$fileColumns = $this->fileColumns($this->filesystem, $this->path);
$schema = $this->schema();
@@ -141,12 +142,12 @@ public function extract(FlowContext $context, ?int $limit = null, Filter $pathFi
$this->sampled = null;
try {
- yield from (new FileReadLoop($fileColumns, $schema))->read(
- iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $pathFilter), false),
+ yield from (new FileReadLoop($fileColumns, $schema, $scan->conjuncts))->read(
+ iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $scan->pathFilter), false),
$batches,
$this->batchSize(),
- $context->backend(),
- new ReadWindow(limit: $limit),
+ $context,
+ new ReadWindow(limit: $scan->limit),
);
} finally {
$batches->close();
diff --git a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/ExcelExtractorTest.php b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/ExcelExtractorTest.php
index b5190be478..57a8c1db70 100644
--- a/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/ExcelExtractorTest.php
+++ b/src/adapter/etl-adapter-excel/tests/Flow/ETL/Adapter/Excel/Tests/Integration/ExcelExtractorTest.php
@@ -14,6 +14,7 @@
use Flow\ETL\Exception\InferredSchemaException;
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\Exception\SchemaMismatchException;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\Signal;
use Flow\ETL\Rows;
use Flow\ETL\Tests\Context\ExtractedRows;
@@ -759,7 +760,7 @@ public function test_limit_reached_on_the_first_file_tail_batch_skips_the_remain
{
$extractor = from_excel(ExcelFixtureContext::file('cross_stream/*/*.xlsx'))->withBatchSize(10);
- static::assertCount(2, ExtractedRows::of($extractor, limit: 2));
+ static::assertCount(2, ExtractedRows::of($extractor, scan: new FileScan(2)));
}
/**
@@ -877,9 +878,9 @@ public function test_a_filtered_extract_infers_over_every_listed_file(): void
$filtered = from_excel(ExcelFixtureContext::path('sniff/*'));
$whole = from_excel(ExcelFixtureContext::path('sniff/*'));
- iterator_to_array($filtered->extract(flow_context(), null, new KeepPaths([ExcelFixtureContext::path(
- 'sniff/a',
- )])), false);
+ iterator_to_array($filtered->scan(flow_context(), new FileScan(pathFilter: new KeepPaths([
+ ExcelFixtureContext::path('sniff/a'),
+ ]))), false);
$whole->schema();
static::assertEquals($whole->statistics()->rows, $filtered->statistics()->rows);
diff --git a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonExtractor.php b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonExtractor.php
index 913e63e3c1..ca88a7caac 100644
--- a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonExtractor.php
+++ b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonExtractor.php
@@ -12,9 +12,11 @@
use Flow\ETL\Extractor\File\FileExtractor;
use Flow\ETL\Extractor\File\FileReading;
use Flow\ETL\Extractor\File\FileReadLoop;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\File\ListedFiles;
use Flow\ETL\Extractor\File\MetadataColumnsExtractor;
use Flow\ETL\Extractor\File\ReadWindow;
+use Flow\ETL\Extractor\File\Scanning;
use Flow\ETL\Extractor\InfersSchema;
use Flow\ETL\Extractor\RewindableExtractor;
use Flow\ETL\Extractor\Signal;
@@ -28,8 +30,6 @@
use Flow\Filesystem\Filesystem;
use Flow\Filesystem\Local\NativeLocalFilesystem;
use Flow\Filesystem\Path;
-use Flow\Filesystem\Path\Filter;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
use Flow\Types\Type\Logical\InstanceOfTypeNarrower;
use Generator;
@@ -46,6 +46,7 @@ final class JsonExtractor implements
{
use Batches;
use FileReading;
+ use Scanning;
private SchemaInference $inference;
@@ -94,13 +95,13 @@ public function isRepeatable(): bool
/**
* @return Generator
*/
- public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator
+ public function scan(FlowContext $context, FileScan $scan): Generator
{
$fileColumns = $this->fileColumns($this->filesystem, $this->path);
$schema = $this->schema();
- $sources = iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $pathFilter), false);
+ $sources = iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $scan->pathFilter), false);
- yield from (new FileReadLoop($fileColumns, $schema))->read(
+ yield from (new FileReadLoop($fileColumns, $schema, $scan->conjuncts))->read(
$sources,
new JsonFileBatches(
$this->filesystem,
@@ -115,8 +116,8 @@ public function extract(FlowContext $context, ?int $limit = null, Filter $pathFi
$this->pointer,
),
$this->batchSize(),
- $context->backend(),
- new ReadWindow(limit: $limit),
+ $context,
+ new ReadWindow(limit: $scan->limit),
);
}
diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonExtractorTest.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonExtractorTest.php
index 2475b04597..21e5a22b8c 100644
--- a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonExtractorTest.php
+++ b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonExtractorTest.php
@@ -12,6 +12,7 @@
use Flow\ETL\Config;
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\Exception\SchemaMismatchException;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\Signal;
use Flow\ETL\Schema\Definition\StringDefinition;
use Flow\ETL\Tests\Context\ExtractedRows;
@@ -197,7 +198,7 @@ public function test_limit(): void
$extractor = from_json(path(__DIR__ . '/../../Fixtures/timezones.json'));
$extractor->withBatchSize(1);
- self::assertExtractedRowsCount(2, $extractor, flow_context(config()), limit: 2);
+ self::assertExtractedRowsCount(2, $extractor, flow_context(config()), scan: new FileScan(2));
}
public function test_schema_appends_the_metadata_column(): void
diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonLinesExtractorTest.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonLinesExtractorTest.php
index 817c815bae..a7d0723814 100644
--- a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonLinesExtractorTest.php
+++ b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonLinesExtractorTest.php
@@ -11,6 +11,7 @@
use Flow\ETL\Config;
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\Exception\SchemaMismatchException;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\Signal;
use Flow\ETL\Schema\Definition\StringDefinition;
use Flow\ETL\Tests\Context\ExtractedRows;
@@ -189,7 +190,7 @@ public function test_limit(): void
$extractor = from_json_lines(path(__DIR__ . '/../../Fixtures/timezones.jsonl'));
$extractor->withBatchSize(1);
- self::assertExtractedRowsCount(2, $extractor, flow_context(config()), limit: 2);
+ self::assertExtractedRowsCount(2, $extractor, flow_context(config()), scan: new FileScan(2));
}
public function test_schema_appends_the_metadata_column(): void
diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/AdaptiveParquetOpenSource.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/AdaptiveParquetOpenSource.php
index 4692df24a8..f2571cdf2a 100644
--- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/AdaptiveParquetOpenSource.php
+++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/AdaptiveParquetOpenSource.php
@@ -9,6 +9,7 @@
use Flow\ETL\Schema;
use Flow\Parquet\Engine\RustParquetFileReader;
use Flow\Parquet\ParquetFile;
+use Flow\Parquet\Reader\ReadPlan;
use Iterator;
final readonly class AdaptiveParquetOpenSource implements ParquetOpenSource
@@ -25,9 +26,9 @@ public function __construct(ParquetFile $file, ?FlowPhpExtension $extension = nu
: new PhpParquetOpenSource($file);
}
- public function batches(Schema $schema, int $batchSize, ?int $offset, ?int $limit, Backend $backend): Iterator
+ public function batches(Schema $schema, int $batchSize, ReadPlan $plan, Backend $backend): Iterator
{
- return $this->source->batches($schema, $batchSize, $offset, $limit, $backend);
+ return $this->source->batches($schema, $batchSize, $plan, $backend);
}
public function close(): void
diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetExtractor.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetExtractor.php
index 1f1f374412..e5706a4437 100644
--- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetExtractor.php
+++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetExtractor.php
@@ -12,8 +12,10 @@
use Flow\ETL\Extractor\File\FileExtractor;
use Flow\ETL\Extractor\File\FileReading;
use Flow\ETL\Extractor\File\FileReadLoop;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\File\MetadataColumnsExtractor;
use Flow\ETL\Extractor\File\ReadWindow;
+use Flow\ETL\Extractor\File\Scanning;
use Flow\ETL\Extractor\File\SelfDescribingFile;
use Flow\ETL\Extractor\File\SourceFile;
use Flow\ETL\Extractor\RewindableExtractor;
@@ -47,6 +49,7 @@ final class ParquetExtractor implements
{
use Batches;
use FileReading;
+ use Scanning;
private ByteOrder $byteOrder = ByteOrder::LITTLE_ENDIAN;
@@ -111,9 +114,9 @@ public function isRepeatable(): bool
/**
* @return Generator
*/
- public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator
+ public function scan(FlowContext $context, FileScan $scan): Generator
{
- $sources = iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $pathFilter), false);
+ $sources = iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $scan->pathFilter), false);
// every file is read under the first file's schema, or the union of all of them
$target = $this->schema();
$fileColumns = $this->fileColumns($this->filesystem, $this->path);
@@ -124,16 +127,18 @@ public function extract(FlowContext $context, ?int $limit = null, Filter $pathFi
$this->unionByName
? null
: new DerivedSchema($this->derivedSchema($this->schemaFiles(), $this->unionByName), $this->derivedFrom),
+ $fileColumns->withoutTail($target),
+ $scan->conjuncts,
);
$this->firstFile = null;
try {
- yield from (new FileReadLoop($fileColumns, $target))->read(
+ yield from (new FileReadLoop($fileColumns, $target, $scan->conjuncts))->read(
$sources,
$batches,
$this->batchSize(),
- $context->backend(),
- new ReadWindow($this->offset ?? 0, $limit),
+ $context,
+ new ReadWindow($this->offset ?? 0, $scan->limit),
);
} finally {
$batches->close();
diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetFileBatches.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetFileBatches.php
index 204b82b3c4..54e707c23f 100644
--- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetFileBatches.php
+++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetFileBatches.php
@@ -4,6 +4,8 @@
namespace Flow\ETL\Adapter\Parquet;
+use Flow\ETL\Adapter\Parquet\Pushdown\FilterTranslator;
+use Flow\ETL\Adapter\Parquet\Pushdown\TranslatedConjunct;
use Flow\ETL\Column\Backend;
use Flow\ETL\Exception\InferredSchemaException;
use Flow\ETL\Extractor\File\DerivedSchema;
@@ -11,26 +13,41 @@
use Flow\ETL\Extractor\File\OffsetSkippingFileBatches;
use Flow\ETL\Extractor\File\ReadWindow;
use Flow\ETL\Extractor\File\SourceFile;
+use Flow\ETL\Function\ScalarFunction;
use Flow\ETL\Rows;
use Flow\ETL\Schema;
+use Flow\Parquet\Filter\All;
+use Flow\Parquet\Filter\Binding\BoundFilter;
+use Flow\Parquet\Reader\ReadPlanner;
use Generator;
final class ParquetFileBatches implements OffsetSkippingFileBatches
{
+ /**
+ * @var list
+ */
+ private readonly array $conjuncts;
+
/**
* @param null|ParquetSourceFile $firstFile the file the schema pass kept open, read instead of opened again
* @param null|DerivedSchema $derived the schema every file must match; null under unionByName
+ * @param list $conjuncts FileScan::$conjuncts
*/
public function __construct(
private readonly ParquetSourceFileOpener $opener,
private readonly FileColumns $fileColumns,
private ?ParquetSourceFile $firstFile,
private readonly ?DerivedSchema $derived,
- ) {}
+ Schema $body,
+ array $conjuncts = [],
+ ) {
+ $this->conjuncts = (new FilterTranslator())->translate($conjuncts, $body);
+ }
/**
* Skips the whole file when the window's offset covers it; otherwise reads from the offset. A file of a narrower
- * schema is matched to $body.
+ * schema is matched to $body. What the conjuncts a file answers exactly prove cannot match is skipped; FileReadLoop
+ * drops the rest that does not match them.
*
* @param int<1, max> $batchSize
*
@@ -57,7 +74,8 @@ public function batches(
$opened = null;
try {
- $fileRows = $file->file->reader()->rowsNumber();
+ $footer = $file->file->reader()->footer();
+ $fileRows = $footer->rowsNumber();
if ($window->offset >= $fileRows) {
return $fileRows;
@@ -66,9 +84,31 @@ public function batches(
$this->derived?->refuseDivergence($file->source()->uri(), $file->schema());
$fileBody = $this->fileColumns->withoutTail($file->schema());
+ $kept = [];
+
+ foreach ($this->conjuncts as $conjunct) {
+ if ($conjunct->readsAs($fileBody, $body)) {
+ $kept[] = $conjunct->filter;
+ }
+ }
+
$opened = $file->open();
- foreach ($opened->batches($fileBody, $batchSize, $window->offset, $window->limit, $backend) as $rows) {
+ // rows matching the conjuncts the file binds; FileReadLoop drops those the rest do not match
+ foreach ($opened->batches(
+ $fileBody,
+ $batchSize,
+ (new ReadPlanner())->plan(
+ $footer,
+ $file->file->indexes(),
+ $kept === []
+ ? null
+ : BoundFilter::bind(new All(...$kept), $footer->schema(), $file->file->options()),
+ $window->offset,
+ $window->limit,
+ ),
+ $backend,
+ ) as $rows) {
yield $fileBody->isSame($body) ? $rows : $rows->matchTo($body, $backend);
}
diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetOpenSource.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetOpenSource.php
index c1cfc09013..bdf3e4314b 100644
--- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetOpenSource.php
+++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetOpenSource.php
@@ -7,6 +7,7 @@
use Flow\ETL\Column\Backend;
use Flow\ETL\Rows;
use Flow\ETL\Schema;
+use Flow\Parquet\Reader\ReadPlan;
use Iterator;
use function interface_exists;
@@ -20,11 +21,11 @@ interface ParquetOpenSource
/**
* Batches of at most $batchSize rows under $schema, every column adopted by $backend.
*
- * @param int<1, max> $batchSize
+ * @param int $batchSize refused below 1
*
* @return Iterator
*/
- public function batches(Schema $schema, int $batchSize, ?int $offset, ?int $limit, Backend $backend): Iterator;
+ public function batches(Schema $schema, int $batchSize, ReadPlan $plan, Backend $backend): Iterator;
public function close(): void;
}
diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetSourceFile.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetSourceFile.php
index 6c9095029b..6ee7f9a20c 100644
--- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetSourceFile.php
+++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetSourceFile.php
@@ -74,8 +74,8 @@ public function source(): SourceFile
public function statistics(): Statistics
{
return new Statistics(
- rows: Cardinality::exact($this->file->reader()->rowsNumber()),
- size: Cardinality::exact($this->file->reader()->totalByteSize()),
+ rows: Cardinality::exact($this->file->reader()->footer()->rowsNumber()),
+ size: Cardinality::exact($this->file->reader()->footer()->totalByteSize()),
);
}
}
diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/PhpParquetOpenSource.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/PhpParquetOpenSource.php
index 68eb9383da..dcda6eb973 100644
--- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/PhpParquetOpenSource.php
+++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/PhpParquetOpenSource.php
@@ -10,6 +10,7 @@
use Flow\ETL\Schema;
use Flow\ETL\Schema\Definition;
use Flow\Parquet\ParquetFile;
+use Flow\Parquet\Reader\ReadPlan;
use Generator;
use function array_map;
@@ -24,13 +25,13 @@ public function __construct(
/**
* @return Generator
*/
- public function batches(Schema $schema, int $batchSize, ?int $offset, ?int $limit, Backend $backend): Generator
+ public function batches(Schema $schema, int $batchSize, ReadPlan $plan, Backend $backend): Generator
{
$names = array_values(array_map(static fn(Definition $definition): string => $definition
->entry()
->name(), $schema->definitions()));
- foreach ($this->file->columns($batchSize, $names, $limit, $offset) as $chunk) {
+ foreach ($this->file->reader()->readColumns($names, $batchSize, $plan) as $chunk) {
$builder = new RowsBuilder($schema, $backend);
foreach ($names as $name) {
diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/Pushdown/FilterTranslator.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/Pushdown/FilterTranslator.php
new file mode 100644
index 0000000000..d498cb6140
--- /dev/null
+++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/Pushdown/FilterTranslator.php
@@ -0,0 +1,355 @@
+ $conjuncts
+ *
+ * @return list one per translatable conjunct
+ */
+ public function translate(array $conjuncts, Schema $body): array
+ {
+ $translated = [];
+
+ foreach ($conjuncts as $conjunct) {
+ $parquet = $this->filter($conjunct, $body);
+
+ if ($parquet !== null) {
+ $translated[] = new TranslatedConjunct(
+ $parquet,
+ array_values((new ReferencedColumns())->in($conjunct)->names()),
+ );
+ }
+ }
+
+ return $translated;
+ }
+
+ private function filter(ScalarFunction $function, Schema $body): ?Filter
+ {
+ $children = $function->children();
+
+ return match (true) {
+ $function instanceof All, $function instanceof Any => $this->combined($function, $body),
+ $function instanceof Not => $children[0] instanceof ScalarFunction
+ ? $this->filter($children[0], $body)?->negate()
+ : null,
+ $function instanceof Equals => $this->comparison($children[0], $children[1], Operator::Equal, $body, false),
+ $function instanceof NotEquals => $this->comparison(
+ $children[0],
+ $children[1],
+ Operator::Equal,
+ $body,
+ false,
+ )?->negate(),
+ $function instanceof Same => $this->comparison($children[0], $children[1], Operator::Equal, $body, true),
+ $function instanceof NotSame => $this->comparison(
+ $children[0],
+ $children[1],
+ Operator::Equal,
+ $body,
+ true,
+ )?->negate(),
+ $function instanceof LessThan => $this->comparison(
+ $children[0],
+ $children[1],
+ Operator::LessThan,
+ $body,
+ false,
+ ),
+ $function instanceof LessThanEqual => $this->comparison(
+ $children[0],
+ $children[1],
+ Operator::LessThanOrEqual,
+ $body,
+ false,
+ ),
+ $function instanceof GreaterThan => $this->comparison(
+ $children[0],
+ $children[1],
+ Operator::GreaterThan,
+ $body,
+ false,
+ ),
+ $function instanceof GreaterThanEqual => $this->comparison(
+ $children[0],
+ $children[1],
+ Operator::GreaterThanOrEqual,
+ $body,
+ false,
+ ),
+ $function instanceof IsNull => $this->isNull($children[0], $body),
+ $function instanceof IsNotNull => $this->isNull($children[0], $body)?->negate(),
+ $function instanceof IsIn => $this->in($children[0], $children[1], $body),
+ $function instanceof StartsWith => $this->startsWith($children[0], $children[1], $body),
+ $function instanceof Between => $this->between($function, $body),
+ default => null,
+ };
+ }
+
+ private function combined(All|Any $function, Schema $body): ?Filter
+ {
+ $filters = [];
+
+ foreach ($function->children() as $child) {
+ $filter = $this->filter($child, $body);
+
+ if ($filter === null) {
+ return null;
+ }
+
+ $filters[] = $filter;
+ }
+
+ return $function instanceof All ? new ParquetAll(...$filters) : new ParquetAny(...$filters);
+ }
+
+ private function comparison(
+ FunctionTree $left,
+ FunctionTree $right,
+ Operator $operator,
+ Schema $body,
+ bool $strict,
+ ): ?Comparison {
+ if ($left instanceof Literal && $right instanceof UnresolvedReference) {
+ return $this->comparison(
+ $right,
+ $left,
+ match ($operator) {
+ Operator::LessThan => Operator::GreaterThan,
+ Operator::LessThanOrEqual => Operator::GreaterThanOrEqual,
+ Operator::GreaterThan => Operator::LessThan,
+ Operator::GreaterThanOrEqual => Operator::LessThanOrEqual,
+ Operator::Equal => Operator::Equal,
+ },
+ $body,
+ $strict,
+ );
+ }
+
+ $definition = $this->definition($left, $body);
+
+ if ($definition === null || !$right instanceof Literal) {
+ return null;
+ }
+
+ $value = $this->literal(
+ type_bare($definition->type()),
+ $right->returns(),
+ $right->value(),
+ $operator !== Operator::Equal,
+ $strict,
+ );
+
+ return $value === null ? null : new Comparison(parquet_column($definition->entry()->name()), $operator, $value);
+ }
+
+ private function isNull(FunctionTree $reference, Schema $body): ?Filter
+ {
+ $definition = $this->definition($reference, $body);
+
+ return $definition === null ? null : parquet_column($definition->entry()->name())->isNull();
+ }
+
+ private function in(FunctionTree $haystack, FunctionTree $needle, Schema $body): ?Filter
+ {
+ $definition = $this->definition($needle, $body);
+ // @mago-expect analysis:mixed-assignment
+ $list = $haystack instanceof Literal ? $haystack->value() : null;
+
+ if ($definition === null || !is_array($list)) {
+ return null;
+ }
+
+ // NULL never matches: x IN (1, NULL) keeps what x IN (1) keeps, and NOT (x IN (1, NULL)) is never TRUE
+ $present = array_values(array_filter($list, static fn(mixed $element): bool => $element !== null));
+ $detector = new TypeDetector();
+ $values = array_map(fn(mixed $element): mixed => $this->literal(
+ type_bare($definition->type()),
+ $detector->detectType($element),
+ $element,
+ false,
+ false,
+ ), $present);
+
+ return in_array(null, $values, true) ? null : parquet_column($definition->entry()->name())->in($values);
+ }
+
+ private function startsWith(FunctionTree $haystack, FunctionTree $needle, Schema $body): ?Filter
+ {
+ $definition = $this->definition($haystack, $body);
+ // @mago-expect analysis:mixed-assignment
+ $prefix = $needle instanceof Literal ? $needle->value() : null;
+
+ return $definition !== null && type_bare($definition->type()) instanceof StringType && is_string($prefix)
+ ? parquet_column($definition->entry()->name())->startsWith($prefix)
+ : null;
+ }
+
+ private function between(Between $between, Schema $body): ?Filter
+ {
+ [$value, $lower, $upper, $boundary] = $between->children();
+
+ // @mago-expect analysis:mixed-assignment
+ $kind = $boundary instanceof Literal ? $boundary->value() : null;
+
+ if (!$kind instanceof Boundary) {
+ return null;
+ }
+
+ $lowerBound = $this->comparison(
+ $value,
+ $lower,
+ $kind === Boundary::INCLUSIVE || $kind === Boundary::LEFT_INCLUSIVE
+ ? Operator::GreaterThanOrEqual
+ : Operator::GreaterThan,
+ $body,
+ false,
+ );
+ $upperBound = $this->comparison(
+ $value,
+ $upper,
+ $kind === Boundary::INCLUSIVE || $kind === Boundary::RIGHT_INCLUSIVE
+ ? Operator::LessThanOrEqual
+ : Operator::LessThan,
+ $body,
+ false,
+ );
+
+ return $lowerBound === null || $upperBound === null ? null : $lowerBound->and($upperBound);
+ }
+
+ /**
+ * @return null|Definition the body column $function is a reference to, null when it is none
+ */
+ private function definition(FunctionTree $function, Schema $body): ?Definition
+ {
+ return $function instanceof UnresolvedReference ? $body->findDefinition($function->to()) : null;
+ }
+
+ /**
+ * @param Type $column
+ * @param Type $literal
+ *
+ * @return null|bool|DateTimeInterface|float|int|string the literal as the parquet filter takes it, null when
+ * the pair cannot be pushed
+ */
+ private function literal(Type $column, Type $literal, mixed $value, bool $ordering, bool $strict): mixed
+ {
+ $literal = type_bare($literal);
+
+ if (is_float($value) && is_nan($value)) {
+ return null;
+ }
+
+ return match (true) {
+ $column instanceof JsonType => null,
+ $column instanceof NumericStringType => !$strict
+ && $literal instanceof StringType
+ && is_string($value)
+ && ctype_digit($value)
+ && ($value === '0' || $value[0] !== '0')
+ ? $value
+ : null,
+ !$strict && $column instanceof FloatType && $literal instanceof IntegerType => is_int($value)
+ && abs($value) <= LiteralDomain::EXACT_FLOAT_INTEGER
+ ? (float) $value
+ : null,
+ !$strict && $column instanceof IntegerType && $literal instanceof FloatType => is_float($value)
+ && floor($value) === $value
+ && abs($value) < LiteralDomain::EXACT_FLOAT_INTEGER
+ ? (int) $value
+ : null,
+ !$strict
+ && (
+ $column instanceof DateTimeType
+ && $literal instanceof DateType
+ || $column instanceof DateType
+ && $literal instanceof DateTimeType
+ )
+ => $value instanceof DateTimeInterface ? $value : null,
+ $column::class !== $literal::class,
+ !(
+ $ordering
+ ? (new ComparableValues())->orderedByPhysical($column)
+ : (new ComparableValues())->equalByPhysical($column)
+ ),
+ => null,
+ $value instanceof Uuid => $value->toString(),
+ is_bool($value),
+ is_int($value),
+ is_float($value),
+ is_string($value),
+ $value instanceof DateTimeInterface,
+ => $value,
+ default => null,
+ };
+ }
+}
diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/Pushdown/TranslatedConjunct.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/Pushdown/TranslatedConjunct.php
new file mode 100644
index 0000000000..025329e36c
--- /dev/null
+++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/Pushdown/TranslatedConjunct.php
@@ -0,0 +1,44 @@
+ $columns Flow column names the predicate reads
+ */
+ public function __construct(
+ public Filter $filter,
+ public array $columns,
+ ) {}
+
+ /**
+ * True when the file stores every column the predicate reads as $body types it, or not at all. A column the file
+ * stores as another type is cast to the body's, so its statistics are of the other type.
+ */
+ public function readsAs(Schema $fileBody, Schema $body): bool
+ {
+ $comparator = new Comparator();
+
+ foreach ($this->columns as $column) {
+ $stored = $fileBody->findDefinition($column);
+
+ if (
+ $stored !== null
+ && !$comparator->equals(type_bare($stored->type()), type_bare($body->get($column)->type()))
+ ) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+}
diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/SchemaConverter.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/SchemaConverter.php
index 8a31da9663..92e4b4903c 100644
--- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/SchemaConverter.php
+++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/SchemaConverter.php
@@ -11,6 +11,7 @@
use Flow\Parquet\ParquetFile\Schema\Column;
use Flow\Parquet\ParquetFile\Schema\FlatColumn;
use Flow\Parquet\ParquetFile\Schema\ListElement;
+use Flow\Parquet\ParquetFile\Schema\LogicalType\Integer;
use Flow\Parquet\ParquetFile\Schema\NestedColumn;
use Flow\Types\Type;
use Flow\Types\Type\Logical\DateTimeType;
@@ -20,6 +21,7 @@
use Flow\Types\Type\Logical\JsonType;
use Flow\Types\Type\Logical\ListType;
use Flow\Types\Type\Logical\MapType;
+use Flow\Types\Type\Logical\NumericStringType;
use Flow\Types\Type\Logical\OptionalType;
use Flow\Types\Type\Logical\StructureElement;
use Flow\Types\Type\Logical\StructureType;
@@ -38,6 +40,7 @@
use function Flow\ETL\DSL\bool_schema;
use function Flow\ETL\DSL\date_schema;
use function Flow\ETL\DSL\datetime_schema;
+use function Flow\ETL\DSL\definition_from_type;
use function Flow\ETL\DSL\float_schema;
use function Flow\ETL\DSL\int_schema;
use function Flow\ETL\DSL\json_schema;
@@ -56,6 +59,7 @@
use function Flow\Types\DSL\type_json;
use function Flow\Types\DSL\type_list;
use function Flow\Types\DSL\type_map;
+use function Flow\Types\DSL\type_numeric_string;
use function Flow\Types\DSL\type_optional;
use function Flow\Types\DSL\type_string;
use function Flow\Types\DSL\type_structure;
@@ -118,6 +122,7 @@ private function flowToParquet(string $name, Type $type, bool $nullable): Column
XMLElementType::class,
XMLType::class,
StringType::class,
+ NumericStringType::class,
NullType::class,
=> FlatColumn::string($name, $repetition),
BooleanType::class => FlatColumn::boolean($name, $repetition),
@@ -171,6 +176,10 @@ private function parquetToFlowDefinition(Column $column): Definition
$nullable = $column->repetition() === ParquetSchema\Repetition::OPTIONAL;
+ if (Integer::forColumn($column)?->is(64, false) === true) {
+ return definition_from_type($column->name(), type_numeric_string(), $nullable);
+ }
+
if ($logicalType === null) {
return match ($column->type()) {
ParquetSchema\PhysicalType::INT32 => match ($column->convertedType()) {
@@ -247,6 +256,10 @@ public function parquetToFlowType(Column $column): Type
$nullable = $column->repetition() === ParquetSchema\Repetition::OPTIONAL;
+ if (Integer::forColumn($column)?->is(64, false) === true) {
+ return $nullable ? type_optional(type_numeric_string()) : type_numeric_string();
+ }
+
if ($logicalType === null) {
$type = match ($column->type()) {
ParquetSchema\PhysicalType::INT32 => match ($column->convertedType()) {
diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Context/ParquetFilesContext.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Context/ParquetFilesContext.php
index fc72becdf0..8ad13b9420 100644
--- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Context/ParquetFilesContext.php
+++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Context/ParquetFilesContext.php
@@ -5,6 +5,7 @@
namespace Flow\ETL\Adapter\Parquet\Tests\Context;
use Flow\ETL\Adapter\Parquet\ParquetOpenSink;
+use Flow\ETL\Adapter\Parquet\ParquetSourceFileOpener;
use Flow\ETL\Adapter\Parquet\RustParquetOpenSink;
use Flow\ETL\Adapter\Parquet\SchemaConverter;
use Flow\ETL\Rows;
@@ -20,14 +21,20 @@
use Flow\Parquet\Options;
use Flow\Parquet\ParquetFile\Compressions;
use Flow\Parquet\ParquetFile\Encodings;
+use Flow\Parquet\ParquetFile\RowGroup\ColumnChunk;
use Flow\Parquet\ParquetFile\Schema;
use Flow\Parquet\ParquetFile\Schema\Column;
use Flow\Parquet\ParquetFile\Schema\FlatColumn;
+use Flow\Parquet\ParquetFile\Schema\LogicalType;
+use Flow\Parquet\ParquetFile\Schema\PhysicalType;
use Flow\Parquet\Reader;
+use Flow\Parquet\Reader\ReadPlan;
+use Flow\Parquet\Reader\ReadPlanner;
use Flow\Parquet\Writer;
use function array_count_values;
use function array_map;
+use function array_sum;
use function Flow\ETL\Adapter\Parquet\to_parquet;
use function Flow\ETL\DSL\data_frame;
use function Flow\ETL\DSL\from_rows;
@@ -37,6 +44,13 @@
final class ParquetFilesContext
{
+ public static function plan(Filesystem $filesystem, string $uri, int $offset = 0, ?int $limit = null): ReadPlan
+ {
+ $file = (new Reader())->readStream($filesystem->readFrom(path($uri)));
+
+ return (new ReadPlanner())->plan($file->reader()->footer(), $file->indexes(), null, $offset, $limit);
+ }
+
/**
* @return array
*/
@@ -149,4 +163,116 @@ public static function opensPerFile(CountingFilesystem $filesystem): array
->path()
->path(), $filesystem->openedStreams));
}
+
+ /**
+ * Bytes read through every stream the filesystem opened.
+ */
+ public static function bytesRead(CountingFilesystem $filesystem): int
+ {
+ return array_sum(array_map(
+ static fn(ClosingSpySourceStream $stream): int => $stream->readBytes,
+ $filesystem->openedStreams,
+ ));
+ }
+
+ /**
+ * The chunks of the given columns in the given row groups, by file uri.
+ *
+ * @param array>> $groups
+ *
+ * @return list
+ */
+ public static function chunks(Filesystem $filesystem, array $groups): array
+ {
+ $chunks = [];
+
+ foreach ($groups as $uri => $columns) {
+ $file = Reader::php()->readStream($filesystem->readFrom(path($uri)));
+ $rowGroups = $file->metadata()->rowGroups()->all();
+
+ foreach ($columns as $column => $indexes) {
+ foreach ($indexes as $index) {
+ $chunks[] = $rowGroups[$index]->getColumnChunk($file->schema()->getFlat($column));
+ }
+ }
+ }
+
+ return $chunks;
+ }
+
+ /**
+ * The bytes of the offset indexes of the given columns in the given row groups, by file uri.
+ *
+ * @param array>> $groups
+ */
+ public static function offsetIndexBytes(Filesystem $filesystem, array $groups): int
+ {
+ return array_sum(array_map(
+ static fn(ColumnChunk $chunk): int => (int) $chunk->offsetIndex()?->length,
+ self::chunks($filesystem, $groups),
+ ));
+ }
+
+ /**
+ * The bytes of the column and offset indexes of the given columns in the given row groups, by file uri.
+ *
+ * @param array>> $groups
+ */
+ public static function pageIndexBytes(Filesystem $filesystem, array $groups): int
+ {
+ return array_sum(array_map(
+ static fn(ColumnChunk $chunk): int => (
+ (int) $chunk->columnIndex()?->length + (int) $chunk->offsetIndex()?->length
+ ),
+ self::chunks($filesystem, $groups),
+ ));
+ }
+
+ /**
+ * A PHP-engine opener of every column under the default options.
+ */
+ public static function opener(Filesystem $filesystem): ParquetSourceFileOpener
+ {
+ return new ParquetSourceFileOpener(
+ $filesystem,
+ new PhpParquetEngine(),
+ Options::default(),
+ new SchemaConverter(),
+ [],
+ );
+ }
+
+ /**
+ * @param iterable $batches
+ */
+ public static function rowCount(iterable $batches): int
+ {
+ $count = 0;
+
+ foreach ($batches as $rows) {
+ $count += $rows->count();
+ }
+
+ return $count;
+ }
+
+ /**
+ * A `UINT_64` column `big`, one row group per value.
+ *
+ * @param list $values
+ */
+ public static function unsignedInt64(Filesystem $filesystem, string $uri, array $values): void
+ {
+ $options = Options::default()->set(Option::ROW_GROUP_SIZE_BYTES, 1)->set(
+ Option::ROW_GROUP_SIZE_CHECK_INTERVAL,
+ 1,
+ );
+ $writer = new Writer(options: $options, engine: new PhpParquetEngine(options: $options));
+ $writer->openForStream(
+ $filesystem->writeTo(path($uri)),
+ Schema::with(new FlatColumn('big', PhysicalType::INT64, logicalType: LogicalType::integer(64, false))),
+ );
+ $writer->writeBatch(array_map(static fn(int|string $value): array => ['big' => $value], $values));
+ $writer->close();
+ }
}
diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Context/PushdownParityContext.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Context/PushdownParityContext.php
new file mode 100644
index 0000000000..89c3f9cd1d
--- /dev/null
+++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Context/PushdownParityContext.php
@@ -0,0 +1,109 @@
+ [
+ 'id' => [0, 1, 2],
+ 'name' => [0, 1, 2],
+ 'score' => [0, 1, 2],
+ 'd92' => [0, 1, 2],
+ 'd182' => [0, 1, 2],
+ 'day' => [0, 1, 2],
+ 'ts_us' => [0, 1, 2],
+ 'ts_ns' => [0, 1, 2],
+ 'uuid' => [0, 1, 2],
+ 'extra' => [0, 1, 2],
+ ],
+ 'memory://parity/b.parquet' => [
+ 'id' => [0],
+ 'name' => [0],
+ 'score' => [0],
+ 'd92' => [0],
+ 'd182' => [0],
+ 'day' => [0],
+ 'ts_us' => [0],
+ 'ts_ns' => [0],
+ 'uuid' => [0],
+ ],
+ ];
+
+ public const string GLOB = 'memory://parity/*.parquet';
+
+ public static function write(Filesystem $filesystem): void
+ {
+ $options = Options::default()->set(Option::ROW_GROUP_SIZE_BYTES, 1)->set(
+ Option::ROW_GROUP_SIZE_CHECK_INTERVAL,
+ 10,
+ );
+ $columns = [
+ FlatColumn::int64('id'),
+ FlatColumn::string('name'),
+ FlatColumn::double('score'),
+ FlatColumn::decimal('d92', 9, 2),
+ FlatColumn::decimal('d182', 18, 2),
+ FlatColumn::date('day'),
+ FlatColumn::dateTime('ts_us'),
+ new FlatColumn(
+ 'ts_ns',
+ PhysicalType::INT64,
+ logicalType: new LogicalType(
+ LogicalType::TIMESTAMP,
+ timestamp: new Timestamp(true, TimeUnit::NANOSECONDS),
+ ),
+ ),
+ FlatColumn::uuid('uuid'),
+ ];
+ $row = static fn(int $id): array => [
+ 'id' => $id,
+ 'name' => (string) $id,
+ 'score' => $id < 10 ? NAN : $id * 1.5,
+ 'd92' => $id + 0.25,
+ 'd182' => $id + 0.25,
+ 'day' => (new DateTimeImmutable('2024-01-01'))->modify(sprintf('+%d days', $id)),
+ 'ts_us' => (new DateTimeImmutable('2024-01-01 00:00:00.123456'))->modify(sprintf('+%d hours', $id)),
+ 'ts_ns' => (new DateTimeImmutable('2024-01-01 00:00:00.123456'))->modify(sprintf('+%d hours', $id)),
+ 'uuid' => sprintf('00000000-0000-4000-8000-%012d', $id),
+ ];
+
+ foreach ([
+ 'a' => [
+ [...$columns, FlatColumn::string('extra')],
+ static fn(int $id): array => [...$row($id), 'extra' => 'x'],
+ ],
+ 'b' => [$columns, $row],
+ ] as $file => [$schema, $values]) {
+ $writer = new Writer(options: $options, engine: new PhpParquetEngine(options: $options));
+ $writer->openForStream(
+ $filesystem->writeTo(path(sprintf('memory://parity/%s.parquet', $file))),
+ Schema::with(...$schema),
+ );
+ $writer->writeBatch(array_map($values, $file === 'a' ? range(0, 29) : range(30, 39)));
+ $writer->close();
+ }
+ }
+}
diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorFooterTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorFooterTest.php
index 8a52068fe0..813011eb62 100644
--- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorFooterTest.php
+++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorFooterTest.php
@@ -6,6 +6,7 @@
use DateTimeImmutable;
use Flow\ETL\Adapter\Parquet\Tests\Context\ParquetFilesContext;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Function\Between\Boundary;
use Flow\ETL\Rows;
use Flow\ETL\Tests\Double\CountingFilesystem;
@@ -113,8 +114,11 @@ public function test_a_path_filter_excluding_the_first_file_closes_the_kept_one(
$extractor = from_parquet(path(self::PAGINATION . '/*.parquet'), filesystem: $filesystem);
$extractor->schema();
- $rows = iterator_to_array($extractor->extract(flow_context(), pathFilter: new KeepPaths([path(self::PAGINATION
- . '/02_500.parquet')])), false);
+ $rows = iterator_to_array(
+ $extractor->scan(flow_context(), new FileScan(pathFilter: new KeepPaths([path(self::PAGINATION
+ . '/02_500.parquet')]))),
+ false,
+ );
static::assertSame(500, array_sum(array_map(static fn(Rows $batch): int => $batch->count(), $rows)));
static::assertSame(
diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorTest.php
index b3735eaeba..d94f898d99 100644
--- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorTest.php
+++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorTest.php
@@ -9,6 +9,7 @@
use Flow\ETL\Adapter\Parquet\Tests\Context\ParquetFilesContext;
use Flow\ETL\Cardinality;
use Flow\ETL\Exception\InferredSchemaException;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\Signal;
use Flow\ETL\Plan\Stage;
use Flow\ETL\Schema\Definition;
@@ -21,6 +22,7 @@
use Flow\Parquet\Binary\ByteOrder;
use Flow\Parquet\Engine\PhpParquetEngine;
use Flow\Parquet\Options;
+use Flow\Parquet\ParquetEngine;
use Flow\Parquet\Reader;
use Flow\Types\Value\Json;
use Flow\Types\Value\Uuid;
@@ -43,9 +45,11 @@
use function Flow\ETL\DSL\from_sequence_number;
use function Flow\ETL\DSL\int_schema;
use function Flow\ETL\DSL\list_schema;
+use function Flow\ETL\DSL\lit;
use function Flow\ETL\DSL\map_schema;
use function Flow\ETL\DSL\partition_by;
use function Flow\ETL\DSL\partition_types;
+use function Flow\ETL\DSL\ref;
use function Flow\ETL\DSL\schema;
use function Flow\ETL\DSL\str_schema;
use function Flow\ETL\DSL\structure_schema;
@@ -64,6 +68,37 @@
final class ParquetExtractorTest extends FlowTestCase
{
+ /**
+ * @return Generator
+ */
+ public static function engines(): Generator
+ {
+ yield 'default engine' => [null];
+ yield 'php engine' => [new PhpParquetEngine()];
+ }
+
+ #[DataProvider('engines')]
+ public function test_a_body_predicate_skips_row_groups(?ParquetEngine $engine): void
+ {
+ $memory = memory_filesystem();
+ ParquetFilesContext::rowGroups($memory, 'memory://groups.parquet', 3000, 1000);
+ $filtered = new CountingFilesystem($memory);
+ $unfiltered = new CountingFilesystem($memory);
+ $predicate = ref('id')->greaterThanEqual(lit(2500));
+
+ $rows = df()
+ ->read(from_parquet(path('memory://groups.parquet'), filesystem: $filtered, engine: $engine))
+ ->filter($predicate)
+ ->fetch();
+ $all = df()
+ ->read(from_parquet(path('memory://groups.parquet'), filesystem: $unfiltered, engine: $engine))
+ ->fetch();
+
+ static::assertCount(500, $rows);
+ static::assertSame(df()->read(from_rows($all))->filter($predicate)->fetch()->toArray(), $rows->toArray());
+ static::assertLessThan(ParquetFilesContext::bytesRead($unfiltered), ParquetFilesContext::bytesRead($filtered));
+ }
+
public function test_limit(): void
{
$extractor = from_parquet(path(__DIR__ . '/Fixtures/orders_1k.parquet'));
@@ -447,7 +482,10 @@ public function test_a_path_filter_narrows_the_read_but_not_the_schema(): void
);
$schema = $extractor->schema();
- $batches = iterator_to_array($extractor->extract(flow_context(config()), pathFilter: new RejectingFilter()));
+ $batches = iterator_to_array($extractor->scan(
+ flow_context(config()),
+ new FileScan(pathFilter: new RejectingFilter()),
+ ));
static::assertSame([], $batches);
static::assertSame(1, $filesystem->readFromCalls);
@@ -482,7 +520,7 @@ public function test_limit_reached_on_the_first_file_tail_batch_skips_the_remain
{
$extractor = from_parquet(path(__DIR__ . '/Fixtures/Pagination/*.parquet'))->withBatchSize(1500);
- static::assertCount(1000, ExtractedRows::of($extractor, limit: 1000));
+ static::assertCount(1000, ExtractedRows::of($extractor, scan: new FileScan(1000)));
}
public function test_the_second_file_is_asked_for_the_remainder_of_the_limit(): void
@@ -501,7 +539,7 @@ public function test_limit_reached_on_a_full_batch_of_the_first_file_skips_the_r
{
$extractor = from_parquet(path(__DIR__ . '/Fixtures/Pagination/*.parquet'))->withBatchSize(500);
- static::assertCount(1000, ExtractedRows::of($extractor, limit: 1000));
+ static::assertCount(1000, ExtractedRows::of($extractor, scan: new FileScan(1000)));
}
public function test_is_repeatable(): void
diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/PushdownParityTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/PushdownParityTest.php
new file mode 100644
index 0000000000..f3d8a46374
--- /dev/null
+++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/PushdownParityTest.php
@@ -0,0 +1,160 @@
+
+ */
+ public static function predicates(): Generator
+ {
+ $predicates = [
+ 'string range' => [ref('name')->greaterThan(lit('3')), true],
+ 'NaN column' => [ref('score')->lessThanEqual(lit(20.0)), true],
+ 'decimal(9,2)' => [ref('d92')->greaterThan(lit(15.25)), true],
+ 'decimal(18,2)' => [ref('d182')->greaterThan(lit(15.25)), false],
+ 'date' => [ref('day')->greaterThanEqual(lit(new DateTimeImmutable('2024-01-21'))), true],
+ 'timestamp micros' => [ref('ts_us')->greaterThan(lit(new DateTimeImmutable('2024-01-01 20:30:00'))), true],
+ 'timestamp nanos' => [ref('ts_ns')->greaterThan(lit(new DateTimeImmutable('2024-01-01 20:30:00'))), true],
+ 'uuid' => [ref('uuid')->equals(lit(new Uuid('00000000-0000-4000-8000-000000000025'))), true],
+ 'a negated is in holding null, or an equality' => [
+ not(ref('id')->isIn(lit([1, null])))->or(ref('id')->equals(lit(5))),
+ false,
+ ],
+ 'absent column equals' => [ref('extra')->equals(lit('x')), true],
+ 'absent column is null' => [ref('extra')->isNull(), true],
+ ];
+
+ foreach (['default engine' => null, 'php engine' => new PhpParquetEngine()] as $engineName => $engine) {
+ foreach ($predicates as $name => [$predicate, $prunes]) {
+ yield $engineName . ', ' . $name => [$engine, $predicate, $prunes];
+ }
+ }
+ }
+
+ /**
+ * @param bool $prunes true when the predicate skips a row group, false when it skips none
+ */
+ #[DataProvider('predicates')]
+ public function test_a_pushed_predicate_returns_what_the_unpushed_one_returns(
+ ?ParquetEngine $engine,
+ ScalarFunction $predicate,
+ bool $prunes,
+ ): void {
+ $memory = memory_filesystem();
+ PushdownParityContext::write($memory);
+ $pushedFilesystem = new CountingFilesystem($memory);
+ $plainFilesystem = new CountingFilesystem($memory);
+
+ $pushed = df()
+ ->read(
+ from_parquet(
+ path(PushdownParityContext::GLOB),
+ engine: $engine,
+ filesystem: $pushedFilesystem,
+ )->unionByName(),
+ )
+ ->filter($predicate)
+ ->fetch();
+ $plain = df(config_builder()->optimizer(Optimizer::default()->without(PushFilterIntoSource::class)))
+ ->read(
+ from_parquet(
+ path(PushdownParityContext::GLOB),
+ engine: $engine,
+ filesystem: $plainFilesystem,
+ )->unionByName(),
+ )
+ ->filter($predicate)
+ ->fetch();
+
+ static::assertNotSame([], $plain->reduceToArray('id'));
+ static::assertSame($plain->reduceToArray('id'), $pushed->reduceToArray('id'));
+ // a pushed read adds at most the page indexes it consults to the chunks it reads
+ static::assertLessThanOrEqual(
+ ParquetFilesContext::bytesRead($plainFilesystem)
+ + ParquetFilesContext::pageIndexBytes($memory, PushdownParityContext::CHUNKS)
+ + ParquetFilesContext::offsetIndexBytes($memory, PushdownParityContext::CHUNKS),
+ ParquetFilesContext::bytesRead($pushedFilesystem),
+ );
+
+ if ($prunes) {
+ static::assertLessThan(
+ ParquetFilesContext::bytesRead($plainFilesystem),
+ ParquetFilesContext::bytesRead($pushedFilesystem),
+ );
+ }
+ }
+
+ /**
+ * @return Generator
+ */
+ public static function engines(): Generator
+ {
+ yield 'default engine' => [null];
+ yield 'php engine' => [new PhpParquetEngine()];
+ }
+
+ #[DataProvider('engines')]
+ public function test_a_limit_beside_a_pushed_filter_reads_fewer_bytes_than_the_unlimited_read(?ParquetEngine $engine): void
+ {
+ $memory = memory_filesystem();
+ PushdownParityContext::write($memory);
+ $limitedFilesystem = new CountingFilesystem($memory);
+ $unlimitedFilesystem = new CountingFilesystem($memory);
+
+ $limited = df()
+ ->read(
+ from_parquet(
+ path(PushdownParityContext::GLOB),
+ engine: $engine,
+ filesystem: $limitedFilesystem,
+ )->unionByName(),
+ )
+ ->filter(ref('id')->lessThan(lit(35)))
+ ->limit(10)
+ ->fetch();
+ $unlimited = df()
+ ->read(
+ from_parquet(
+ path(PushdownParityContext::GLOB),
+ engine: $engine,
+ filesystem: $unlimitedFilesystem,
+ )->unionByName(),
+ )
+ ->filter(ref('id')->lessThan(lit(35)))
+ ->fetch();
+
+ static::assertCount(10, $limited);
+ static::assertCount(35, $unlimited);
+ static::assertLessThan(
+ ParquetFilesContext::bytesRead($unlimitedFilesystem),
+ ParquetFilesContext::bytesRead($limitedFilesystem),
+ );
+ }
+}
diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/UnsignedInt64ColumnTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/UnsignedInt64ColumnTest.php
new file mode 100644
index 0000000000..f0339e2ccb
--- /dev/null
+++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/UnsignedInt64ColumnTest.php
@@ -0,0 +1,44 @@
+openForStream(
+ $memory->writeTo($path),
+ Schema::with(new FlatColumn('id', PhysicalType::INT64, logicalType: LogicalType::integer(64, false))),
+ );
+ $writer->writeBatch([['id' => 0], ['id' => '9223372036854775808'], ['id' => '18446744073709551615']]);
+ $writer->close();
+
+ $df = data_frame()->read(from_parquet($path, filesystem: $memory));
+ $rows = $df->fetch();
+
+ static::assertTrue(type_equals(type_numeric_string(), $df->schema()->get('id')->type()));
+ static::assertTrue(type_equals(type_numeric_string(), $rows->schema()->get('id')->type()));
+ static::assertTrue(type_equals(type_numeric_string(), $rows->column('id')->type()));
+ static::assertSame(['0', '9223372036854775808', '18446744073709551615'], $rows->column('id')->values());
+ }
+}
diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Mother/BodySchemaMother.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Mother/BodySchemaMother.php
new file mode 100644
index 0000000000..d452587c64
--- /dev/null
+++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Mother/BodySchemaMother.php
@@ -0,0 +1,40 @@
+batches(
schema(int_schema('id')),
10,
- null,
- null,
+ (new ReadPlanner())->plan($file->reader()->footer(), $file->indexes(), null),
new PhpBackend(),
));
$source->close();
@@ -48,8 +48,7 @@ public function test_with_flow_php_a_rust_engine_file_reads_through_rust(): void
static::assertInstanceOf(RustIterator::class, $source->batches(
schema(int_schema('id')),
10,
- null,
- null,
+ (new ReadPlanner())->plan($file->reader()->footer(), $file->indexes(), null),
new PhpBackend(),
));
$source->close();
@@ -69,8 +68,7 @@ public function test_without_flow_php_a_rust_engine_file_reads_through_php(): vo
static::assertInstanceOf(Generator::class, $source->batches(
schema(int_schema('id')),
10,
- null,
- null,
+ (new ReadPlanner())->plan($file->reader()->footer(), $file->indexes(), null),
new PhpBackend(),
));
$source->close();
diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/FlowToParquetSchemaTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/FlowToParquetSchemaTest.php
index 8fce623443..50295d4b68 100644
--- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/FlowToParquetSchemaTest.php
+++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/FlowToParquetSchemaTest.php
@@ -13,6 +13,7 @@
use function Flow\ETL\DSL\bool_schema;
use function Flow\ETL\DSL\datetime_schema;
+use function Flow\ETL\DSL\definition_from_type;
use function Flow\ETL\DSL\float_schema;
use function Flow\ETL\DSL\integer_schema;
use function Flow\ETL\DSL\json_schema;
@@ -26,6 +27,7 @@
use function Flow\Types\DSL\type_integer;
use function Flow\Types\DSL\type_list;
use function Flow\Types\DSL\type_map;
+use function Flow\Types\DSL\type_numeric_string;
use function Flow\Types\DSL\type_string;
use function Flow\Types\DSL\type_structure;
@@ -122,4 +124,12 @@ public function test_converting_nested_structure_with_optional_elements_throws()
'a' => type_structure(['b' => type_string(), 'c' => structure_element('c', type_string(), optional: true)]),
]))));
}
+
+ public function test_a_numeric_string_column_is_written_as_a_string(): void
+ {
+ static::assertEquals(
+ ParquetSchema::with(FlatColumn::string('id')),
+ (new SchemaConverter())->toParquet(schema(definition_from_type('id', type_numeric_string(), true))),
+ );
+ }
}
diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileBatchesTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileBatchesTest.php
index 2d91a62f77..cc66637925 100644
--- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileBatchesTest.php
+++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileBatchesTest.php
@@ -23,6 +23,8 @@
use function array_map;
use function Flow\ETL\DSL\array_to_rows;
use function Flow\ETL\DSL\int_schema;
+use function Flow\ETL\DSL\lit;
+use function Flow\ETL\DSL\ref;
use function Flow\ETL\DSL\schema;
use function Flow\ETL\DSL\str_schema;
use function Flow\Filesystem\DSL\memory_filesystem;
@@ -45,7 +47,13 @@ public function test_an_offset_that_covers_the_file_skips_it_and_returns_its_row
new SchemaConverter(),
[],
);
- $file = (new ParquetFileBatches($opener, FileColumnsContext::discovering(names: []), null, null))->batches(
+ $file = (new ParquetFileBatches(
+ $opener,
+ FileColumnsContext::discovering(names: []),
+ null,
+ null,
+ schema(int_schema('id', nullable: true)),
+ ))->batches(
new SourceFile(path('memory://a.parquet')),
schema(int_schema('id', nullable: true)),
10,
@@ -72,7 +80,13 @@ public function test_an_offset_inside_the_file_starts_the_read_there_and_is_cons
new SchemaConverter(),
[],
);
- $file = (new ParquetFileBatches($opener, FileColumnsContext::discovering(names: []), null, null))->batches(
+ $file = (new ParquetFileBatches(
+ $opener,
+ FileColumnsContext::discovering(names: []),
+ null,
+ null,
+ schema(int_schema('id', nullable: true)),
+ ))->batches(
new SourceFile(path('memory://a.parquet')),
schema(int_schema('id', nullable: true)),
10,
@@ -108,6 +122,7 @@ public function test_the_kept_first_file_is_read_instead_of_opened_again(): void
FileColumnsContext::discovering(names: []),
$kept,
null,
+ schema(int_schema('id', nullable: true)),
))->batches(
new SourceFile(path('memory://a.parquet')),
schema(int_schema('id', nullable: true)),
@@ -137,7 +152,13 @@ public function test_a_kept_first_file_of_another_path_is_closed_and_the_file_op
$kept = $opener->open(new SourceFile(path('memory://a.parquet')));
$rows = iterator_to_array(
- (new ParquetFileBatches($opener, FileColumnsContext::discovering(names: []), $kept, null))->batches(
+ (new ParquetFileBatches(
+ $opener,
+ FileColumnsContext::discovering(names: []),
+ $kept,
+ null,
+ schema(int_schema('id', nullable: true)),
+ ))->batches(
new SourceFile(path('memory://b.parquet')),
schema(int_schema('id', nullable: true)),
10,
@@ -166,7 +187,13 @@ public function test_close_releases_a_kept_first_file_the_read_never_reached():
);
$kept = $opener->open(new SourceFile(path('memory://a.parquet')));
- (new ParquetFileBatches($opener, FileColumnsContext::discovering(names: []), $kept, null))->close();
+ (new ParquetFileBatches(
+ $opener,
+ FileColumnsContext::discovering(names: []),
+ $kept,
+ null,
+ schema(int_schema('id', nullable: true)),
+ ))->close();
static::assertSame($filesystem->readFromCalls, $filesystem->closedStreams());
}
@@ -191,6 +218,7 @@ public function test_a_file_that_diverges_from_the_derived_schema_is_refused_and
FileColumnsContext::discovering(names: []),
null,
new DerivedSchema(schema(int_schema('id', nullable: true)), 'memory://a.parquet'),
+ schema(int_schema('id')),
))->batches(
new SourceFile(path('memory://b.parquet')),
schema(int_schema('id')),
@@ -203,4 +231,132 @@ public function test_a_file_that_diverges_from_the_derived_schema_is_refused_and
static::assertSame($filesystem->readFromCalls, $filesystem->closedStreams());
}
}
+
+ public function test_a_body_predicate_skips_the_row_groups_it_proves_cannot_match(): void
+ {
+ $filesystem = memory_filesystem();
+ ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 9, 3);
+ $file = (new ParquetFileBatches(
+ ParquetFilesContext::opener($filesystem),
+ FileColumnsContext::discovering(names: []),
+ null,
+ null,
+ schema(int_schema('id', nullable: true), str_schema('name', nullable: true)),
+ [ref('id')->greaterThan(lit(5))],
+ ))->batches(
+ new SourceFile(path('memory://groups.parquet')),
+ schema(int_schema('id', nullable: true), str_schema('name', nullable: true)),
+ 10,
+ new PhpBackend(),
+ new ReadWindow(0),
+ );
+
+ static::assertSame(3, ParquetFilesContext::rowCount($file));
+ }
+
+ public function test_the_conjunct_on_a_column_the_file_stores_as_another_type_is_dropped(): void
+ {
+ $filesystem = memory_filesystem();
+ ParquetFilesContext::unsignedInt64($filesystem, 'memory://uint64.parquet', ['9', '10', '100']);
+ // the body reads the UINT_64 leaf as a string, ordered by bytes: '10' and '100' are below '2', numerically none is
+ $file = (new ParquetFileBatches(
+ ParquetFilesContext::opener($filesystem),
+ FileColumnsContext::discovering(names: []),
+ null,
+ null,
+ schema(str_schema('big', nullable: true)),
+ [ref('big')->lessThan(lit('2'))],
+ ))->batches(
+ new SourceFile(path('memory://uint64.parquet')),
+ schema(str_schema('big', nullable: true)),
+ 10,
+ new PhpBackend(),
+ new ReadWindow(0),
+ );
+
+ static::assertSame(3, ParquetFilesContext::rowCount($file));
+ }
+
+ public function test_equality_on_a_column_absent_from_the_file_reads_no_row(): void
+ {
+ $filesystem = memory_filesystem();
+ ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 9, 3);
+ $file = (new ParquetFileBatches(
+ ParquetFilesContext::opener($filesystem),
+ FileColumnsContext::discovering(names: []),
+ null,
+ null,
+ schema(
+ int_schema('id', nullable: true),
+ str_schema('name', nullable: true),
+ str_schema('missing', nullable: true),
+ ),
+ [ref('missing')->equals(lit('x'))],
+ ))->batches(
+ new SourceFile(path('memory://groups.parquet')),
+ schema(
+ int_schema('id', nullable: true),
+ str_schema('name', nullable: true),
+ str_schema('missing', nullable: true),
+ ),
+ 10,
+ new PhpBackend(),
+ new ReadWindow(0),
+ );
+
+ static::assertSame(0, ParquetFilesContext::rowCount($file));
+ }
+
+ public function test_is_null_on_a_column_absent_from_the_file_reads_every_row(): void
+ {
+ $filesystem = memory_filesystem();
+ ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 9, 3);
+ $file = (new ParquetFileBatches(
+ ParquetFilesContext::opener($filesystem),
+ FileColumnsContext::discovering(names: []),
+ null,
+ null,
+ schema(
+ int_schema('id', nullable: true),
+ str_schema('name', nullable: true),
+ str_schema('missing', nullable: true),
+ ),
+ [ref('missing')->isNull()],
+ ))->batches(
+ new SourceFile(path('memory://groups.parquet')),
+ schema(
+ int_schema('id', nullable: true),
+ str_schema('name', nullable: true),
+ str_schema('missing', nullable: true),
+ ),
+ 10,
+ new PhpBackend(),
+ new ReadWindow(0),
+ );
+
+ static::assertSame(9, ParquetFilesContext::rowCount($file));
+ }
+
+ public function test_every_group_pruned_still_consumes_the_raw_offset(): void
+ {
+ $filesystem = memory_filesystem();
+ ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 9, 3);
+ $file = (new ParquetFileBatches(
+ ParquetFilesContext::opener($filesystem),
+ FileColumnsContext::discovering(names: []),
+ null,
+ null,
+ schema(int_schema('id', nullable: true), str_schema('name', nullable: true)),
+ [ref('id')->greaterThan(lit(100))],
+ ))->batches(
+ new SourceFile(path('memory://groups.parquet')),
+ schema(int_schema('id', nullable: true), str_schema('name', nullable: true)),
+ 10,
+ new PhpBackend(),
+ new ReadWindow(2),
+ );
+
+ static::assertSame(0, ParquetFilesContext::rowCount($file));
+ static::assertSame(2, $file->getReturn());
+ }
}
diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSourceFileTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSourceFileTest.php
index 2ffe2c9b15..f1ed6118bf 100644
--- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSourceFileTest.php
+++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSourceFileTest.php
@@ -10,6 +10,7 @@
use Flow\ETL\Tests\Double\CountingFilesystem;
use Flow\ETL\Tests\FlowTestCase;
use Flow\Filesystem\Local\NativeLocalFilesystem;
+use Flow\Parquet\Reader\ReadPlanner;
use function Flow\ETL\DSL\int_schema;
use function Flow\ETL\DSL\schema;
@@ -32,7 +33,12 @@ public function test_open_reads_the_batches_of_the_file(): void
$source = $file->open();
$counts = [];
- foreach ($source->batches(schema(int_schema('id')), 400, 100, 850, new PhpBackend()) as $rows) {
+ foreach ($source->batches(
+ schema(int_schema('id')),
+ 400,
+ (new ReadPlanner())->plan($file->file->reader()->footer(), $file->file->indexes(), null, 100, 850),
+ new PhpBackend(),
+ ) as $rows) {
$counts[] = $rows->count();
}
diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetToFlowSchemaTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetToFlowSchemaTest.php
index 51d51ae828..62a9647f2a 100644
--- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetToFlowSchemaTest.php
+++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetToFlowSchemaTest.php
@@ -7,11 +7,15 @@
use Flow\ETL\Adapter\Parquet\SchemaConverter;
use Flow\ETL\Tests\FlowTestCase;
use Flow\Parquet\ParquetFile\Schema;
+use Flow\Parquet\ParquetFile\Schema\ConvertedType;
use Flow\Parquet\ParquetFile\Schema\FlatColumn;
use Flow\Parquet\ParquetFile\Schema\ListElement;
+use Flow\Parquet\ParquetFile\Schema\LogicalType;
use Flow\Parquet\ParquetFile\Schema\MapKey;
use Flow\Parquet\ParquetFile\Schema\MapValue;
use Flow\Parquet\ParquetFile\Schema\NestedColumn;
+use Flow\Parquet\ParquetFile\Schema\PhysicalType;
+use Flow\Parquet\ParquetFile\Schema\Repetition;
use function Flow\ETL\DSL\bool_schema;
use function Flow\ETL\DSL\date_schema;
@@ -27,9 +31,11 @@
use function Flow\ETL\DSL\time_schema;
use function Flow\ETL\DSL\uuid_schema;
use function Flow\Types\DSL\type_boolean;
+use function Flow\Types\DSL\type_equals;
use function Flow\Types\DSL\type_integer;
use function Flow\Types\DSL\type_list;
use function Flow\Types\DSL\type_map;
+use function Flow\Types\DSL\type_numeric_string;
use function Flow\Types\DSL\type_optional;
use function Flow\Types\DSL\type_string;
use function Flow\Types\DSL\type_structure;
@@ -123,4 +129,31 @@ public function test_converting_struct_to_flow_schema(): void
$flowSchema,
);
}
+
+ public function test_uint64_columns_are_numeric_strings(): void
+ {
+ $flowSchema = (new SchemaConverter())->toFlow(Schema::with(
+ new FlatColumn('converted', PhysicalType::INT64, ConvertedType::UINT_64, repetition: Repetition::REQUIRED),
+ new FlatColumn('logical', PhysicalType::INT64, logicalType: LogicalType::integer(64, false)),
+ new FlatColumn('uint32', PhysicalType::INT32, ConvertedType::UINT_32),
+ ));
+
+ static::assertTrue(type_equals(type_numeric_string(), $flowSchema->get('converted')->type()));
+ static::assertFalse($flowSchema->get('converted')->isNullable());
+ static::assertTrue(type_equals(type_numeric_string(), $flowSchema->get('logical')->type()));
+ static::assertTrue($flowSchema->get('logical')->isNullable());
+ static::assertTrue(type_equals(type_integer(), $flowSchema->get('uint32')->type()));
+ static::assertTrue(type_equals(
+ type_optional(type_numeric_string()),
+ (new SchemaConverter())->parquetToFlowType(
+ new FlatColumn('u', PhysicalType::INT64, logicalType: LogicalType::integer(64, false)),
+ ),
+ ));
+ static::assertTrue(type_equals(
+ type_numeric_string(),
+ (new SchemaConverter())->parquetToFlowType(
+ new FlatColumn('u', PhysicalType::INT64, ConvertedType::UINT_64, repetition: Repetition::REQUIRED),
+ ),
+ ));
+ }
}
diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/PhpParquetOpenSourceTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/PhpParquetOpenSourceTest.php
index 61e9591630..209be0be23 100644
--- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/PhpParquetOpenSourceTest.php
+++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/PhpParquetOpenSourceTest.php
@@ -9,6 +9,7 @@
use Flow\ETL\Column\PhpBackend;
use Flow\ETL\Tests\FlowTestCase;
use Flow\Parquet\Engine\PhpParquetEngine;
+use Flow\Parquet\Exception\InvalidArgumentException;
use Flow\Parquet\Reader;
use PHPUnit\Framework\Attributes\TestWith;
@@ -18,6 +19,7 @@
use function Flow\ETL\DSL\str_schema;
use function Flow\Filesystem\DSL\memory_filesystem;
use function Flow\Filesystem\DSL\path;
+use function iterator_to_array;
use function range;
final class PhpParquetOpenSourceTest extends FlowTestCase
@@ -25,11 +27,11 @@ final class PhpParquetOpenSourceTest extends FlowTestCase
/**
* @param list $ids
*/
- #[TestWith([null, null, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]])]
+ #[TestWith([0, null, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]])]
#[TestWith([3, 5, [3, 4, 5, 6, 7]])]
#[TestWith([6, null, [6, 7, 8, 9]])]
#[TestWith([0, 1, [0]])]
- public function test_offset_and_limit_cross_row_groups(?int $offset, ?int $limit, array $ids): void
+ public function test_a_planned_window_crosses_row_groups(int $offset, ?int $limit, array $ids): void
{
$filesystem = memory_filesystem();
ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 10, 4);
@@ -41,8 +43,7 @@ public function test_offset_and_limit_cross_row_groups(?int $offset, ?int $limit
))->readStream($filesystem->readFrom(path('memory://groups.parquet')))))->batches(
schema(int_schema('id')),
3,
- $offset,
- $limit,
+ ParquetFilesContext::plan($filesystem, 'memory://groups.parquet', $offset, $limit),
new PhpBackend(),
) as $rows) {
static::assertLessThanOrEqual(3, $rows->count());
@@ -62,7 +63,12 @@ public function test_batches_hold_the_schema_columns(): void
$names = [];
- foreach ($source->batches(schema(str_schema('name')), 100, null, null, new PhpBackend()) as $rows) {
+ foreach ($source->batches(
+ schema(str_schema('name')),
+ 100,
+ ParquetFilesContext::plan($filesystem, 'memory://groups.parquet'),
+ new PhpBackend(),
+ ) as $rows) {
static::assertEquals(schema(str_schema('name')), $rows->schema());
$names = [...$names, ...$rows->column('name')->values()];
}
@@ -71,4 +77,40 @@ public function test_batches_hold_the_schema_columns(): void
static::assertSame(array_map(static fn(int $id): string => 'n' . $id, range(0, 9)), $names);
}
+
+ public function test_a_column_the_file_lacks_is_refused(): void
+ {
+ $filesystem = memory_filesystem();
+ ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 10, 4);
+
+ $this->expectException(InvalidArgumentException::class);
+ $this->expectExceptionMessage('Parquet file has no column "missing"');
+
+ iterator_to_array((new PhpParquetOpenSource((new Reader(
+ engine: new PhpParquetEngine(),
+ ))->readStream($filesystem->readFrom(path('memory://groups.parquet')))))->batches(
+ schema(int_schema('missing')),
+ 10,
+ ParquetFilesContext::plan($filesystem, 'memory://groups.parquet'),
+ new PhpBackend(),
+ ));
+ }
+
+ public function test_a_batch_size_below_one_is_refused(): void
+ {
+ $filesystem = memory_filesystem();
+ ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 10, 4);
+
+ $this->expectException(InvalidArgumentException::class);
+ $this->expectExceptionMessage('Parquet batch size must be greater than 0');
+
+ iterator_to_array((new PhpParquetOpenSource((new Reader(
+ engine: new PhpParquetEngine(),
+ ))->readStream($filesystem->readFrom(path('memory://groups.parquet')))))->batches(
+ schema(int_schema('id')),
+ 0,
+ ParquetFilesContext::plan($filesystem, 'memory://groups.parquet'),
+ new PhpBackend(),
+ ));
+ }
}
diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/Pushdown/FilterTranslatorTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/Pushdown/FilterTranslatorTest.php
new file mode 100644
index 0000000000..b918586c4f
--- /dev/null
+++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/Pushdown/FilterTranslatorTest.php
@@ -0,0 +1,195 @@
+ [ref('id')->equals(lit(5)), parquet_column('id')->equals(5)];
+ yield 'not equals' => [ref('id')->notEquals(lit(5)), parquet_column('id')->notEquals(5)];
+ yield 'less than' => [ref('id')->lessThan(lit(5)), parquet_column('id')->lessThan(5)];
+ yield 'less than or equal' => [ref('id')->lessThanEqual(lit(5)), parquet_column('id')->lessThanOrEqual(5)];
+ yield 'greater than' => [ref('id')->greaterThan(lit(5)), parquet_column('id')->greaterThan(5)];
+ yield 'greater than or equal' => [
+ ref('id')->greaterThanEqual(lit(5)),
+ parquet_column('id')->greaterThanOrEqual(5),
+ ];
+ yield 'a literal on the left is flipped' => [lit(5)->lessThan(ref('id')), parquet_column('id')->greaterThan(5)];
+ yield 'a literal on the left of an equality' => [lit(5)->equals(ref('id')), parquet_column('id')->equals(5)];
+ yield 'same on the same type' => [ref('id')->same(lit(5)), parquet_column('id')->equals(5)];
+ yield 'not same on the same type' => [ref('id')->notSame(lit(5)), parquet_column('id')->notEquals(5)];
+ yield 'is true' => [ref('flag')->isTrue(), parquet_column('flag')->equals(true)];
+ yield 'is null' => [ref('id')->isNull(), parquet_column('id')->isNull()];
+ yield 'is not null' => [ref('id')->isNotNull(), parquet_column('id')->isNotNull()];
+ yield 'not' => [not(ref('id')->equals(lit(5))), parquet_column('id')->equals(5)->negate()];
+ yield 'an or' => [
+ ref('id')->equals(lit(5))->or(ref('name')->equals(lit('a'))),
+ parquet_column('id')->equals(5)->or(parquet_column('name')->equals('a')),
+ ];
+ yield 'an and inside an or' => [
+ ref('id')->equals(lit(5))->or(ref('id')->greaterThan(lit(1))->and(ref('name')->equals(lit('a')))),
+ parquet_column('id')->equals(5)->or(
+ parquet_column('id')->greaterThan(1)->and(parquet_column('name')->equals('a')),
+ ),
+ ];
+ yield 'is in' => [ref('id')->isIn(lit([1, 2])), parquet_column('id')->in([1, 2])];
+ yield 'is in drops null elements' => [ref('id')->isIn(lit([1, null, 2])), parquet_column('id')->in([1, 2])];
+ yield 'a negated is in without null' => [
+ not(ref('id')->isIn(lit([1, 2]))),
+ parquet_column('id')->in([1, 2])->negate(),
+ ];
+ yield 'a negated is in holding null drops the null' => [
+ not(ref('id')->isIn(lit([1, null]))),
+ parquet_column('id')->in([1])->negate(),
+ ];
+ yield 'starts with' => [ref('name')->startsWith(lit('ab')), parquet_column('name')->startsWith('ab')];
+ yield 'between left inclusive' => [
+ ref('id')->between(lit(1), lit(5)),
+ parquet_column('id')->greaterThanOrEqual(1)->and(parquet_column('id')->lessThan(5)),
+ ];
+ yield 'between inclusive' => [
+ ref('id')->between(lit(1), lit(5), lit(Boundary::INCLUSIVE)),
+ parquet_column('id')->greaterThanOrEqual(1)->and(parquet_column('id')->lessThanOrEqual(5)),
+ ];
+ yield 'between right inclusive' => [
+ ref('id')->between(lit(1), lit(5), lit(Boundary::RIGHT_INCLUSIVE)),
+ parquet_column('id')->greaterThan(1)->and(parquet_column('id')->lessThanOrEqual(5)),
+ ];
+ yield 'between exclusive' => [
+ ref('id')->between(lit(1), lit(5), lit(Boundary::EXCLUSIVE)),
+ parquet_column('id')->greaterThan(1)->and(parquet_column('id')->lessThan(5)),
+ ];
+ yield 'an int literal on a float column becomes a float' => [
+ ref('price')->greaterThan(lit(10)),
+ parquet_column('price')->greaterThan(10.0),
+ ];
+ yield 'an int literal on a float column at 2^53' => [
+ ref('price')->equals(lit(9_007_199_254_740_992)),
+ parquet_column('price')->equals(9_007_199_254_740_992.0),
+ ];
+ yield 'an integral float literal on an int column becomes an int' => [
+ ref('id')->lessThan(lit(5.0)),
+ parquet_column('id')->lessThan(5),
+ ];
+ yield 'a float literal on a float column' => [
+ ref('price')->lessThan(lit(1.5)),
+ parquet_column('price')->lessThan(1.5),
+ ];
+ yield 'strings' => [ref('name')->greaterThan(lit('b')), parquet_column('name')->greaterThan('b')];
+ yield 'a canonical digit string on a numeric string column' => [
+ ref('big')->greaterThan(lit('18446744073709551614')),
+ parquet_column('big')->greaterThan('18446744073709551614'),
+ ];
+ yield 'zero on a numeric string column' => [ref('big')->equals(lit('0')), parquet_column('big')->equals('0')];
+ yield 'a uuid literal becomes its string form' => [
+ ref('uuid')->equals(lit(new Uuid('6c2f6e9e-4ec0-4ef7-9b2e-7f2d3f2f6b11'))),
+ parquet_column('uuid')->equals('6c2f6e9e-4ec0-4ef7-9b2e-7f2d3f2f6b11'),
+ ];
+ yield 'a datetime literal passes through' => [
+ ref('ts')->greaterThan(lit(new DateTimeImmutable('2024-01-01 12:00:00'))),
+ parquet_column('ts')->greaterThan(new DateTimeImmutable('2024-01-01 12:00:00')),
+ ];
+ yield 'a date literal passes through' => [
+ ref('day')->lessThan(lit(new DateTimeImmutable('2024-01-01'))),
+ parquet_column('day')->lessThan(new DateTimeImmutable('2024-01-01')),
+ ];
+ yield 'a date literal on a datetime column' => [
+ ref('ts')->greaterThanEqual(lit(new DateTimeImmutable('2024-01-01'))),
+ parquet_column('ts')->greaterThanOrEqual(new DateTimeImmutable('2024-01-01')),
+ ];
+ yield 'a datetime literal on a date column' => [
+ ref('day')->lessThan(lit(new DateTimeImmutable('2024-01-01 12:00:00'))),
+ parquet_column('day')->lessThan(new DateTimeImmutable('2024-01-01 12:00:00')),
+ ];
+ yield 'json is null' => [ref('doc')->isNull(), parquet_column('doc')->isNull()];
+ }
+
+ #[DataProvider('translated')]
+ public function test_a_pushable_conjunct_is_translated(ScalarFunction $function, Filter $expected): void
+ {
+ $translated = (new FilterTranslator())->translate([$function], BodySchemaMother::everyLeafType());
+
+ static::assertCount(1, $translated);
+ static::assertEquals($expected, $translated[0]->filter);
+ }
+
+ public static function refused(): Generator
+ {
+ yield 'a column outside the body' => [ref('year')->equals(lit(2024))];
+ yield 'two references' => [ref('id')->equals(ref('price'))];
+ yield 'two literals' => [lit(1)->equals(lit(1))];
+ yield 'an unsupported function' => [ref('name')->endsWith(lit('a'))];
+ yield 'an unsupported function inside a not' => [not(ref('name')->endsWith(lit('a')))];
+ yield 'an unsupported function inside an or' => [ref('id')->equals(lit(5))->or(ref('name')->endsWith(lit(
+ 'a',
+ )))];
+ yield 'is in over a reference haystack' => [ref('id')->isIn(ref('ids'))];
+ yield 'is in with a refused element' => [ref('id')->isIn(lit([1, 'a']))];
+ yield 'starts with on an int column' => [ref('id')->startsWith(lit('1'))];
+ yield 'between with a refused bound' => [ref('id')->between(lit(1), lit('5'))];
+ yield 'same across int and float' => [ref('id')->same(lit(5.0))];
+ yield 'same on a numeric string column' => [ref('big')->same(lit('5'))];
+ yield 'an int literal on a float column above 2^53' => [ref('price')->equals(lit(9_007_199_254_740_993))];
+ yield 'a fractional float literal on an int column' => [ref('id')->lessThan(lit(5.5))];
+ yield 'an integral float literal on an int column at 2^53' => [ref('id')->equals(lit(9_007_199_254_740_992.0))];
+ yield 'a NaN literal' => [ref('price')->equals(lit(NAN))];
+ yield 'an int literal on a string column' => [ref('name')->equals(lit(5))];
+ yield 'a string literal on an int column' => [ref('id')->equals(lit('5'))];
+ yield 'a numeric string column with a leading zero literal' => [ref('big')->equals(lit('05'))];
+ yield 'a numeric string column with a non digit literal' => [ref('big')->equals(lit('-5'))];
+ yield 'a numeric string column with an int literal' => [ref('big')->equals(lit(5))];
+ yield 'ordering on a uuid column' => [
+ ref('uuid')->greaterThan(lit(new Uuid('6c2f6e9e-4ec0-4ef7-9b2e-7f2d3f2f6b11'))),
+ ];
+ yield 'equality on a json column' => [ref('doc')->equals(lit('{}'))];
+ yield 'same across date and datetime' => [ref('day')->same(lit(new DateTimeImmutable('2024-01-01 12:00:00')))];
+ }
+
+ #[DataProvider('refused')]
+ public function test_a_conjunct_that_cannot_be_pushed_is_refused(ScalarFunction $function): void
+ {
+ static::assertSame([], (new FilterTranslator())->translate([$function], BodySchemaMother::everyLeafType()));
+ }
+
+ public function test_every_conjunct_is_translated_on_its_own(): void
+ {
+ $translated = (new FilterTranslator())->translate([
+ ref('id')->greaterThan(lit(1)),
+ ref('name')->endsWith(lit('a')),
+ ref('name')->equals(lit('b'))->or(ref('price')->lessThan(lit(2.0))),
+ ], BodySchemaMother::everyLeafType());
+
+ static::assertEquals(
+ [
+ new TranslatedConjunct(parquet_column('id')->greaterThan(1), ['id']),
+ new TranslatedConjunct(
+ parquet_column('name')->equals('b')->or(parquet_column('price')->lessThan(2.0)),
+ ['name', 'price'],
+ ),
+ ],
+ $translated,
+ );
+ }
+}
diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/Pushdown/TranslatedConjunctTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/Pushdown/TranslatedConjunctTest.php
new file mode 100644
index 0000000000..f740e85ec8
--- /dev/null
+++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/Pushdown/TranslatedConjunctTest.php
@@ -0,0 +1,42 @@
+equals(1), ['id', 'name']))->readsAs(
+ schema(int_schema('id'), str_schema('name')),
+ schema(int_schema('id', nullable: true), str_schema('name', nullable: true)),
+ ));
+ }
+
+ public function test_a_column_the_file_stores_as_another_type_is_not_read_as_is(): void
+ {
+ static::assertFalse((new TranslatedConjunct(parquet_column('big')->lessThan('2'), ['id', 'big']))->readsAs(
+ schema(int_schema('id'), definition_from_type('big', type_numeric_string())),
+ schema(int_schema('id'), str_schema('big')),
+ ));
+ }
+
+ public function test_a_column_absent_from_the_file_is_read_as_is(): void
+ {
+ static::assertTrue((new TranslatedConjunct(parquet_column('missing')->isNull(), ['missing']))->readsAs(
+ schema(int_schema('id')),
+ schema(int_schema('id'), str_schema('missing', nullable: true)),
+ ));
+ }
+}
diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/RustParquetOpenSourceTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/RustParquetOpenSourceTest.php
index dd5bfd380a..2293928b33 100644
--- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/RustParquetOpenSourceTest.php
+++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/RustParquetOpenSourceTest.php
@@ -31,11 +31,11 @@ final class RustParquetOpenSourceTest extends FlowTestCase
/**
* @param list $ids
*/
- #[TestWith([null, null, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]])]
+ #[TestWith([0, null, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]])]
#[TestWith([3, 5, [3, 4, 5, 6, 7]])]
#[TestWith([6, null, [6, 7, 8, 9]])]
#[TestWith([0, 1, [0]])]
- public function test_offset_and_limit_cross_row_groups(?int $offset, ?int $limit, array $ids): void
+ public function test_a_planned_window_crosses_row_groups(int $offset, ?int $limit, array $ids): void
{
$filesystem = memory_filesystem();
ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 10, 4);
@@ -44,7 +44,12 @@ public function test_offset_and_limit_cross_row_groups(?int $offset, ?int $limit
foreach ((new RustParquetOpenSource(new RustParquetFileReader($filesystem->readFrom(path(
'memory://groups.parquet',
- )))))->batches(schema(int_schema('id')), 3, $offset, $limit, new AdaptiveBackend()) as $rows) {
+ )))))->batches(
+ schema(int_schema('id')),
+ 3,
+ ParquetFilesContext::plan($filesystem, 'memory://groups.parquet', $offset, $limit),
+ new AdaptiveBackend(),
+ ) as $rows) {
static::assertLessThanOrEqual(3, $rows->count());
$read = [...$read, ...$rows->column('id')->values()];
}
@@ -62,7 +67,12 @@ public function test_columns_are_adopted_by_the_backend(): void
$names = [];
- foreach ($source->batches(schema(str_schema('name')), 100, null, null, new PhpBackend()) as $rows) {
+ foreach ($source->batches(
+ schema(str_schema('name')),
+ 100,
+ ParquetFilesContext::plan($filesystem, 'memory://groups.parquet'),
+ new PhpBackend(),
+ ) as $rows) {
static::assertEquals(schema(str_schema('name')), $rows->schema());
static::assertInstanceOf(ScalarColumn::class, $rows->column('name'));
$names = [...$names, ...$rows->column('name')->values()];
@@ -80,13 +90,23 @@ public function test_a_second_batches_call_closes_the_first_reader(): void
$source = new RustParquetOpenSource(new RustParquetFileReader($filesystem->readFrom(path(
'memory://groups.parquet',
))));
- $first = $source->batches(schema(int_schema('id')), 3, null, null, new AdaptiveBackend());
+ $first = $source->batches(
+ schema(int_schema('id')),
+ 3,
+ ParquetFilesContext::plan($filesystem, 'memory://groups.parquet'),
+ new AdaptiveBackend(),
+ );
$first->rewind();
static::assertSame([0, 1, 2], $first->current()->column('id')->values());
$second = [];
- foreach ($source->batches(schema(int_schema('id')), 3, null, null, new AdaptiveBackend()) as $rows) {
+ foreach ($source->batches(
+ schema(int_schema('id')),
+ 3,
+ ParquetFilesContext::plan($filesystem, 'memory://groups.parquet'),
+ new AdaptiveBackend(),
+ ) as $rows) {
$second = [...$second, ...$rows->column('id')->values()];
}
diff --git a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlCursorExtractorTest.php b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlCursorExtractorTest.php
index b1f9eb39c7..36c4df399e 100644
--- a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlCursorExtractorTest.php
+++ b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlCursorExtractorTest.php
@@ -11,6 +11,7 @@
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\Exception\SchemaMismatchException;
use Flow\ETL\Exception\SchemaNotDerivableException;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Tests\Double\SpyBackend;
use Flow\ETL\Tests\FlowTestCase;
use Flow\PostgreSql\AST\Transformers\ExplainConfig;
@@ -322,7 +323,7 @@ public function test_pushed_limit_issues_no_query_once_satisfied(): void
);
$extractor = from_pgsql_cursor($client, 'SELECT id FROM t');
- self::assertExtractedRowsCount(1500, $extractor, limit: 1500);
+ self::assertExtractedRowsCount(1500, $extractor, scan: new FileScan(1500));
// FETCH 1000, then FETCH 500 - a full narrowed fetch, which only the limit itself can stop
static::assertSame(2, $client->callsTo('cursor'));
}
diff --git a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlKeySetExtractorTest.php b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlKeySetExtractorTest.php
index 598840c54b..511bcf23e1 100644
--- a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlKeySetExtractorTest.php
+++ b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlKeySetExtractorTest.php
@@ -14,6 +14,7 @@
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\Exception\RuntimeException;
use Flow\ETL\Exception\SchemaNotDerivableException;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Tests\Double\SpyBackend;
use Flow\ETL\Tests\FlowTestCase;
use Flow\PostgreSql\AST\Transformers\ExplainConfig;
@@ -387,7 +388,7 @@ public function test_pushed_limit_issues_no_query_once_satisfied(): void
pgsql_pagination_key_set(pgsql_pagination_key_asc('id')),
);
- self::assertExtractedRowsCount(1500, $extractor, limit: 1500);
+ self::assertExtractedRowsCount(1500, $extractor, scan: new FileScan(1500));
// 1000, then a page narrowed to the 500 still wanted - and no third query for row 1501
static::assertSame(2, $client->callsTo('cursor'));
}
diff --git a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLimitOffsetExtractorTest.php b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLimitOffsetExtractorTest.php
index 17b3e1d473..e38cf78f8b 100644
--- a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLimitOffsetExtractorTest.php
+++ b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLimitOffsetExtractorTest.php
@@ -10,6 +10,7 @@
use Flow\ETL\Cardinality;
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\Exception\SchemaNotDerivableException;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Rows;
use Flow\ETL\Tests\Double\SpyBackend;
use Flow\ETL\Tests\FlowTestCase;
@@ -303,7 +304,7 @@ public function test_pushed_limit_skips_the_count_query(): void
->willReturnCursors(new StubCursor([['id' => '1'], ['id' => '2']]), new StubCursor([['id' => '3']]));
$extractor = from_pgsql_limit_offset($client, 'SELECT id FROM t ORDER BY id')->withBatchSize(2);
- self::assertExtractedRowsCount(3, $extractor, limit: 3);
+ self::assertExtractedRowsCount(3, $extractor, scan: new FileScan(3));
// the count query is the only caller of fetchScalarInt()
static::assertSame(0, $client->callsTo('fetchScalarInt'));
}
diff --git a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextExtractor.php b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextExtractor.php
index 6bbb6fbcba..c404495d37 100644
--- a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextExtractor.php
+++ b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextExtractor.php
@@ -11,9 +11,11 @@
use Flow\ETL\Extractor\File\FileExtractor;
use Flow\ETL\Extractor\File\FileReading;
use Flow\ETL\Extractor\File\FileReadLoop;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\File\ListedFiles;
use Flow\ETL\Extractor\File\MetadataColumnsExtractor;
use Flow\ETL\Extractor\File\ReadWindow;
+use Flow\ETL\Extractor\File\Scanning;
use Flow\ETL\Extractor\RewindableExtractor;
use Flow\ETL\Extractor\Signal;
use Flow\ETL\Extractor\Statistics;
@@ -23,8 +25,6 @@
use Flow\Filesystem\Filesystem;
use Flow\Filesystem\Local\NativeLocalFilesystem;
use Flow\Filesystem\Path;
-use Flow\Filesystem\Path\Filter;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
use Generator;
use function Flow\ETL\DSL\schema;
@@ -40,6 +40,8 @@ final class TextExtractor implements
MetadataColumnsExtractor,
RewindableExtractor
{
+ use Scanning;
+
private ?Schema $schema = null;
use Batches;
@@ -74,19 +76,20 @@ public function isRepeatable(): bool
/**
* @return Generator
*/
- public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator
+ public function scan(FlowContext $context, FileScan $scan): Generator
{
$fileColumns = $this->fileColumns($this->filesystem, $this->path);
yield from (new FileReadLoop(
$fileColumns,
$fileColumns->declare($this->schema ?? schema(str_schema('text'))),
+ $scan->conjuncts,
))->read(
- iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $pathFilter), false),
+ iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $scan->pathFilter), false),
new TextFileBatches($this->filesystem),
$this->batchSize(),
- $context->backend(),
- new ReadWindow(limit: $limit),
+ $context,
+ new ReadWindow(limit: $scan->limit),
);
}
diff --git a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTest.php b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTest.php
index 66513a4c61..c8c7f26f4a 100644
--- a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTest.php
+++ b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTest.php
@@ -8,6 +8,7 @@
use Flow\ETL\Adapter\Text\TextExtractor;
use Flow\ETL\Cardinality;
use Flow\ETL\Exception\InvalidArgumentException;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\Signal;
use Flow\ETL\Tests\Context\ExtractedRows;
use Flow\ETL\Tests\Context\MemoryFiles;
@@ -56,7 +57,7 @@ public function test_limit(): void
$extractor = from_text(path_real(__DIR__ . '/../Fixtures/orders_flow.csv'));
$extractor->withBatchSize(1);
- self::assertExtractedRowsCount(2, $extractor, flow_context(config()), limit: 2);
+ self::assertExtractedRowsCount(2, $extractor, flow_context(config()), scan: new FileScan(2));
}
public function test_partition_columns_are_not_leaking_between_streams(): void
@@ -116,7 +117,7 @@ public function test_limit_reached_on_the_first_file_tail_batch_skips_the_remain
{
$extractor = from_text(__DIR__ . '/../Fixtures/cross_stream/*/data.txt')->withBatchSize(10);
- static::assertCount(1, ExtractedRows::of($extractor, limit: 1));
+ static::assertCount(1, ExtractedRows::of($extractor, scan: new FileScan(1)));
}
public function test_metadata_columns_extend_a_declared_schema(): void
diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLParserExtractor.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLParserExtractor.php
index 08ac7ea654..36eb7d0254 100644
--- a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLParserExtractor.php
+++ b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLParserExtractor.php
@@ -11,9 +11,11 @@
use Flow\ETL\Extractor\File\FileExtractor;
use Flow\ETL\Extractor\File\FileReading;
use Flow\ETL\Extractor\File\FileReadLoop;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\File\ListedFiles;
use Flow\ETL\Extractor\File\MetadataColumnsExtractor;
use Flow\ETL\Extractor\File\ReadWindow;
+use Flow\ETL\Extractor\File\Scanning;
use Flow\ETL\Extractor\RewindableExtractor;
use Flow\ETL\Extractor\Signal;
use Flow\ETL\Extractor\Statistics;
@@ -23,8 +25,6 @@
use Flow\Filesystem\Filesystem;
use Flow\Filesystem\Local\NativeLocalFilesystem;
use Flow\Filesystem\Path;
-use Flow\Filesystem\Path\Filter;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
use Generator;
use function Flow\ETL\DSL\schema;
@@ -42,6 +42,7 @@ final class XMLParserExtractor implements
{
use Batches;
use FileReading;
+ use Scanning;
/**
* @var int<1, max>
@@ -96,19 +97,20 @@ public function isRepeatable(): bool
/**
* @return Generator
*/
- public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator
+ public function scan(FlowContext $context, FileScan $scan): Generator
{
$fileColumns = $this->fileColumns($this->filesystem, $this->path);
yield from (new FileReadLoop(
$fileColumns,
$fileColumns->declare($this->schema ?? schema(xml_schema('node'))),
+ $scan->conjuncts,
))->read(
- iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $pathFilter), false),
+ iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $scan->pathFilter), false),
new XMLFileBatches($this->filesystem, $this->xmlNodePath, $this->bufferSize),
$this->batchSize(),
- $context->backend(),
- new ReadWindow(limit: $limit),
+ $context,
+ new ReadWindow(limit: $scan->limit),
);
}
diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLParserExtractorTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLParserExtractorTest.php
index f61c618cf1..70dbfd67ba 100644
--- a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLParserExtractorTest.php
+++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLParserExtractorTest.php
@@ -11,6 +11,7 @@
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\Exception\RuntimeException;
use Flow\ETL\Exception\SchemaMismatchException;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\Signal;
use Flow\ETL\Tests\Context\ExtractedRows;
use Flow\ETL\Tests\Context\MemoryFiles;
@@ -386,7 +387,7 @@ public function test_limit_reached_on_the_first_file_tail_batch_skips_the_remain
{
$extractor = from_xml(__DIR__ . '/../Fixtures/cross_stream/*/file.xml', 'root/item')->withBatchSize(10);
- static::assertCount(1, ExtractedRows::of($extractor, limit: 1));
+ static::assertCount(1, ExtractedRows::of($extractor, scan: new FileScan(1)));
}
public function test_is_repeatable(): void
diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/PhysicalFor.php b/src/core/etl/src/Flow/ETL/Column/Physical/PhysicalFor.php
index ccd58a6b90..600f942940 100644
--- a/src/core/etl/src/Flow/ETL/Column/Physical/PhysicalFor.php
+++ b/src/core/etl/src/Flow/ETL/Column/Physical/PhysicalFor.php
@@ -120,7 +120,6 @@ public function supports(Type $type): bool
$base instanceof EmptyArrayType,
$base instanceof PositiveIntegerType,
$base instanceof NonEmptyStringType,
- $base instanceof NumericStringType,
$base instanceof ClassStringType,
=> false,
default => $this->supportsChild($base),
diff --git a/src/core/etl/src/Flow/ETL/DSL/functions.php b/src/core/etl/src/Flow/ETL/DSL/functions.php
index 6212068b75..18326aeee3 100644
--- a/src/core/etl/src/Flow/ETL/DSL/functions.php
+++ b/src/core/etl/src/Flow/ETL/DSL/functions.php
@@ -226,6 +226,7 @@
use Flow\Types\Type\Logical\JsonType;
use Flow\Types\Type\Logical\ListType;
use Flow\Types\Type\Logical\MapType;
+use Flow\Types\Type\Logical\NumericStringType;
use Flow\Types\Type\Logical\OptionalType;
use Flow\Types\Type\Logical\StructureType;
use Flow\Types\Type\Logical\TimeType;
@@ -1620,6 +1621,7 @@ function definition_from_type(
$type instanceof IntegerType => new IntegerDefinition($ref, $nullable, $metadata),
$type instanceof FloatType => new FloatDefinition($ref, $nullable, $metadata),
$type instanceof StringType => new StringDefinition($ref, $nullable, $metadata),
+ $type instanceof NumericStringType => new StringDefinition($ref, $nullable, $metadata, $type),
$type instanceof DateType => new DateDefinition($ref, $nullable, $metadata),
$type instanceof DateTimeType => new DateTimeDefinition($ref, $type, $nullable, $metadata),
$type instanceof TimeType => new TimeDefinition($ref, $nullable, $metadata),
diff --git a/src/core/etl/src/Flow/ETL/Executor.php b/src/core/etl/src/Flow/ETL/Executor.php
index 60b6e75af9..2a2988812b 100644
--- a/src/core/etl/src/Flow/ETL/Executor.php
+++ b/src/core/etl/src/Flow/ETL/Executor.php
@@ -10,7 +10,6 @@
use Flow\ETL\Executor\PhysicalPlan;
use Flow\ETL\Executor\Pipeline;
use Flow\ETL\Extractor\File\FileExtractor;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
use Generator;
use Throwable;
use WeakMap;
@@ -142,19 +141,16 @@ public function executePipeline(Pipeline $pipeline): Generator
'pipeline #%d',
$leaf->id,
));
+ $scan = $leaf->scan();
$generator = (new AdoptedBatches($leaf->context()->backend()))->of(
$source instanceof FileExtractor
- ? $source->extract($leaf->context(), $leaf->limit(), $leaf->pathFilter())
- : $source->extract($leaf->context(), $leaf->limit()),
+ ? $source->scan($leaf->context(), $scan)
+ : $source->extract($leaf->context(), $scan->limit),
);
$sources = $leaf->sources();
if ($sources !== null) {
- $generator = $sources->count(
- $source,
- $generator,
- $leaf->limit() !== null || !$leaf->pathFilter() instanceof OnlyFiles,
- );
+ $generator = $sources->count($source, $generator, $scan->narrows());
}
foreach ($chain as $stage) {
diff --git a/src/core/etl/src/Flow/ETL/Executor/Pipeline.php b/src/core/etl/src/Flow/ETL/Executor/Pipeline.php
index 534927d41b..f5b5bce309 100644
--- a/src/core/etl/src/Flow/ETL/Executor/Pipeline.php
+++ b/src/core/etl/src/Flow/ETL/Executor/Pipeline.php
@@ -4,9 +4,8 @@
namespace Flow\ETL\Executor;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\FlowContext;
-use Flow\Filesystem\Path\Filter;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
final readonly class Pipeline
{
@@ -18,10 +17,8 @@
* @param FlowContext $context the frame this pipeline was planned in
* @param null|self $input the pipeline whose output is this one's rows; null when this pipeline
* reads the extractor in $segments
- * @param null|int<1, max> $limit handed to the source extractor when this pipeline reads it; meaningless
- * behind an input edge
- * @param Filter $pathFilter handed to a FileExtractor source when this pipeline reads it; meaningless
- * behind an input edge
+ * @param FileScan $scan handed to a FileExtractor source (only its limit to any other source) when this
+ * pipeline reads it; meaningless behind an input edge
* @param null|SourceRows $sources counts the rows of the source when this pipeline reads it and the run is
* analyzed with source statistics; null otherwise
*/
@@ -30,8 +27,7 @@ public function __construct(
private Segments $segments,
private FlowContext $context,
private ?self $input = null,
- private ?int $limit = null,
- private Filter $pathFilter = new OnlyFiles(),
+ private FileScan $scan = new FileScan(),
private ?SourceRows $sources = null,
) {}
@@ -45,17 +41,9 @@ public function context(): FlowContext
return $this->context;
}
- /**
- * @return null|int<1, max>
- */
- public function limit(): ?int
- {
- return $this->limit;
- }
-
- public function pathFilter(): Filter
+ public function scan(): FileScan
{
- return $this->pathFilter;
+ return $this->scan;
}
public function sources(): ?SourceRows
diff --git a/src/core/etl/src/Flow/ETL/Executor/SourceRead.php b/src/core/etl/src/Flow/ETL/Executor/SourceRead.php
index 602a908493..ea2572bb3a 100644
--- a/src/core/etl/src/Flow/ETL/Executor/SourceRead.php
+++ b/src/core/etl/src/Flow/ETL/Executor/SourceRead.php
@@ -34,7 +34,7 @@ public function rows(): int
}
/**
- * Every read ran to its end and none had a limit or a partition filter pushed into it.
+ * Every read ran to its end and none had a limit, a partition filter or a predicate pushed into it.
*/
public function isComplete(): bool
{
diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/FileBatches.php b/src/core/etl/src/Flow/ETL/Extractor/File/FileBatches.php
index 6e33736842..fb793345c3 100644
--- a/src/core/etl/src/Flow/ETL/Extractor/File/FileBatches.php
+++ b/src/core/etl/src/Flow/ETL/Extractor/File/FileBatches.php
@@ -18,7 +18,9 @@ interface FileBatches
* The return value is the rows of the window's offset this file consumed. A format that cannot skip rows is given
* offset 0 and a limit that counts the offset still to skip, and returns 0: FileReadLoop slices the offset from its
* rows. An OffsetSkippingFileBatches is given the offset and returns the rows it skipped. The limit is a hint a
- * format may exceed - FileReadLoop trims what goes past it.
+ * format may exceed - FileReadLoop trims what goes past it - and null when the read carries conjuncts, whose rows
+ * FileReadLoop counts. FileReadLoop applies the conjuncts to every row returned; a format given them when built may
+ * skip what they prove cannot match.
*
* A file's own resources are released by the generator (finally). Resources that span files - a kept first file,
* an inference sample - belong to the per-read object; the extractor that built it calls its close() in a finally
diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/FileExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/File/FileExtractor.php
index f8b5369cc0..f874f6b49c 100644
--- a/src/core/etl/src/Flow/ETL/Extractor/File/FileExtractor.php
+++ b/src/core/etl/src/Flow/ETL/Extractor/File/FileExtractor.php
@@ -10,8 +10,6 @@
use Flow\ETL\Rows;
use Flow\ETL\Schema;
use Flow\Filesystem\Path;
-use Flow\Filesystem\Path\Filter;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
use Generator;
/**
@@ -21,13 +19,13 @@
interface FileExtractor extends Extractor
{
/**
- * @param null|int<1, max> $limit see Extractor::extract()
- * @param Filter $pathFilter what this read lists. The Filter step stays in the plan, so a source that lists
- * more, or ignores it, is still correct.
+ * Reads what the plan pushed into this read and returns only rows matching every one of $scan->conjuncts. The
+ * limit and the path filter are hints: the plan still enforces the limit, and every path filter is also one of
+ * the conjuncts, so a source that reads or lists more is still correct. extract() is scan() with only a limit.
*
* @return Generator
*/
- public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator;
+ public function scan(FlowContext $context, FileScan $scan): Generator;
public function source(): Path;
diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/FileMetadataRow.php b/src/core/etl/src/Flow/ETL/Extractor/File/FileMetadataRow.php
new file mode 100644
index 0000000000..d5dbc02d79
--- /dev/null
+++ b/src/core/etl/src/Flow/ETL/Extractor/File/FileMetadataRow.php
@@ -0,0 +1,27 @@
+path->extension();
+
+ return [
+ 'path' => $status->path->path(),
+ 'protocol' => $status->path->protocol(),
+ 'file_name' => $status->path->filename(),
+ 'base_name' => $status->path->basename(),
+ 'is_file' => $status->isFile(),
+ 'is_dir' => $status->isDirectory(),
+ // Path::extension() answers false for an extensionless file; the column is one
+ // type, so the absence is spelled null rather than a boolean in a string column.
+ 'extension' => $extension === false ? null : $extension,
+ ];
+ }
+}
diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/FileReadLoop.php b/src/core/etl/src/Flow/ETL/Extractor/File/FileReadLoop.php
index 3375c2b4d1..6080039d8c 100644
--- a/src/core/etl/src/Flow/ETL/Extractor/File/FileReadLoop.php
+++ b/src/core/etl/src/Flow/ETL/Extractor/File/FileReadLoop.php
@@ -4,8 +4,10 @@
namespace Flow\ETL\Extractor\File;
-use Flow\ETL\Column\Backend;
use Flow\ETL\Extractor\Signal;
+use Flow\ETL\FlowContext;
+use Flow\ETL\Function\Evaluation\Filter;
+use Flow\ETL\Function\ScalarFunction;
use Flow\ETL\Rows;
use Flow\ETL\Schema;
use Generator;
@@ -14,9 +16,13 @@
final readonly class FileReadLoop
{
+ /**
+ * @param list $conjuncts FileScan::$conjuncts
+ */
public function __construct(
private FileColumns $fileColumns,
private Schema $schema,
+ private array $conjuncts = [],
) {}
/**
@@ -29,17 +35,20 @@ public function read(
array $sources,
FileBatches $batches,
int $batchSize,
- Backend $backend,
+ FlowContext $context,
ReadWindow $window,
): Generator {
$body = $this->fileColumns->withoutTail($this->schema);
+ $backend = $context->backend();
+ $filter = Filter::bind($this->schema, ...$this->conjuncts);
$offset = $window->offset;
$skips = $batches instanceof OffsetSkippingFileBatches;
$yielded = 0;
foreach ($sources as $source) {
$constants = $this->fileColumns->forFile($source, $this->schema);
- $wanted = $window->limit === null ? null : $window->limit - $yielded;
+ // a limit counts the rows the filter keeps, which a format would count before it
+ $wanted = $window->limit === null || $this->conjuncts !== [] ? null : $window->limit - $yielded;
$file = $batches->batches(
$source,
$body,
@@ -63,11 +72,16 @@ public function read(
$batch = $batch->slice($skipped, $batch->count() - $skipped);
}
- if ($window->limit !== null && $batch->count() > ($window->limit - $yielded)) {
- $batch = $batch->slice(0, $window->limit - $yielded);
+ $rows = $filter->filter($constants->fillRows($batch, $this->schema, $backend), $context);
+
+ if ($rows->isEmpty()) {
+ continue;
+ }
+
+ if ($window->limit !== null && $rows->count() > ($window->limit - $yielded)) {
+ $rows = $rows->slice(0, $window->limit - $yielded);
}
- $rows = $constants->fillRows($batch, $this->schema, $backend);
$yielded += $rows->count();
$signal = yield $rows;
diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/FileScan.php b/src/core/etl/src/Flow/ETL/Extractor/File/FileScan.php
new file mode 100644
index 0000000000..2d43e85028
--- /dev/null
+++ b/src/core/etl/src/Flow/ETL/Extractor/File/FileScan.php
@@ -0,0 +1,77 @@
+
+ */
+ public ?int $limit;
+
+ /**
+ * @param null|int $limit the rows the plan needs, see Extractor::extract()
+ * @param Filter $pathFilter what this read lists
+ * @param list $conjuncts predicates over the source's columns, in push order; the source returns
+ * only rows matching every one, and $limit counts those rows
+ *
+ * @throws InvalidArgumentException when $limit is not greater than 0
+ */
+ public function __construct(
+ ?int $limit = null,
+ public Filter $pathFilter = new OnlyFiles(),
+ public array $conjuncts = [],
+ ) {
+ if ($limit !== null && $limit <= 0) {
+ throw new InvalidArgumentException('Limit must be greater than 0');
+ }
+
+ $this->limit = $limit;
+ }
+
+ public function narrows(): bool
+ {
+ return $this->limit !== null || !$this->pathFilter instanceof OnlyFiles || $this->conjuncts !== [];
+ }
+
+ public function withFilter(ScalarFunction $filter): self
+ {
+ return new self($this->limit, $this->pathFilter, [...$this->conjuncts, ...(new Conjuncts())->of($filter)]);
+ }
+
+ /**
+ * Narrowing only: a second push may lower the limit, never raise it.
+ *
+ * @throws InvalidArgumentException when $limit is not greater than 0
+ */
+ public function withLimit(int $limit): self
+ {
+ return new self(
+ $this->limit === null ? $limit : min($this->limit, $limit),
+ $this->pathFilter,
+ $this->conjuncts,
+ );
+ }
+
+ public function withPathFilter(Filter $filter): self
+ {
+ return new self(
+ $this->limit,
+ $this->pathFilter instanceof Filters
+ ? $this->pathFilter->add($filter)
+ : new Filters($this->pathFilter, $filter),
+ $this->conjuncts,
+ );
+ }
+}
diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/ListedFileRow.php b/src/core/etl/src/Flow/ETL/Extractor/File/ListedFileRow.php
new file mode 100644
index 0000000000..ada03cbc48
--- /dev/null
+++ b/src/core/etl/src/Flow/ETL/Extractor/File/ListedFileRow.php
@@ -0,0 +1,15 @@
+ the row of one listed file, its partition columns left out
+ */
+ public function row(FileStatus $status): array;
+}
diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/ListingScan.php b/src/core/etl/src/Flow/ETL/Extractor/File/ListingScan.php
new file mode 100644
index 0000000000..0dae904f41
--- /dev/null
+++ b/src/core/etl/src/Flow/ETL/Extractor/File/ListingScan.php
@@ -0,0 +1,79 @@
+ $batchSize
+ *
+ * @return Generator
+ */
+ public function scan(FlowContext $context, FileScan $scan, Schema $schema, int $batchSize): Generator
+ {
+ $backend = $context->backend();
+ $filter = Filter::bind($schema, ...$scan->conjuncts);
+ $builder = new RowsBuilder($schema, $backend);
+ $yielded = 0;
+ $listing = (new FileListing($this->filesystem))->list($this->path, $scan->pathFilter);
+
+ while ($listing->valid()) {
+ $fileStatus = type_instance_of(FileStatus::class)->assert($listing->current());
+ $listing->next();
+
+ $builder->append([
+ ...$this->row->row($fileStatus),
+ ...$this->fileColumns->forFile(new SourceFile($fileStatus->path, $fileStatus->size), $schema)->values(),
+ ]);
+
+ if ($builder->count() < $batchSize && $listing->valid()) {
+ continue;
+ }
+
+ $rows = $filter->filter($builder->finish(), $context);
+ $builder = new RowsBuilder($schema, $backend);
+
+ if ($rows->isEmpty()) {
+ continue;
+ }
+
+ if ($scan->limit !== null && $rows->count() > ($scan->limit - $yielded)) {
+ $rows = $rows->slice(0, $scan->limit - $yielded);
+ }
+
+ $yielded += $rows->count();
+ $signal = yield $rows;
+
+ if ($signal === Signal::STOP) {
+ return;
+ }
+
+ if ($scan->limit !== null && $yielded >= $scan->limit) {
+ return;
+ }
+ }
+ }
+}
diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/PathPartitionsRow.php b/src/core/etl/src/Flow/ETL/Extractor/File/PathPartitionsRow.php
new file mode 100644
index 0000000000..b9fdf9267f
--- /dev/null
+++ b/src/core/etl/src/Flow/ETL/Extractor/File/PathPartitionsRow.php
@@ -0,0 +1,25 @@
+ $status->path->uri(),
+ 'partitions' => array_merge(...array_values(array_map(static fn(Partition $p) => [
+ $p->name => $p->value,
+ ], $status->path->partitions()->toArray()))),
+ ];
+ }
+}
diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/Scanning.php b/src/core/etl/src/Flow/ETL/Extractor/File/Scanning.php
new file mode 100644
index 0000000000..9d6ec43bfe
--- /dev/null
+++ b/src/core/etl/src/Flow/ETL/Extractor/File/Scanning.php
@@ -0,0 +1,24 @@
+
+ */
+ public function extract(FlowContext $context, ?int $limit = null): Generator
+ {
+ return $this->scan($context, new FileScan(limit: $limit));
+ }
+}
diff --git a/src/core/etl/src/Flow/ETL/Extractor/FilesExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/FilesExtractor.php
index 41295a8a14..3924b29ebb 100644
--- a/src/core/etl/src/Flow/ETL/Extractor/FilesExtractor.php
+++ b/src/core/etl/src/Flow/ETL/Extractor/FilesExtractor.php
@@ -8,17 +8,19 @@
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\Extractor;
use Flow\ETL\Extractor\File\FileExtractor;
+use Flow\ETL\Extractor\File\FileMetadataRow;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\File\ListedFiles;
use Flow\ETL\Extractor\File\ListingPartitions;
-use Flow\ETL\Extractor\File\SourceFile;
+use Flow\ETL\Extractor\File\ListingScan;
+use Flow\ETL\Extractor\File\Scanning;
use Flow\ETL\FlowContext;
-use Flow\ETL\Rows\RowsBuilder;
+use Flow\ETL\Rows;
use Flow\ETL\Schema;
use Flow\Filesystem\FileListing;
use Flow\Filesystem\Filesystem;
use Flow\Filesystem\Local\NativeLocalFilesystem;
use Flow\Filesystem\Path;
-use Flow\Filesystem\Path\Filter;
use Flow\Filesystem\Path\Filter\OnlyFiles;
use Generator;
@@ -34,6 +36,7 @@ final class FilesExtractor implements BatchableExtractor, Extractor, FileExtract
private ?Statistics $statistics = null;
use Batches;
+ use Scanning;
use ListingPartitions;
private readonly Filesystem $filesystem;
@@ -61,56 +64,16 @@ public function isRepeatable(): bool
}
/**
- * @return Generator
+ * @return Generator
*/
- public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator
+ public function scan(FlowContext $context, FileScan $scan): Generator
{
- $batchSize = $this->batchSize();
- $backend = $context->backend();
- $fileColumns = $this->fileColumns($this->filesystem, $this->path);
- $schema = $this->schema();
- $builder = new RowsBuilder($schema, $backend);
- $yielded = 0;
-
- foreach ((new FileListing($this->filesystem))->list($this->path, $pathFilter) as $fileStatus) {
- $extension = $fileStatus->path->extension();
-
- $builder->append([
- 'path' => $fileStatus->path->path(),
- 'protocol' => $fileStatus->path->protocol(),
- 'file_name' => $fileStatus->path->filename(),
- 'base_name' => $fileStatus->path->basename(),
- 'is_file' => $fileStatus->isFile(),
- 'is_dir' => $fileStatus->isDirectory(),
- // Path::extension() answers false for an extensionless file; the column is one
- // type, so the absence is spelled null rather than a boolean in a string column.
- 'extension' => $extension === false ? null : $extension,
- ...$fileColumns->forFile(new SourceFile($fileStatus->path, $fileStatus->size), $schema)->values(),
- ]);
-
- if ($builder->count() < $batchSize) {
- continue;
- }
-
- $rows = $builder->finish();
- $yielded += $rows->count();
-
- $signal = yield $rows;
-
- if ($signal === Signal::STOP) {
- return;
- }
-
- $builder = new RowsBuilder($schema, $backend);
-
- if ($limit !== null && $yielded >= $limit) {
- return;
- }
- }
-
- if ($builder->count() > 0) {
- yield $builder->finish();
- }
+ return (new ListingScan(
+ $this->filesystem,
+ $this->path,
+ $this->fileColumns($this->filesystem, $this->path),
+ new FileMetadataRow(),
+ ))->scan($context, $scan, $this->schema(), $this->batchSize());
}
public function partitionSchema(): Schema
diff --git a/src/core/etl/src/Flow/ETL/Extractor/PathPartitionsExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/PathPartitionsExtractor.php
index d9a9fd0220..9713a9c95a 100644
--- a/src/core/etl/src/Flow/ETL/Extractor/PathPartitionsExtractor.php
+++ b/src/core/etl/src/Flow/ETL/Extractor/PathPartitionsExtractor.php
@@ -8,23 +8,21 @@
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\Extractor;
use Flow\ETL\Extractor\File\FileExtractor;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\File\ListingPartitions;
-use Flow\ETL\Extractor\File\SourceFile;
+use Flow\ETL\Extractor\File\ListingScan;
+use Flow\ETL\Extractor\File\PathPartitionsRow;
+use Flow\ETL\Extractor\File\Scanning;
use Flow\ETL\FlowContext;
-use Flow\ETL\Rows\RowsBuilder;
+use Flow\ETL\Rows;
use Flow\ETL\Schema;
use Flow\Filesystem\FileListing;
use Flow\Filesystem\Filesystem;
use Flow\Filesystem\Local\NativeLocalFilesystem;
-use Flow\Filesystem\Partition;
use Flow\Filesystem\Path;
-use Flow\Filesystem\Path\Filter;
use Flow\Filesystem\Path\Filter\OnlyFiles;
use Generator;
-use function array_map;
-use function array_merge;
-use function array_values;
use function Flow\ETL\DSL\map_schema;
use function Flow\ETL\DSL\schema;
use function Flow\ETL\DSL\str_schema;
@@ -40,6 +38,7 @@ final class PathPartitionsExtractor implements BatchableExtractor, Extractor, Fi
private ?Statistics $statistics = null;
use Batches;
+ use Scanning;
use ListingPartitions;
private readonly Filesystem $filesystem;
@@ -67,49 +66,16 @@ public function isRepeatable(): bool
}
/**
- * @return Generator
+ * @return Generator
*/
- public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator
+ public function scan(FlowContext $context, FileScan $scan): Generator
{
- $batchSize = $this->batchSize();
- $backend = $context->backend();
- $fileColumns = $this->fileColumns($this->filesystem, $this->path);
- $schema = $this->schema();
- $builder = new RowsBuilder($schema, $backend);
- $yielded = 0;
-
- foreach ((new FileListing($this->filesystem))->list($this->path, $pathFilter) as $fileStatus) {
- $builder->append([
- 'path' => $fileStatus->path->uri(),
- 'partitions' => array_merge(...array_values(array_map(static fn(Partition $p) => [
- $p->name => $p->value,
- ], $fileStatus->path->partitions()->toArray()))),
- ...$fileColumns->forFile(new SourceFile($fileStatus->path, $fileStatus->size), $schema)->values(),
- ]);
-
- if ($builder->count() < $batchSize) {
- continue;
- }
-
- $rows = $builder->finish();
- $yielded += $rows->count();
-
- $signal = yield $rows;
-
- if ($signal === Signal::STOP) {
- return;
- }
-
- $builder = new RowsBuilder($schema, $backend);
-
- if ($limit !== null && $yielded >= $limit) {
- return;
- }
- }
-
- if ($builder->count() > 0) {
- yield $builder->finish();
- }
+ return (new ListingScan(
+ $this->filesystem,
+ $this->path,
+ $this->fileColumns($this->filesystem, $this->path),
+ new PathPartitionsRow(),
+ ))->scan($context, $scan, $this->schema(), $this->batchSize());
}
public function partitionSchema(): Schema
diff --git a/src/core/etl/src/Flow/ETL/FlowPhpExtension.php b/src/core/etl/src/Flow/ETL/FlowPhpExtension.php
index 20844015ee..0569a67795 100644
--- a/src/core/etl/src/Flow/ETL/FlowPhpExtension.php
+++ b/src/core/etl/src/Flow/ETL/FlowPhpExtension.php
@@ -18,7 +18,7 @@
* The contract version this package expects from flow_php, equal to the extension's FLOW_PHP_ABI. Both sides change
* together whenever a class or interface the extension registers changes.
*/
- public const int ABI = 1;
+ public const int ABI = 2;
public function __construct(
public bool $loaded,
diff --git a/src/core/etl/src/Flow/ETL/Function/Between.php b/src/core/etl/src/Flow/ETL/Function/Between.php
index 89f9dd2e83..7edc5f315a 100644
--- a/src/core/etl/src/Flow/ETL/Function/Between.php
+++ b/src/core/etl/src/Flow/ETL/Function/Between.php
@@ -111,6 +111,7 @@ public function eval(Rows $rows, FlowContext $context): Column
[$values, $lowerBounds, $upperBounds] = array_map(static fn(Column $column): array => $sameType
? $comparable->ordering($column)
: $column->values(), $columns);
+ $order = new ValueOrder(...array_map(static fn(Column $column): Type => $column->type(), $columns));
$results = [];
$i = 0;
@@ -120,7 +121,7 @@ public function eval(Rows $rows, FlowContext $context): Column
throw new InvalidArgumentException('Between function requires valid boundary');
}
- $results[] = $boundary->compare($values[$i], $lowerBounds[$i], $upperBounds[$i]);
+ $results[] = $boundary->compare($values[$i], $lowerBounds[$i], $upperBounds[$i], $order);
}
} catch (Exception $e) {
throw EvaluationException::at($i, $e);
diff --git a/src/core/etl/src/Flow/ETL/Function/Between/Boundary.php b/src/core/etl/src/Flow/ETL/Function/Between/Boundary.php
index 20e538645d..c7b181d281 100644
--- a/src/core/etl/src/Flow/ETL/Function/Between/Boundary.php
+++ b/src/core/etl/src/Flow/ETL/Function/Between/Boundary.php
@@ -4,15 +4,7 @@
namespace Flow\ETL\Function\Between;
-use DateInterval;
-use DateTimeImmutable;
-use DateTimeInterface;
-use Flow\ETL\Exception\InvalidArgumentException;
-
-use function is_array;
-use function is_bool;
-use function is_numeric;
-use function is_string;
+use Flow\ETL\Function\ValueOrder;
enum Boundary
{
@@ -21,8 +13,12 @@ enum Boundary
case LEFT_INCLUSIVE;
case RIGHT_INCLUSIVE;
- public function compare(mixed $value, mixed $lowerBound, mixed $upperbound): ?bool
- {
+ public function compare(
+ mixed $value,
+ mixed $lowerBound,
+ mixed $upperbound,
+ ValueOrder $order = new ValueOrder(),
+ ): ?bool {
if ($value === null) {
return null;
}
@@ -31,8 +27,8 @@ public function compare(mixed $value, mixed $lowerBound, mixed $upperbound): ?bo
if ($lowerBound !== null) {
$aboveLower = match ($this) {
- self::INCLUSIVE, self::LEFT_INCLUSIVE => $this->compareValues($value, $lowerBound) >= 0,
- self::EXCLUSIVE, self::RIGHT_INCLUSIVE => $this->compareValues($value, $lowerBound) > 0,
+ self::INCLUSIVE, self::LEFT_INCLUSIVE => $order->compare($lowerBound, $value) <= 0,
+ self::EXCLUSIVE, self::RIGHT_INCLUSIVE => $order->compare($lowerBound, $value) < 0,
};
}
@@ -40,8 +36,8 @@ public function compare(mixed $value, mixed $lowerBound, mixed $upperbound): ?bo
if ($upperbound !== null) {
$belowUpper = match ($this) {
- self::INCLUSIVE, self::RIGHT_INCLUSIVE => $this->compareValues($value, $upperbound) <= 0,
- self::EXCLUSIVE, self::LEFT_INCLUSIVE => $this->compareValues($value, $upperbound) < 0,
+ self::INCLUSIVE, self::RIGHT_INCLUSIVE => $order->compare($value, $upperbound) <= 0,
+ self::EXCLUSIVE, self::LEFT_INCLUSIVE => $order->compare($value, $upperbound) < 0,
};
}
@@ -56,37 +52,4 @@ public function compare(mixed $value, mixed $lowerBound, mixed $upperbound): ?bo
return true;
}
-
- public function compareValues(mixed $left, mixed $right): int
- {
- if (is_numeric($left) && is_numeric($right)) {
- return (float) $left <=> (float) $right;
- }
-
- if (is_string($left) && is_string($right)) {
- return $left <=> $right;
- }
-
- if ($left instanceof DateTimeInterface && $right instanceof DateTimeInterface) {
- return $left <=> $right;
- }
-
- if ($left instanceof DateInterval && $right instanceof DateInterval) {
- $reference = new DateTimeImmutable('@0');
-
- return $reference->add($left) <=> $reference->add($right);
- }
-
- if (is_bool($left) && is_bool($right)) {
- return $left <=> $right;
- }
-
- if (is_array($left) && is_array($right)) {
- return $left <=> $right;
- }
-
- throw new InvalidArgumentException(
- 'Boundary::compare requires value and bounds to be of a comparable type (numeric, string, DateTimeInterface, DateInterval, bool, or array).',
- );
- }
}
diff --git a/src/core/etl/src/Flow/ETL/Function/Conjuncts.php b/src/core/etl/src/Flow/ETL/Function/Conjuncts.php
new file mode 100644
index 0000000000..e6a5e5dcbf
--- /dev/null
+++ b/src/core/etl/src/Flow/ETL/Function/Conjuncts.php
@@ -0,0 +1,28 @@
+
+ */
+ public function of(ScalarFunction $function): array
+ {
+ if (!$function instanceof All) {
+ return [$function];
+ }
+
+ $conjuncts = [];
+
+ foreach ($function->children() as $child) {
+ $conjuncts[] = $this->of($child);
+ }
+
+ return array_merge(...$conjuncts);
+ }
+}
diff --git a/src/core/etl/src/Flow/ETL/Function/Evaluation/Filter.php b/src/core/etl/src/Flow/ETL/Function/Evaluation/Filter.php
new file mode 100644
index 0000000000..ee2da8bdad
--- /dev/null
+++ b/src/core/etl/src/Flow/ETL/Function/Evaluation/Filter.php
@@ -0,0 +1,103 @@
+ $predicates resolved against the bound schema
+ */
+ private function __construct(
+ private array $predicates,
+ ) {}
+
+ /**
+ * @throws InvalidArgumentException
+ */
+ public static function bind(Schema $schema, ScalarFunction ...$predicates): self
+ {
+ $resolver = new ReferenceResolver();
+ $resolved = [];
+
+ foreach ($predicates as $predicate) {
+ $function = $resolver->resolve($predicate, $schema);
+ $resolver->assertResolved($function, $schema);
+ (new ExpandingFunctions())->refuse($function, 'filter');
+
+ // type_bare() keeps the gate blind to nullability - a null-propagating predicate declares
+ // ?boolean, and a NULL row is dropped like a false one.
+ if (!type_equals(type_bare($function->returns()), type_boolean())) {
+ throw new InvalidArgumentException(sprintf(
+ 'filter() requires a predicate returning boolean, "%s" returns "%s". '
+ . 'Use an explicit comparison, e.g. ->notEquals(lit(0)).',
+ $function::class,
+ $function->returns()->toString(),
+ ));
+ }
+
+ $resolved[] = $function;
+ }
+
+ return new self($resolved);
+ }
+
+ /**
+ * The rows every predicate is TRUE for: each is evaluated only on the rows the ones before it kept.
+ */
+ public function filter(Rows $rows, FlowContext $context): Rows
+ {
+ if ($this->predicates === []) {
+ return $rows;
+ }
+
+ $kept = $rows->isEmpty() ? [] : range(0, $rows->count() - 1);
+
+ foreach ($this->predicates as $predicate) {
+ if ($kept === []) {
+ break;
+ }
+
+ $truths = array_keys(
+ (new Selection($kept))
+ ->evaluate($predicate, $rows, $context)
+ ->physicals(),
+ true,
+ true,
+ );
+
+ if (count($kept) === $rows->count()) {
+ $kept = $truths;
+
+ continue;
+ }
+
+ $next = [];
+
+ foreach ($truths as $row) {
+ $next[] = $kept[$row];
+ }
+
+ $kept = $next;
+ }
+
+ return count($kept) === $rows->count() ? $rows : $rows->gather($kept);
+ }
+}
diff --git a/src/core/etl/src/Flow/ETL/Function/GreaterThan.php b/src/core/etl/src/Flow/ETL/Function/GreaterThan.php
index d87f2fd5c2..066e2cdb41 100644
--- a/src/core/etl/src/Flow/ETL/Function/GreaterThan.php
+++ b/src/core/etl/src/Flow/ETL/Function/GreaterThan.php
@@ -4,8 +4,6 @@
namespace Flow\ETL\Function;
-use DateInterval;
-use DateTimeImmutable;
use Exception;
use Flow\ETL\Column\Column;
use Flow\ETL\Column\ComparableValues;
@@ -85,6 +83,7 @@ public function eval(Rows $rows, FlowContext $context): Column
[$lefts, $rights] = type_bare($leftColumn->type())::class === type_bare($rightColumn->type())::class
? [$comparable->ordering($leftColumn), $comparable->ordering($rightColumn)]
: [$leftColumn->values(), $rightColumn->values()];
+ $order = new ValueOrder($leftColumn->type(), $rightColumn->type());
$results = [];
$i = 0;
@@ -100,17 +99,7 @@ public function eval(Rows $rows, FlowContext $context): Column
continue;
}
- // The dispatch picks a comparison strategy from the values, never a column type - bind has
- // already proved the pair comparable in returns().
- if ($left instanceof DateInterval && $right instanceof DateInterval) {
- $reference = new DateTimeImmutable('@0');
- $results[] = $reference->add($left) > $reference->add($right);
-
- continue;
- }
-
- // @mago-expect analysis:mixed-operand(2)
- $results[] = $left > $right;
+ $results[] = $order->compare($right, $left) < 0;
}
} catch (Exception $e) {
throw EvaluationException::at($i, $e);
diff --git a/src/core/etl/src/Flow/ETL/Function/GreaterThanEqual.php b/src/core/etl/src/Flow/ETL/Function/GreaterThanEqual.php
index 41d7f760df..827371c05c 100644
--- a/src/core/etl/src/Flow/ETL/Function/GreaterThanEqual.php
+++ b/src/core/etl/src/Flow/ETL/Function/GreaterThanEqual.php
@@ -4,8 +4,6 @@
namespace Flow\ETL\Function;
-use DateInterval;
-use DateTimeImmutable;
use Exception;
use Flow\ETL\Column\Column;
use Flow\ETL\Column\ComparableValues;
@@ -85,6 +83,7 @@ public function eval(Rows $rows, FlowContext $context): Column
[$lefts, $rights] = type_bare($leftColumn->type())::class === type_bare($rightColumn->type())::class
? [$comparable->ordering($leftColumn), $comparable->ordering($rightColumn)]
: [$leftColumn->values(), $rightColumn->values()];
+ $order = new ValueOrder($leftColumn->type(), $rightColumn->type());
$results = [];
$i = 0;
@@ -100,17 +99,7 @@ public function eval(Rows $rows, FlowContext $context): Column
continue;
}
- // The dispatch picks a comparison strategy from the values, never a column type - bind has
- // already proved the pair comparable in returns().
- if ($left instanceof DateInterval && $right instanceof DateInterval) {
- $reference = new DateTimeImmutable('@0');
- $results[] = $reference->add($left) >= $reference->add($right);
-
- continue;
- }
-
- // @mago-expect analysis:mixed-operand(2)
- $results[] = $left >= $right;
+ $results[] = $order->compare($right, $left) <= 0;
}
} catch (Exception $e) {
throw EvaluationException::at($i, $e);
diff --git a/src/core/etl/src/Flow/ETL/Function/Greatest.php b/src/core/etl/src/Flow/ETL/Function/Greatest.php
index ac6f45fa1e..abae972f82 100644
--- a/src/core/etl/src/Flow/ETL/Function/Greatest.php
+++ b/src/core/etl/src/Flow/ETL/Function/Greatest.php
@@ -16,7 +16,6 @@
use function array_map;
use function array_values;
use function Flow\ETL\DSL\lit;
-use function max;
final class Greatest implements ScalarFunction
{
@@ -73,23 +72,33 @@ public function returns(): Type
public function eval(Rows $rows, FlowContext $context): Column
{
- $arguments = array_map(static fn(ScalarFunction $value): array => (new Parameter($value))->values(
+ $columns = array_map(static fn(ScalarFunction $value): Column => (new Parameter($value))->column(
$rows,
$context,
), $this->values);
+ $order = new ValueOrder(...array_map(static fn(Column $column): Type => $column->type(), $columns));
+ $arguments = array_map(static fn(Column $column): array => $column->values(), $columns);
$results = [];
for ($i = 0, $count = $rows->count(); $i < $count; $i++) {
// NULL iff every argument is NULL - nulls are skipped, never compared
- $values = [];
+ $greatest = null;
foreach ($arguments as $argument) {
- if ($argument[$i] !== null) {
- $values[] = $argument[$i];
+ // @mago-ignore analysis:mixed-assignment
+ $value = $argument[$i];
+
+ if ($value === null) {
+ continue;
+ }
+
+ if ($greatest === null || $order->compare($greatest, $value) < 0) {
+ // @mago-ignore analysis:mixed-assignment
+ $greatest = $value;
}
}
- $results[] = $values === [] ? null : max($values);
+ $results[] = $greatest;
}
return (new ResultColumn($context->backend()))->of($this, $results);
diff --git a/src/core/etl/src/Flow/ETL/Function/Least.php b/src/core/etl/src/Flow/ETL/Function/Least.php
index c89fd37a4e..ecabdee12f 100644
--- a/src/core/etl/src/Flow/ETL/Function/Least.php
+++ b/src/core/etl/src/Flow/ETL/Function/Least.php
@@ -16,7 +16,6 @@
use function array_map;
use function array_values;
use function Flow\ETL\DSL\lit;
-use function min;
final class Least implements ScalarFunction
{
@@ -73,23 +72,33 @@ public function returns(): Type
public function eval(Rows $rows, FlowContext $context): Column
{
- $arguments = array_map(static fn(ScalarFunction $value): array => (new Parameter($value))->values(
+ $columns = array_map(static fn(ScalarFunction $value): Column => (new Parameter($value))->column(
$rows,
$context,
), $this->values);
+ $order = new ValueOrder(...array_map(static fn(Column $column): Type => $column->type(), $columns));
+ $arguments = array_map(static fn(Column $column): array => $column->values(), $columns);
$results = [];
for ($i = 0, $count = $rows->count(); $i < $count; $i++) {
// NULL iff every argument is NULL - nulls are skipped, never compared
- $values = [];
+ $least = null;
foreach ($arguments as $argument) {
- if ($argument[$i] !== null) {
- $values[] = $argument[$i];
+ // @mago-ignore analysis:mixed-assignment
+ $value = $argument[$i];
+
+ if ($value === null) {
+ continue;
+ }
+
+ if ($least === null || $order->compare($value, $least) < 0) {
+ // @mago-ignore analysis:mixed-assignment
+ $least = $value;
}
}
- $results[] = $values === [] ? null : min($values);
+ $results[] = $least;
}
return (new ResultColumn($context->backend()))->of($this, $results);
diff --git a/src/core/etl/src/Flow/ETL/Function/LessThan.php b/src/core/etl/src/Flow/ETL/Function/LessThan.php
index c644f4c1c6..57ab86434b 100644
--- a/src/core/etl/src/Flow/ETL/Function/LessThan.php
+++ b/src/core/etl/src/Flow/ETL/Function/LessThan.php
@@ -4,8 +4,6 @@
namespace Flow\ETL\Function;
-use DateInterval;
-use DateTimeImmutable;
use Exception;
use Flow\ETL\Column\Column;
use Flow\ETL\Column\ComparableValues;
@@ -85,6 +83,7 @@ public function eval(Rows $rows, FlowContext $context): Column
[$lefts, $rights] = type_bare($leftColumn->type())::class === type_bare($rightColumn->type())::class
? [$comparable->ordering($leftColumn), $comparable->ordering($rightColumn)]
: [$leftColumn->values(), $rightColumn->values()];
+ $order = new ValueOrder($leftColumn->type(), $rightColumn->type());
$results = [];
$i = 0;
@@ -100,17 +99,7 @@ public function eval(Rows $rows, FlowContext $context): Column
continue;
}
- // The dispatch picks a comparison strategy from the values, never a column type - bind has
- // already proved the pair comparable in returns().
- if ($left instanceof DateInterval && $right instanceof DateInterval) {
- $reference = new DateTimeImmutable('@0');
- $results[] = $reference->add($left) < $reference->add($right);
-
- continue;
- }
-
- // @mago-expect analysis:mixed-operand(2)
- $results[] = $left < $right;
+ $results[] = $order->compare($left, $right) < 0;
}
} catch (Exception $e) {
throw EvaluationException::at($i, $e);
diff --git a/src/core/etl/src/Flow/ETL/Function/LessThanEqual.php b/src/core/etl/src/Flow/ETL/Function/LessThanEqual.php
index c94d5de930..589b6e392d 100644
--- a/src/core/etl/src/Flow/ETL/Function/LessThanEqual.php
+++ b/src/core/etl/src/Flow/ETL/Function/LessThanEqual.php
@@ -4,8 +4,6 @@
namespace Flow\ETL\Function;
-use DateInterval;
-use DateTimeImmutable;
use Exception;
use Flow\ETL\Column\Column;
use Flow\ETL\Column\ComparableValues;
@@ -85,6 +83,7 @@ public function eval(Rows $rows, FlowContext $context): Column
[$lefts, $rights] = type_bare($leftColumn->type())::class === type_bare($rightColumn->type())::class
? [$comparable->ordering($leftColumn), $comparable->ordering($rightColumn)]
: [$leftColumn->values(), $rightColumn->values()];
+ $order = new ValueOrder($leftColumn->type(), $rightColumn->type());
$results = [];
$i = 0;
@@ -100,17 +99,7 @@ public function eval(Rows $rows, FlowContext $context): Column
continue;
}
- // The dispatch picks a comparison strategy from the values, never a column type - bind has
- // already proved the pair comparable in returns().
- if ($left instanceof DateInterval && $right instanceof DateInterval) {
- $reference = new DateTimeImmutable('@0');
- $results[] = $reference->add($left) <= $reference->add($right);
-
- continue;
- }
-
- // @mago-expect analysis:mixed-operand(2)
- $results[] = $left <= $right;
+ $results[] = $order->compare($left, $right) <= 0;
}
} catch (Exception $e) {
throw EvaluationException::at($i, $e);
diff --git a/src/core/etl/src/Flow/ETL/Function/ValueOrder.php b/src/core/etl/src/Flow/ETL/Function/ValueOrder.php
new file mode 100644
index 0000000000..d3eacc304a
--- /dev/null
+++ b/src/core/etl/src/Flow/ETL/Function/ValueOrder.php
@@ -0,0 +1,54 @@
+, so `$a < $b` is `compare($a, $b) < 0` and `$a > $b` is `compare($b, $a) < 0`,
+ * NaN included.
+ */
+final readonly class ValueOrder
+{
+ private bool $numericStrings;
+
+ /**
+ * @param Type ...$operands
+ */
+ public function __construct(Type ...$operands)
+ {
+ $numericStrings = false;
+
+ foreach ($operands as $operand) {
+ $numericStrings = $numericStrings || type_bare($operand) instanceof NumericStringType;
+ }
+
+ $this->numericStrings = $numericStrings;
+ }
+
+ public function compare(mixed $left, mixed $right): int
+ {
+ if (!$this->numericStrings && is_string($left) && is_string($right)) {
+ return strcmp($left, $right) <=> 0;
+ }
+
+ if ($left instanceof DateInterval && $right instanceof DateInterval) {
+ $reference = new DateTimeImmutable('@0');
+
+ return $reference->add($left) <=> $reference->add($right);
+ }
+
+ // @mago-expect analysis:mixed-operand(2)
+ return $left <=> $right;
+ }
+}
diff --git a/src/core/etl/src/Flow/ETL/Optimizer.php b/src/core/etl/src/Flow/ETL/Optimizer.php
index 909def5325..bbb8c6103f 100644
--- a/src/core/etl/src/Flow/ETL/Optimizer.php
+++ b/src/core/etl/src/Flow/ETL/Optimizer.php
@@ -37,8 +37,8 @@ public static function default(): self
return new self(
new CombineLimits(),
new CombineSortAndLimit(),
- new PushLimitIntoSource(),
new PushFilterIntoSource(),
+ new PushLimitIntoSource(),
new CountFromStatistics(),
new PushProjectionIntoExpand(),
);
diff --git a/src/core/etl/src/Flow/ETL/Optimizer/FilterWalk.php b/src/core/etl/src/Flow/ETL/Optimizer/FilterWalk.php
index 098733d12f..016d69e024 100644
--- a/src/core/etl/src/Flow/ETL/Optimizer/FilterWalk.php
+++ b/src/core/etl/src/Flow/ETL/Optimizer/FilterWalk.php
@@ -35,9 +35,9 @@ public function reachedByEveryConsumer(Node\Filter $filter, Node ...$consumerInp
/**
* $predicate as $leaf sees it, or null when it may not be evaluated there instead of above $filter.
- * Another Filter is TRANSPARENT here: it only removes rows, and the pushed Filter node stays, so pruning a
- * file drops rows that filter would have dropped anyway. Limit/Offset/Until/Distinct/Discard are not: they
- * make WHICH rows arrive depend on what the source read.
+ * Another Filter is TRANSPARENT here: it only removes rows, so the order two filters drop rows in does not change
+ * which rows survive both. Limit/Offset/Until/Distinct/Discard are not: they make WHICH rows arrive depend on
+ * what the source read.
*
* A node that redefines a column the predicate reads blocks it, unless the column is a plain alias of a
* column below (Rename, a WithColumn of a bare reference): the predicate is rewritten to read that column.
diff --git a/src/core/etl/src/Flow/ETL/Optimizer/JoinSides.php b/src/core/etl/src/Flow/ETL/Optimizer/JoinSides.php
index 2eea368fbb..26ac44d21e 100644
--- a/src/core/etl/src/Flow/ETL/Optimizer/JoinSides.php
+++ b/src/core/etl/src/Flow/ETL/Optimizer/JoinSides.php
@@ -21,15 +21,15 @@ public function __construct(
private FlowContext $context,
) {}
- public function of(Node $node): Node
+ public function of(Node $original, Node $rebuilt): Node
{
- if (!$node instanceof JoinsFrame) {
- return $node;
+ if (!$rebuilt instanceof JoinsFrame) {
+ return $rebuilt;
}
- return $node->withChildren([
- $node->children()[0],
- $this->optimizer->optimize(new LogicalPlan($node->right()), $this->context)->root,
+ return $rebuilt->withChildren([
+ $rebuilt->children()[0],
+ $this->optimizer->optimize(new LogicalPlan($rebuilt->right()), $this->context)->root,
]);
}
}
diff --git a/src/core/etl/src/Flow/ETL/Optimizer/Rule/CombineLimits.php b/src/core/etl/src/Flow/ETL/Optimizer/Rule/CombineLimits.php
index 6837db8868..a9d893143e 100644
--- a/src/core/etl/src/Flow/ETL/Optimizer/Rule/CombineLimits.php
+++ b/src/core/etl/src/Flow/ETL/Optimizer/Rule/CombineLimits.php
@@ -23,14 +23,16 @@ public function apply(LogicalPlan $plan, FlowContext $context): LogicalPlan
return $plan->transformUp($this);
}
- public function of(Node $node): Node
+ public function of(Node $original, Node $rebuilt): Node
{
- if (!$node instanceof Limit) {
- return $node;
+ if (!$rebuilt instanceof Limit) {
+ return $rebuilt;
}
- $child = $node->children()[0];
+ $child = $rebuilt->children()[0];
- return $child instanceof Limit ? new Limit($child->children()[0], min($node->limit, $child->limit)) : $node;
+ return $child instanceof Limit
+ ? new Limit($child->children()[0], min($rebuilt->limit, $child->limit))
+ : $rebuilt;
}
}
diff --git a/src/core/etl/src/Flow/ETL/Optimizer/Rule/CountFromStatistics.php b/src/core/etl/src/Flow/ETL/Optimizer/Rule/CountFromStatistics.php
index 6ab5bce034..ce90b859d6 100644
--- a/src/core/etl/src/Flow/ETL/Optimizer/Rule/CountFromStatistics.php
+++ b/src/core/etl/src/Flow/ETL/Optimizer/Rule/CountFromStatistics.php
@@ -11,7 +11,6 @@
use Flow\ETL\Plan\Node\Read;
use Flow\ETL\Plan\Node\Result;
use Flow\ETL\Processor\CountingProcessor;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
use function Flow\ETL\DSL\from_rows;
@@ -22,7 +21,7 @@ public function apply(LogicalPlan $plan, FlowContext $context): LogicalPlan
$count = $plan->root instanceof Result ? $plan->root->children()[0] : null;
$read = $count instanceof Count ? $count->children()[0] : null;
- if (!$read instanceof Read || $read->limit() !== null || !$read->pathFilter() instanceof OnlyFiles) {
+ if (!$read instanceof Read || $read->scan()->narrows()) {
return $plan;
}
diff --git a/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushFilterIntoSource.php b/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushFilterIntoSource.php
index 3d597af875..a696fdd9ca 100644
--- a/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushFilterIntoSource.php
+++ b/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushFilterIntoSource.php
@@ -8,19 +8,24 @@
use Flow\ETL\Filesystem\ScalarFunctionFilter;
use Flow\ETL\FlowContext;
use Flow\ETL\Function\All;
+use Flow\ETL\Function\Conjuncts;
use Flow\ETL\Function\ReferencedColumns;
use Flow\ETL\Optimizer\FilterWalk;
use Flow\ETL\Optimizer\Rule;
use Flow\ETL\Plan\LogicalPlan;
use Flow\ETL\Plan\Node;
-use Flow\ETL\Plan\ReplaceLeaf;
+use Flow\ETL\Plan\ReplaceNodes;
+use SplObjectStorage;
use function array_diff;
+use function array_reverse;
+use function count;
/**
- * Evaluate a partition-only predicate once per FILE instead of once per row: the file never opens. The Filter node
- * stays, so a source that lists more than the pushed filter admits still returns the right rows. The parts of an AND
- * that read only partition columns are pushed one by one; an OR is pushed only as a whole.
+ * Moves every deterministic part of an AND into the file source, which returns only the rows matching it; a
+ * partition-only part is also pushed as a path filter, so a file it rejects is never listed. A pushed part leaves its
+ * Filter node, and a node left with no part is removed. An OR is pushed only as a whole. Nodes are pushed leaf first;
+ * a part kept in a node stops the push of every part after it and of every node above it.
*/
final readonly class PushFilterIntoSource implements Rule
{
@@ -34,43 +39,70 @@ public function apply(LogicalPlan $plan, FlowContext $context): LogicalPlan
}
$partitions = $extractor->partitionSchema();
-
- if ($partitions->count() === 0) {
- return $plan;
- }
-
$consumerInputs = $plan->consumerInputs();
$walk = new FilterWalk();
$columns = new ReferencedColumns();
+ $conjuncts = new Conjuncts();
$names = $partitions->references()->names();
$read = $leaf;
+ /** @var SplObjectStorage $replacements */
+ $replacements = new SplObjectStorage();
+ $removals = [];
+ $filters = [];
for ($node = $plan->root; $node->children() !== []; $node = $node->children()[0]) {
- if (!$node instanceof Node\Filter || !$walk->reachedByEveryConsumer($node, ...$consumerInputs)) {
- continue;
+ if ($node instanceof Node\Filter && $walk->reachedByEveryConsumer($node, ...$consumerInputs)) {
+ $filters[] = $node;
}
+ }
- foreach ($node->function instanceof All ? $node->function->children() : [$node->function] as $conjunct) {
- if (!$conjunct->deterministic()) {
- continue;
- }
+ // leaf first, so the source evaluates a lower filter before the one above it relies on what it removed
+ foreach (array_reverse($filters) as $filter) {
+ $parts = $conjuncts->of($filter->function);
+ $kept = [];
- $atLeaf = $walk->predicateAtLeaf($node, $leaf, $conjunct);
+ foreach ($parts as $conjunct) {
+ // a conjunct after a kept one would be evaluated on rows the kept one removes
+ $atLeaf = $kept === [] && $conjunct->deterministic()
+ ? $walk->predicateAtLeaf($filter, $leaf, $conjunct)
+ : null;
if ($atLeaf === null) {
+ $kept[] = $conjunct;
+
continue;
}
- $refs = $columns->in($atLeaf);
+ $referenced = $columns->in($atLeaf)->names();
- if ($refs->all() === [] || array_diff($refs->names(), $names) !== []) {
- continue;
+ if ($referenced !== [] && array_diff($referenced, $names) === []) {
+ $read = $read->withPathFilter(new ScalarFunctionFilter($atLeaf, $partitions, $context));
}
- $read = $read->withPathFilter(new ScalarFunctionFilter($atLeaf, $partitions, $context));
+ $read = $read->withFilter($atLeaf);
+ }
+
+ if ($kept === []) {
+ $removals[] = $filter;
+ } elseif (count($kept) !== count($parts)) {
+ $replacements[$filter] = new Node\Filter(
+ $filter->children()[0],
+ count($kept) === 1 ? $kept[0] : new All(...$kept),
+ );
}
+
+ // a filter above a kept conjunct would be evaluated on rows that conjunct removes
+ if ($kept !== []) {
+ break;
+ }
+ }
+
+ if ($read !== $leaf) {
+ $replacements[$leaf] = $read;
}
- return $read === $leaf ? $plan : $plan->transformUp(new ReplaceLeaf($leaf, $read));
+ return $replacements->count() === 0 && $removals === []
+ ? $plan
+ : $plan->transformUp(new ReplaceNodes($replacements, $removals));
}
}
diff --git a/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushLimitIntoSource.php b/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushLimitIntoSource.php
index 36beb317b9..70edda1588 100644
--- a/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushLimitIntoSource.php
+++ b/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushLimitIntoSource.php
@@ -8,7 +8,7 @@
use Flow\ETL\Optimizer\LimitWalk;
use Flow\ETL\Optimizer\Rule;
use Flow\ETL\Plan\LogicalPlan;
-use Flow\ETL\Plan\ReplaceLeaf;
+use Flow\ETL\Plan\ReplaceNodes;
use function max;
@@ -34,6 +34,6 @@ public function apply(LogicalPlan $plan, FlowContext $context): LogicalPlan
$limits[] = $limit;
}
- return $plan->transformUp(new ReplaceLeaf($leaf, $leaf->withLimit(max($limits))));
+ return $plan->transformUp(ReplaceNodes::one($leaf, $leaf->withLimit(max($limits))));
}
}
diff --git a/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushProjectionIntoExpand.php b/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushProjectionIntoExpand.php
index 4a32d1fd81..cf91544685 100644
--- a/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushProjectionIntoExpand.php
+++ b/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushProjectionIntoExpand.php
@@ -9,13 +9,10 @@
use Flow\ETL\Optimizer\Rule;
use Flow\ETL\Plan\LogicalPlan;
use Flow\ETL\Plan\Node;
-use Flow\ETL\Plan\ReplaceLeaf;
+use Flow\ETL\Plan\ReplaceNodes;
use Flow\ETL\Plan\RequiredColumns;
use SplObjectStorage;
-use function arsort;
-use function spl_object_id;
-
/**
* Columns no consumer reads are not copied into the rows an expand emits (Spark: ColumnPruning setting
* Generate.unrequiredChildIndex).
@@ -40,33 +37,15 @@ public function apply(LogicalPlan $plan, FlowContext $context): LogicalPlan
}
}
- $depths = [];
- $targets = [];
+ /** @var SplObjectStorage $replacements */
+ $replacements = new SplObjectStorage();
foreach ($demands as $node) {
- if ($demands[$node]->isAll()) {
- continue;
- }
-
- $id = spl_object_id($node);
- $targets[$id] = $node;
- $depths[$id] = 0;
-
- for ($below = $node; $below->children() !== []; $below = $below->children()[0]) {
- $depths[$id]++;
+ if (!$demands[$node]->isAll()) {
+ $replacements[$node] = $node->withCarries($demands[$node]);
}
}
- // TransformUp hands Rewrite::of() the node rebuilt over its new children, so a node whose descendant was
- // replaced first would no longer be the target; the one farthest from the leaf goes first
- arsort($depths);
-
- foreach ($depths as $id => $_) {
- $plan = $plan->transformUp(
- new ReplaceLeaf($targets[$id], $targets[$id]->withCarries($demands[$targets[$id]])),
- );
- }
-
- return $plan;
+ return $replacements->count() === 0 ? $plan : $plan->transformUp(new ReplaceNodes($replacements));
}
}
diff --git a/src/core/etl/src/Flow/ETL/Optimizer/TopNRewrite.php b/src/core/etl/src/Flow/ETL/Optimizer/TopNRewrite.php
index 2beb7e7720..af11e63585 100644
--- a/src/core/etl/src/Flow/ETL/Optimizer/TopNRewrite.php
+++ b/src/core/etl/src/Flow/ETL/Optimizer/TopNRewrite.php
@@ -15,19 +15,19 @@
*/
final readonly class TopNRewrite implements Rewrite
{
- public function of(Node $node): Node
+ public function of(Node $original, Node $rebuilt): Node
{
- if (!$node instanceof Limit) {
- return $node;
+ if (!$rebuilt instanceof Limit) {
+ return $rebuilt;
}
- $sort = $node->children()[0];
+ $sort = $rebuilt->children()[0];
if (!$sort instanceof Sort) {
- return $node;
+ return $rebuilt;
}
// a TopN past its memory limit falls back to the sort's own spilling path, so any limit is safe to rewrite
- return new TopN($sort->children()[0], $sort->refs, $node->limit, $sort->algorithm);
+ return new TopN($sort->children()[0], $sort->refs, $rebuilt->limit, $sort->algorithm);
}
}
diff --git a/src/core/etl/src/Flow/ETL/Plan/Explain/Details.php b/src/core/etl/src/Flow/ETL/Plan/Explain/Details.php
index a6815e6530..a28edc17f3 100644
--- a/src/core/etl/src/Flow/ETL/Plan/Explain/Details.php
+++ b/src/core/etl/src/Flow/ETL/Plan/Explain/Details.php
@@ -5,6 +5,7 @@
namespace Flow\ETL\Plan\Explain;
use Flow\ETL\Extractor\File\FileExtractor;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Join\Comparison\Any;
use Flow\ETL\Plan\Materialization;
use Flow\ETL\Plan\Node;
@@ -80,7 +81,7 @@ public function labelled(Node $node): array
$node instanceof Offset => ['Skip: ' . $node->offset],
$node instanceof TopN => ['Top: ' . $node->limit],
$node instanceof Rename => [sprintf('Rename: %s → %s', $node->from, $node->to)],
- $node instanceof Select => ['Columns: ' . $this->columns($node->entries)],
+ $node instanceof Select => ['Keeps only: ' . $this->columns($node->entries)],
$node instanceof Drop => ['Drops: ' . $this->columns($node->entries)],
$node instanceof Distinct => [
'Distinct on: ' . ($node->entries === [] ? 'every column' : $this->columns($node->entries)),
@@ -206,15 +207,29 @@ public function read(Read $read): array
$lines[] = 'Source: ' . $extractor->source()->uri();
}
- $limit = $read->limit();
+ return [...$lines, ...$this->scan($read->scan())];
+ }
+
+ /**
+ * What the plan pushed into a read, each conjunct the source applies on its own line.
+ *
+ * @return list
+ */
+ public function scan(FileScan $scan): array
+ {
+ $lines = [];
- if ($limit !== null) {
- $lines[] = 'Limit: ' . $limit;
+ if ($scan->limit !== null) {
+ $lines[] = 'Limit: ' . $scan->limit;
}
// every file source reads only files unless a filter was pushed into it
- if (!$read->pathFilter() instanceof OnlyFiles) {
- $lines[] = 'Files: ' . $this->name($read->pathFilter());
+ if (!$scan->pathFilter instanceof OnlyFiles) {
+ $lines[] = 'Files: ' . $this->name($scan->pathFilter);
+ }
+
+ foreach ($scan->conjuncts as $conjunct) {
+ $lines[] = 'Filter: ' . $this->name($conjunct);
}
return $lines;
@@ -234,10 +249,10 @@ public function notes(Node $node): array
$notes[] = 'Buffers all rows before passing them on';
}
- if ($redefines->unknown) {
- $notes[] = 'Defines columns known only at run time';
- } elseif ($redefines->names !== []) {
- $notes[] = 'Defines columns: ' . implode(', ', $redefines->names);
+ if ($redefines->isUnknown()) {
+ $notes[] = 'Changes columns known only at run time';
+ } elseif (!$redefines->isAllBut() && $redefines->names !== []) {
+ $notes[] = 'Changes columns: ' . implode(', ', $redefines->names);
}
return $notes;
@@ -255,8 +270,10 @@ public function declarations(Node $node): string
$node->materialization()->name,
];
- if ($redefines->unknown) {
+ if ($redefines->isUnknown()) {
$declarations[] = 'redefines unknown';
+ } elseif ($redefines->isAllBut()) {
+ $declarations[] = 'redefines all but ' . implode(', ', $redefines->names);
} elseif ($redefines->names !== []) {
$declarations[] = 'redefines ' . implode(', ', $redefines->names);
}
diff --git a/src/core/etl/src/Flow/ETL/Plan/Explain/PhysicalOutline.php b/src/core/etl/src/Flow/ETL/Plan/Explain/PhysicalOutline.php
index cea5a9af82..3c893d8e6e 100644
--- a/src/core/etl/src/Flow/ETL/Plan/Explain/PhysicalOutline.php
+++ b/src/core/etl/src/Flow/ETL/Plan/Explain/PhysicalOutline.php
@@ -9,8 +9,8 @@
use Flow\ETL\Executor\Pipeline;
use Flow\ETL\Processor\HashJoinProcessor;
use Flow\ETL\Transformer\CrossJoinRowsTransformer;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
+use function array_map;
use function implode;
final readonly class PhysicalOutline
@@ -94,18 +94,7 @@ public function joined(PhysicalPlan $right): Entry
*/
public function source(Pipeline $pipeline): array
{
- $lines = [];
- $limit = $pipeline->limit();
-
- if ($limit !== null) {
- $lines[] = ' Limit: ' . $limit;
- }
-
- if (!$pipeline->pathFilter() instanceof OnlyFiles) {
- $lines[] = ' Files: ' . $this->details->name($pipeline->pathFilter());
- }
-
- return $lines;
+ return array_map(static fn(string $line): string => ' ' . $line, $this->details->scan($pipeline->scan()));
}
/**
diff --git a/src/core/etl/src/Flow/ETL/Plan/Node.php b/src/core/etl/src/Flow/ETL/Plan/Node.php
index 9b2e4d0129..e9d0a3c87d 100644
--- a/src/core/etl/src/Flow/ETL/Plan/Node.php
+++ b/src/core/etl/src/Flow/ETL/Plan/Node.php
@@ -54,7 +54,7 @@ public function transparency(): Transparency;
public function materialization(): Materialization;
/**
- * Columns this node introduces or renames on its output. A predicate that references one of them means a
+ * Columns this node introduces, renames or removes on its output. A predicate that references one of them means a
* different column below this node, so it cannot be pushed past it. Answered from what the node holds,
* never from a class list.
*/
diff --git a/src/core/etl/src/Flow/ETL/Plan/Node/Drop.php b/src/core/etl/src/Flow/ETL/Plan/Node/Drop.php
index 0b3360c587..e35cac6df4 100644
--- a/src/core/etl/src/Flow/ETL/Plan/Node/Drop.php
+++ b/src/core/etl/src/Flow/ETL/Plan/Node/Drop.php
@@ -10,6 +10,7 @@
use Flow\ETL\Plan\RowCount;
use Flow\ETL\Plan\Transparency;
use Flow\ETL\Row\Reference;
+use Flow\ETL\Row\References;
final readonly class Drop implements Node
{
@@ -51,6 +52,6 @@ public function materialization(): Materialization
public function redefines(): Redefined
{
- return Redefined::none();
+ return Redefined::names(...References::init(...$this->entries)->names());
}
}
diff --git a/src/core/etl/src/Flow/ETL/Plan/Node/Read.php b/src/core/etl/src/Flow/ETL/Plan/Node/Read.php
index f94adb446e..5a3de95d3c 100644
--- a/src/core/etl/src/Flow/ETL/Plan/Node/Read.php
+++ b/src/core/etl/src/Flow/ETL/Plan/Node/Read.php
@@ -7,6 +7,9 @@
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\Exception\SchemaNotDerivableException;
use Flow\ETL\Extractor;
+use Flow\ETL\Extractor\File\FileExtractor;
+use Flow\ETL\Extractor\File\FileScan;
+use Flow\ETL\Function\ScalarFunction;
use Flow\ETL\Plan\Materialization;
use Flow\ETL\Plan\Node;
use Flow\ETL\Plan\Redefined;
@@ -14,22 +17,17 @@
use Flow\ETL\Plan\Transparency;
use Flow\ETL\Schema;
use Flow\Filesystem\Path\Filter;
-use Flow\Filesystem\Path\Filter\Filters;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
-
-use function min;
final readonly class Read implements Node
{
/**
* @param Extractor $extractor the instance the user handed to read(); the planner never mutates or copies it
- * @param null|int<1, max> $limit the pushed limit, handed to Extractor::extract() when the plan runs
- * @param Filter $pathFilter the pushed partition filter, handed to FileExtractor::extract() when the plan runs
+ * @param FileScan $scan what the plan pushed into the read, handed to FileExtractor::scan() when the plan
+ * runs; a non-file extractor receives only its limit
*/
public function __construct(
private Extractor $extractor,
- private ?int $limit = null,
- private Filter $pathFilter = new OnlyFiles(),
+ private FileScan $scan = new FileScan(),
) {}
public function extractor(): Extractor
@@ -37,28 +35,17 @@ public function extractor(): Extractor
return $this->extractor;
}
- /**
- * @return null|int<1, max>
- */
- public function limit(): ?int
+ public function scan(): FileScan
{
- return $this->limit;
+ return $this->scan;
}
- public function pathFilter(): Filter
- {
- return $this->pathFilter;
- }
-
- public function withPathFilter(Filter $filter): self
+ /**
+ * @throws InvalidArgumentException when the extractor is no FileExtractor, which has nowhere to apply it
+ */
+ public function withFilter(ScalarFunction $filter): self
{
- return new self(
- $this->extractor,
- $this->limit,
- $this->pathFilter instanceof Filters
- ? $this->pathFilter->add($filter)
- : new Filters($this->pathFilter, $filter),
- );
+ return new self($this->fileExtractor(), $this->scan->withFilter($filter));
}
/**
@@ -68,15 +55,15 @@ public function withPathFilter(Filter $filter): self
*/
public function withLimit(int $limit): self
{
- if ($limit <= 0) {
- throw new InvalidArgumentException('Limit must be greater than 0');
- }
-
- return new self(
- $this->extractor,
- $this->limit === null ? $limit : min($this->limit, $limit),
- $this->pathFilter,
- );
+ return new self($this->extractor, $this->scan->withLimit($limit));
+ }
+
+ /**
+ * @throws InvalidArgumentException when the extractor is no FileExtractor, which has nowhere to apply it
+ */
+ public function withPathFilter(Filter $filter): self
+ {
+ return new self($this->fileExtractor(), $this->scan->withPathFilter($filter));
}
/**
@@ -119,4 +106,13 @@ public function redefines(): Redefined
{
return Redefined::none();
}
+
+ private function fileExtractor(): FileExtractor
+ {
+ return $this->extractor instanceof FileExtractor
+ ? $this->extractor
+ : throw new InvalidArgumentException(
+ 'Only a FileExtractor reads a pushed filter, ' . $this->extractor::class . ' does not',
+ );
+ }
}
diff --git a/src/core/etl/src/Flow/ETL/Plan/Node/Rename.php b/src/core/etl/src/Flow/ETL/Plan/Node/Rename.php
index c3f29e3ee8..c7806345eb 100644
--- a/src/core/etl/src/Flow/ETL/Plan/Node/Rename.php
+++ b/src/core/etl/src/Flow/ETL/Plan/Node/Rename.php
@@ -48,6 +48,6 @@ public function materialization(): Materialization
public function redefines(): Redefined
{
- return Redefined::alias($this->to, $this->from);
+ return Redefined::rename($this->from, $this->to);
}
}
diff --git a/src/core/etl/src/Flow/ETL/Plan/Node/Select.php b/src/core/etl/src/Flow/ETL/Plan/Node/Select.php
index 0221ceca07..c651bf7a17 100644
--- a/src/core/etl/src/Flow/ETL/Plan/Node/Select.php
+++ b/src/core/etl/src/Flow/ETL/Plan/Node/Select.php
@@ -10,6 +10,7 @@
use Flow\ETL\Plan\RowCount;
use Flow\ETL\Plan\Transparency;
use Flow\ETL\Row\Reference;
+use Flow\ETL\Row\References;
final readonly class Select implements Node
{
@@ -51,6 +52,6 @@ public function materialization(): Materialization
public function redefines(): Redefined
{
- return Redefined::none();
+ return Redefined::allBut(...References::init(...$this->entries)->names());
}
}
diff --git a/src/core/etl/src/Flow/ETL/Plan/Redefined.php b/src/core/etl/src/Flow/ETL/Plan/Redefined.php
index b64220a352..4eafeeacb7 100644
--- a/src/core/etl/src/Flow/ETL/Plan/Redefined.php
+++ b/src/core/etl/src/Flow/ETL/Plan/Redefined.php
@@ -15,19 +15,27 @@
* @param array $aliases a redefined name => the column below the node it only renames
*/
private function __construct(
- public bool $unknown,
+ private Redefinition $redefinition,
public array $names,
private array $aliases,
) {}
public static function none(): self
{
- return new self(false, [], []);
+ return new self(Redefinition::names, [], []);
}
public static function names(string ...$names): self
{
- return new self(false, array_values($names), []);
+ return new self(Redefinition::names, array_values($names), []);
+ }
+
+ /**
+ * The node keeps only $kept (Select): every other column is gone above it.
+ */
+ public static function allBut(string ...$kept): self
+ {
+ return new self(Redefinition::allBut, array_values($kept), []);
}
/**
@@ -36,7 +44,15 @@ public static function names(string ...$names): self
*/
public static function alias(string $name, string $below): self
{
- return new self(false, [$name], [$name => $below]);
+ return new self(Redefinition::names, [$name], [$name => $below]);
+ }
+
+ /**
+ * $to holds exactly the values of $from, which is gone above the node.
+ */
+ public static function rename(string $from, string $to): self
+ {
+ return new self(Redefinition::names, [$to, $from], [$to => $from]);
}
/**
@@ -44,12 +60,26 @@ public static function alias(string $name, string $below): self
*/
public static function unknown(): self
{
- return new self(true, [], []);
+ return new self(Redefinition::unknown, [], []);
}
public function defines(string $name): bool
{
- return $this->unknown || in_array($name, $this->names, true);
+ return match ($this->redefinition) {
+ Redefinition::unknown => true,
+ Redefinition::allBut => !in_array($name, $this->names, true),
+ Redefinition::names => in_array($name, $this->names, true),
+ };
+ }
+
+ public function isAllBut(): bool
+ {
+ return $this->redefinition === Redefinition::allBut;
+ }
+
+ public function isUnknown(): bool
+ {
+ return $this->redefinition === Redefinition::unknown;
}
/**
diff --git a/src/core/etl/src/Flow/ETL/Plan/Redefinition.php b/src/core/etl/src/Flow/ETL/Plan/Redefinition.php
new file mode 100644
index 0000000000..51dedd4f57
--- /dev/null
+++ b/src/core/etl/src/Flow/ETL/Plan/Redefinition.php
@@ -0,0 +1,12 @@
+target ? $this->replacement : $node;
- }
-}
diff --git a/src/core/etl/src/Flow/ETL/Plan/ReplaceNodes.php b/src/core/etl/src/Flow/ETL/Plan/ReplaceNodes.php
new file mode 100644
index 0000000000..2074e2e685
--- /dev/null
+++ b/src/core/etl/src/Flow/ETL/Plan/ReplaceNodes.php
@@ -0,0 +1,59 @@
+ $replacements target => replacement, which takes the target's rewritten
+ * children
+ * @param list $removals nodes of one child, each giving way to its rewritten child
+ */
+ public function __construct(
+ private SplObjectStorage $replacements,
+ private array $removals = [],
+ ) {
+ foreach ($removals as $removal) {
+ if ($replacements->offsetExists($removal)) {
+ throw new InvalidArgumentException(
+ 'A node is either replaced or removed, ' . $removal::class . ' is both',
+ );
+ }
+ }
+ }
+
+ public static function one(Node $target, Node $replacement): self
+ {
+ /** @var SplObjectStorage $replacements */
+ $replacements = new SplObjectStorage();
+ $replacements[$target] = $replacement;
+
+ return new self($replacements);
+ }
+
+ public function of(Node $original, Node $rebuilt): Node
+ {
+ if (in_array($original, $this->removals, true)) {
+ return count($rebuilt->children()) === 1
+ ? $rebuilt->children()[0]
+ : throw new InvalidArgumentException(
+ 'Only a node of one child gives way to it, ' . $original::class . ' has '
+ . count($rebuilt->children()),
+ );
+ }
+
+ return $this->replacements->offsetExists($original)
+ ? $this->replacements[$original]->withChildren($rebuilt->children())
+ : $rebuilt;
+ }
+}
diff --git a/src/core/etl/src/Flow/ETL/Plan/Rewrite.php b/src/core/etl/src/Flow/ETL/Plan/Rewrite.php
index 5b60893224..b466330d12 100644
--- a/src/core/etl/src/Flow/ETL/Plan/Rewrite.php
+++ b/src/core/etl/src/Flow/ETL/Plan/Rewrite.php
@@ -7,10 +7,11 @@
interface Rewrite
{
/**
- * The replacement for this node, or the node itself when nothing changes.
+ * The replacement for $original, given as $rebuilt over its rewritten children; $rebuilt itself when nothing
+ * changes.
*
* Apply it to a plan through LogicalPlan::transformUp(): its memo rewrites a node every consumer shares
* exactly once, so the shared prefix stays shared. It never reaches into a join's right side.
*/
- public function of(Node $node): Node;
+ public function of(Node $original, Node $rebuilt): Node;
}
diff --git a/src/core/etl/src/Flow/ETL/Plan/TransformUp.php b/src/core/etl/src/Flow/ETL/Plan/TransformUp.php
index 2a674b80c7..f3d2db87ca 100644
--- a/src/core/etl/src/Flow/ETL/Plan/TransformUp.php
+++ b/src/core/etl/src/Flow/ETL/Plan/TransformUp.php
@@ -41,6 +41,6 @@ public function of(Node $node, Rewrite $rewrite): Node
$children[$i] = $this->of($children[$i], $rewrite);
}
- return $this->memo[$node] = $rewrite->of($node->withChildren($children));
+ return $this->memo[$node] = $rewrite->of($node, $node->withChildren($children));
}
}
diff --git a/src/core/etl/src/Flow/ETL/Planner/PipelineSplit.php b/src/core/etl/src/Flow/ETL/Planner/PipelineSplit.php
index fb5645b886..fcd00eb8d8 100644
--- a/src/core/etl/src/Flow/ETL/Planner/PipelineSplit.php
+++ b/src/core/etl/src/Flow/ETL/Planner/PipelineSplit.php
@@ -11,11 +11,11 @@
use Flow\ETL\Executor\Raw;
use Flow\ETL\Executor\Segments;
use Flow\ETL\Executor\SourceRows;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\FlowContext;
use Flow\ETL\Plan\LogicalPlan;
use Flow\ETL\Plan\Materialization;
use Flow\ETL\Plan\Node;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
use SplObjectStorage;
use function array_reverse;
@@ -59,8 +59,7 @@ public function of(
$read = $logical->source();
$input = null;
$segments = new Segments($read->extractor());
- $limit = $read->limit();
- $pathFilter = $read->pathFilter();
+ $scan = $read->scan();
$attachment = new SinkAttachment($planned, $context);
$remembered = $attachment->attach($logical->sinksOnSpine(), $onSpine);
@@ -76,15 +75,14 @@ public function of(
}
if ($node->materialization() === Materialization::blocking && $node !== $top) {
- $input = new Pipeline($id++, $segments, $context, $input, $limit, $pathFilter, $sources);
+ $input = new Pipeline($id++, $segments, $context, $input, $scan, $sources);
$segments = new Segments();
- $limit = null;
- $pathFilter = new OnlyFiles();
+ $scan = new FileScan();
$sources = null;
}
}
- $pipeline = new Pipeline($id, $segments, $context, $input, $limit, $pathFilter, $sources);
+ $pipeline = new Pipeline($id, $segments, $context, $input, $scan, $sources);
$refusal = $planned->refusal();
return $refusal === null
diff --git a/src/core/etl/src/Flow/ETL/Schema/Definition/StringDefinition.php b/src/core/etl/src/Flow/ETL/Schema/Definition/StringDefinition.php
index e17bb917cd..8a2e04dce3 100644
--- a/src/core/etl/src/Flow/ETL/Schema/Definition/StringDefinition.php
+++ b/src/core/etl/src/Flow/ETL/Schema/Definition/StringDefinition.php
@@ -29,14 +29,18 @@
*/
private Type $type;
+ /**
+ * @param null|Type $type
+ */
public function __construct(
string|Reference $ref,
private bool $nullable = false,
?Metadata $metadata = null,
+ ?Type $type = null,
) {
$this->ref = UnresolvedReference::init($ref);
$this->metadata = $metadata ?? Metadata::empty();
- $this->type = type_string();
+ $this->type = $type ?? type_string();
}
/**
@@ -44,7 +48,7 @@ public function __construct(
*/
public function addMetadata(string $key, int|string|bool|float|array $value): static
{
- return new self($this->ref, $this->nullable, $this->metadata->add($key, $value));
+ return new self($this->ref, $this->nullable, $this->metadata->add($key, $value), $this->type);
}
public function entry(): Reference
@@ -85,7 +89,7 @@ public function isSame(Definition $definition): bool
public function makeNullable(bool $nullable = true): static
{
- return new self($this->ref, $nullable, $this->metadata);
+ return new self($this->ref, $nullable, $this->metadata, $this->type);
}
public function matches(mixed $value): bool
@@ -111,6 +115,7 @@ public function merge(Definition $definition): Definition
$this->ref,
$this->nullable || $definition->isNullable(),
$this->metadata->merge($definition->metadata()),
+ type_equals($this->type, $definition->type()) ? $this->type : type_string(),
);
}
@@ -134,12 +139,12 @@ public function normalize(): array
public function rename(string $newName): static
{
- return new self($newName, $this->nullable, $this->metadata);
+ return new self($newName, $this->nullable, $this->metadata, $this->type);
}
public function setMetadata(Metadata $metadata): static
{
- return new self($this->ref, $this->nullable, $metadata);
+ return new self($this->ref, $this->nullable, $metadata, $this->type);
}
public function type(): Type
diff --git a/src/core/etl/src/Flow/ETL/Schema/Formatter/PHPFormatter/TypeFormatter.php b/src/core/etl/src/Flow/ETL/Schema/Formatter/PHPFormatter/TypeFormatter.php
index 3be4378cc4..2026001445 100644
--- a/src/core/etl/src/Flow/ETL/Schema/Formatter/PHPFormatter/TypeFormatter.php
+++ b/src/core/etl/src/Flow/ETL/Schema/Formatter/PHPFormatter/TypeFormatter.php
@@ -12,6 +12,7 @@
use Flow\Types\Type\Logical\JsonType;
use Flow\Types\Type\Logical\ListType;
use Flow\Types\Type\Logical\MapType;
+use Flow\Types\Type\Logical\NumericStringType;
use Flow\Types\Type\Logical\OptionalType;
use Flow\Types\Type\Logical\StructureType;
use Flow\Types\Type\Logical\TimeType;
@@ -109,6 +110,7 @@ private function formatSimpleType(Type $type, bool $nullable): string
$reflection = match ($type::class) {
ArrayType::class => new ReflectionFunction('\\Flow\\Types\\DSL\\type_array'),
StringType::class => new ReflectionFunction('\\Flow\\Types\\DSL\\type_string'),
+ NumericStringType::class => new ReflectionFunction('\\Flow\\Types\\DSL\\type_numeric_string'),
IntegerType::class => new ReflectionFunction('\\Flow\\Types\\DSL\\type_integer'),
BooleanType::class => new ReflectionFunction('\\Flow\\Types\\DSL\\type_boolean'),
FloatType::class => new ReflectionFunction('\\Flow\\Types\\DSL\\type_float'),
diff --git a/src/core/etl/src/Flow/ETL/Schema/Formatter/PHPSchemaFormatter.php b/src/core/etl/src/Flow/ETL/Schema/Formatter/PHPSchemaFormatter.php
index ee33dae1eb..2ba94537fd 100644
--- a/src/core/etl/src/Flow/ETL/Schema/Formatter/PHPSchemaFormatter.php
+++ b/src/core/etl/src/Flow/ETL/Schema/Formatter/PHPSchemaFormatter.php
@@ -19,6 +19,7 @@
use Flow\Types\Type\Logical\JsonType;
use Flow\Types\Type\Logical\ListType;
use Flow\Types\Type\Logical\MapType;
+use Flow\Types\Type\Logical\NumericStringType;
use Flow\Types\Type\Logical\StructureType;
use Flow\Types\Type\Logical\TimeType;
use Flow\Types\Type\Logical\TimeZoneType;
@@ -152,6 +153,7 @@ private function formatSchema(Schema $schema, int $level = 1): string
DateTimeType::class => $this->dateTimeType($definition),
FloatType::class => $this->floatType($definition),
EnumType::class => $this->enumType($definition),
+ NumericStringType::class => $this->numericStringType($definition),
ListType::class => $this->listType($definition),
MapType::class => $this->mapType($definition),
StructureType::class => $this->structureType($definition),
@@ -201,6 +203,23 @@ private function mapType(Definition $definition): string
);
}
+ /**
+ * @param Definition $definition
+ */
+ private function numericStringType(Definition $definition): string
+ {
+ $reflection = new ReflectionFunction("\Flow\ETL\DSL\\definition_from_type");
+
+ return sprintf(
+ '\%s("%s", type: %s, nullable: %s, metadata: %s)',
+ $reflection->getName(),
+ $definition->entry()->name(),
+ $this->typeFormatter->format($definition->type()),
+ $definition->isNullable() ? 'true' : 'false',
+ $this->formatMetadata($definition->metadata()),
+ );
+ }
+
/**
* @param Definition $definition
*/
diff --git a/src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionFilterTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionFilterTransformer.php
index c7a9b6ef23..36772860c5 100644
--- a/src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionFilterTransformer.php
+++ b/src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionFilterTransformer.php
@@ -8,33 +8,29 @@
use Flow\ETL\Config\Telemetry\TelemetryAttributes;
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\FlowContext;
-use Flow\ETL\Function\ExpandingFunctions;
-use Flow\ETL\Function\ReferenceResolver;
+use Flow\ETL\Function\Evaluation\Filter;
use Flow\ETL\Function\ScalarFunction;
use Flow\ETL\Rows;
use Flow\ETL\Schema;
use Flow\ETL\Transformer;
use Throwable;
-use function array_keys;
-use function Flow\Types\DSL\type_bare;
-use function Flow\Types\DSL\type_boolean;
-use function Flow\Types\DSL\type_equals;
-use function sprintf;
-
final readonly class ScalarFunctionFilterTransformer implements Transformer
{
/**
- * @param null|ScalarFunction $resolved the predicate resolved against the bound schema; only bind() sets it
+ * @param null|Filter $filter the predicate bound to the input schema; only bind() sets it
*/
public function __construct(
public ScalarFunction $function,
- private ?ScalarFunction $resolved = null,
+ private ?Filter $filter = null,
) {}
+ /**
+ * @throws InvalidArgumentException
+ */
public function bind(Schema $input): BoundStep
{
- return new BoundStep(new self($this->function, $this->resolve($input)), $input);
+ return new BoundStep(new self($this->function, Filter::bind($input, $this->function)), $input);
}
public function transform(Rows $rows, FlowContext $context): Rows
@@ -42,8 +38,7 @@ public function transform(Rows $rows, FlowContext $context): Rows
$context->telemetry()->transformationStarted($this);
try {
- $function = $this->resolved ?? $this->resolve($rows->schema());
- $result = $rows->gather(array_keys($function->eval($rows, $context)->physicals(), true, true));
+ $result = ($this->filter ?? Filter::bind($rows->schema(), $this->function))->filter($rows, $context);
$context->telemetry()->transformationCompleted($this, [
TelemetryAttributes::ATTR_TRANSFORMATION_INPUT_ROWS => $rows->count(),
@@ -57,28 +52,4 @@ public function transform(Rows $rows, FlowContext $context): Rows
throw $e;
}
}
-
- /**
- * @throws InvalidArgumentException
- */
- private function resolve(Schema $input): ScalarFunction
- {
- $resolver = new ReferenceResolver();
- $resolved = $resolver->resolve($this->function, $input);
- $resolver->assertResolved($resolved, $input);
- (new ExpandingFunctions())->refuse($resolved, 'filter');
-
- // type_bare() keeps the gate blind to nullability - a null-propagating predicate declares
- // ?boolean, and (bool) null === false is how a NULL row is dropped.
- if (!type_equals(type_bare($resolved->returns()), type_boolean())) {
- throw new InvalidArgumentException(sprintf(
- 'filter() requires a predicate returning boolean, "%s" returns "%s". '
- . 'Use an explicit comparison, e.g. ->notEquals(lit(0)).',
- $resolved::class,
- $resolved->returns()->toString(),
- ));
- }
-
- return $resolved;
- }
}
diff --git a/src/core/etl/src/Flow/Floe/FloeExtractor.php b/src/core/etl/src/Flow/Floe/FloeExtractor.php
index 1aa43bd420..d65e2f3ffb 100644
--- a/src/core/etl/src/Flow/Floe/FloeExtractor.php
+++ b/src/core/etl/src/Flow/Floe/FloeExtractor.php
@@ -13,8 +13,10 @@
use Flow\ETL\Extractor\File\FileExtractor;
use Flow\ETL\Extractor\File\FileReading;
use Flow\ETL\Extractor\File\FileReadLoop;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\File\MetadataColumnsExtractor;
use Flow\ETL\Extractor\File\ReadWindow;
+use Flow\ETL\Extractor\File\Scanning;
use Flow\ETL\Extractor\RewindableExtractor;
use Flow\ETL\Extractor\Signal;
use Flow\ETL\Extractor\Statistics;
@@ -24,8 +26,6 @@
use Flow\Filesystem\Filesystem;
use Flow\Filesystem\Local\NativeLocalFilesystem;
use Flow\Filesystem\Path;
-use Flow\Filesystem\Path\Filter;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
use Flow\Floe\Codec\NoopCodec;
use Generator;
@@ -42,6 +42,7 @@ final class FloeExtractor implements
{
use Batches;
use FileReading;
+ use Scanning;
private ?int $offset = null;
@@ -78,12 +79,12 @@ public function isRepeatable(): bool
/**
* @return Generator
*/
- public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator
+ public function scan(FlowContext $context, FileScan $scan): Generator
{
$fileColumns = $this->fileColumns($this->filesystem, $this->path);
- yield from (new FileReadLoop($fileColumns, $this->schema()))->read(
- iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $pathFilter), false),
+ yield from (new FileReadLoop($fileColumns, $this->schema(), $scan->conjuncts))->read(
+ iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $scan->pathFilter), false),
new FloeFileBatches(
$this->filesystem,
$this->codec,
@@ -94,8 +95,8 @@ public function extract(FlowContext $context, ?int $limit = null, Filter $pathFi
: new DerivedSchema($this->derivedSchema($this->files(), $this->unionByName), $this->derivedFrom),
),
$this->batchSize(),
- $context->backend(),
- new ReadWindow($this->offset ?? 0, $limit),
+ $context,
+ new ReadWindow($this->offset ?? 0, $scan->limit),
);
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/ExtractedRows.php b/src/core/etl/tests/Flow/ETL/Tests/Context/ExtractedRows.php
index 836b22c590..2a02c79b1e 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Context/ExtractedRows.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Context/ExtractedRows.php
@@ -5,11 +5,10 @@
namespace Flow\ETL\Tests\Context;
use Flow\ETL\Extractor;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\FlowContext;
use Flow\ETL\Rows;
use Flow\ETL\Tests\FlowTestCase;
-use Flow\Filesystem\Path\Filter;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
use function Flow\ETL\DSL\flow_context;
use function Flow\ETL\DSL\rows;
@@ -17,19 +16,12 @@
final class ExtractedRows
{
- /**
- * @param null|int<1, max> $limit
- */
- public static function of(
- Extractor $extractor,
- ?FlowContext $context = null,
- ?int $limit = null,
- Filter $pathFilter = new OnlyFiles(),
- ): Rows {
+ public static function of(Extractor $extractor, ?FlowContext $context = null, FileScan $scan = new FileScan()): Rows
+ {
$context ??= flow_context();
$extracted = null;
- foreach (FlowTestCase::extracted($extractor, $context, $limit, $pathFilter) as $batch) {
+ foreach (FlowTestCase::extracted($extractor, $context, $scan) as $batch) {
$extracted = $extracted === null ? $batch : $extracted->concat($context->backend(), $batch);
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/RecordingFileExtractor.php b/src/core/etl/tests/Flow/ETL/Tests/Double/RecordingFileExtractor.php
index 5e9cbe6011..f66fd572be 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Double/RecordingFileExtractor.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Double/RecordingFileExtractor.php
@@ -5,33 +5,31 @@
namespace Flow\ETL\Tests\Double;
use Flow\ETL\Extractor\File\FileExtractor;
+use Flow\ETL\Extractor\File\FileScan;
+use Flow\ETL\Extractor\File\Scanning;
use Flow\ETL\Extractor\Statistics;
use Flow\ETL\FlowContext;
+use Flow\ETL\Function\Evaluation\Filter;
use Flow\ETL\Rows;
use Flow\ETL\Schema;
use Flow\Filesystem\Path;
-use Flow\Filesystem\Path\Filter;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
use Generator;
use function array_values;
use function Flow\Filesystem\DSL\path;
/**
- * A file source that records the limit and path filter of every read, so a test can prove what the planner
- * pushed reached the read - and that the instance itself was left alone.
+ * A file source that records the scan of every read, so a test can prove what the planner pushed reached the
+ * read - and that the instance itself was left alone. It returns only the rows matching the scan's filter.
*/
final class RecordingFileExtractor implements FileExtractor
{
- /**
- * @var list
- */
- public array $limits = [];
+ use Scanning;
/**
- * @var list
+ * @var list
*/
- public array $pathFilters = [];
+ public array $scans = [];
/**
* @var list
@@ -48,13 +46,13 @@ public function __construct(
$this->partitions = new Schema();
}
- public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator
+ public function scan(FlowContext $context, FileScan $scan): Generator
{
- $this->limits[] = $limit;
- $this->pathFilters[] = $pathFilter;
+ $this->scans[] = $scan;
+ $filter = Filter::bind($this->schema, ...$scan->conjuncts);
foreach ($this->batches as $rows) {
- yield $rows;
+ yield $filter->filter($rows, $context);
}
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/RenameSelectRewrite.php b/src/core/etl/tests/Flow/ETL/Tests/Double/RenameSelectRewrite.php
index 223e428f12..5bc42d6fbe 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Double/RenameSelectRewrite.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Double/RenameSelectRewrite.php
@@ -11,8 +11,8 @@
final readonly class RenameSelectRewrite implements Rewrite
{
- public function of(Node $node): Node
+ public function of(Node $original, Node $rebuilt): Node
{
- return $node instanceof Select ? NodeMother::select($node->children()[0], 'name') : $node;
+ return $rebuilt instanceof Select ? NodeMother::select($rebuilt->children()[0], 'name') : $rebuilt;
}
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/SpineCopyingRule.php b/src/core/etl/tests/Flow/ETL/Tests/Double/SpineCopyingRule.php
index 6a243df2cd..614e6f6803 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Double/SpineCopyingRule.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Double/SpineCopyingRule.php
@@ -24,8 +24,8 @@ public function apply(LogicalPlan $plan, FlowContext $context): LogicalPlan
return new LogicalPlan((new TransformUp())->of($plan->root, $this));
}
- public function of(Node $node): Node
+ public function of(Node $original, Node $rebuilt): Node
{
- return $node instanceof Read ? new Read($node->extractor(), $node->limit(), $node->pathFilter()) : $node;
+ return $rebuilt instanceof Read ? new Read($rebuilt->extractor(), $rebuilt->scan()) : $rebuilt;
}
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php b/src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php
index a62327205e..5702d6f613 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php
@@ -8,12 +8,11 @@
use Flow\ETL\Extractor;
use Flow\ETL\Extractor\BatchableExtractor;
use Flow\ETL\Extractor\File\FileExtractor;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\RewindableExtractor;
use Flow\ETL\Extractor\Signal;
use Flow\ETL\FlowContext;
use Flow\ETL\Rows;
-use Flow\Filesystem\Path\Filter;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
use Generator;
use PHPUnit\Framework\TestCase;
use RuntimeException;
@@ -110,20 +109,18 @@ final public static function assertExtractorHonoursMaximum(
/**
* @param array $expectedArray
- * @param null|int<1, max> $limit
*/
final public static function assertExtractedRowsAsArrayEquals(
array $expectedArray,
Extractor $extractor,
?FlowContext $flowContext = null,
string $message = '',
- ?int $limit = null,
- Filter $pathFilter = new OnlyFiles(),
+ FileScan $scan = new FileScan(),
): void {
$flowContext ??= flow_context();
$extractedRows = rows(schema());
- foreach (self::extracted($extractor, $flowContext, $limit, $pathFilter) as $nextRows) {
+ foreach (self::extracted($extractor, $flowContext, $scan) as $nextRows) {
$extractedRows = $extractedRows->isEmpty()
? $nextRows
: $extractedRows->concat(new PhpBackend(), $nextRows);
@@ -132,42 +129,34 @@ final public static function assertExtractedRowsAsArrayEquals(
static::assertEquals($expectedArray, $extractedRows->toArray(), $message);
}
- /**
- * @param null|int<1, max> $limit
- */
final public static function assertExtractedRowsCount(
int $expectedCount,
Extractor $extractor,
?FlowContext $flowContext = null,
string $message = '',
- ?int $limit = null,
- Filter $pathFilter = new OnlyFiles(),
+ FileScan $scan = new FileScan(),
): void {
$flowContext ??= flow_context();
$totalRows = 0;
- foreach (self::extracted($extractor, $flowContext, $limit, $pathFilter) as $rows) {
+ foreach (self::extracted($extractor, $flowContext, $scan) as $rows) {
$totalRows += $rows->count();
}
static::assertSame($expectedCount, $totalRows, $message);
}
- /**
- * @param null|int<1, max> $limit
- */
final public static function assertExtractedRowsEquals(
Rows $expectedRows,
Extractor $extractor,
?FlowContext $flowContext = null,
string $message = '',
- ?int $limit = null,
- Filter $pathFilter = new OnlyFiles(),
+ FileScan $scan = new FileScan(),
): void {
$flowContext ??= flow_context();
$extractedRows = rows(schema());
- foreach (self::extracted($extractor, $flowContext, $limit, $pathFilter) as $nextRows) {
+ foreach (self::extracted($extractor, $flowContext, $scan) as $nextRows) {
$extractedRows = $extractedRows->isEmpty()
? $nextRows
: $extractedRows->concat(new PhpBackend(), $nextRows);
@@ -177,19 +166,13 @@ final public static function assertExtractedRowsEquals(
}
/**
- * @param null|int<1, max> $limit
- *
* @return Generator
*/
- final public static function extracted(
- Extractor $extractor,
- FlowContext $context,
- ?int $limit,
- Filter $pathFilter,
- ): Generator {
+ final public static function extracted(Extractor $extractor, FlowContext $context, FileScan $scan): Generator
+ {
return $extractor instanceof FileExtractor
- ? $extractor->extract($context, $limit, $pathFilter)
- : $extractor->extract($context, $limit);
+ ? $extractor->scan($context, $scan)
+ : $extractor->extract($context, $scan->limit);
}
public function repositoryRoot(): string
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/AnalyzeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/AnalyzeTest.php
index 9b149c6ff1..3cfd0fedfc 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/AnalyzeTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/AnalyzeTest.php
@@ -18,12 +18,17 @@
use Flow\ETL\Tests\Double\InlineLoader;
use Flow\ETL\Tests\FlowIntegrationTestCase;
use Flow\ETL\Tests\Mother\MarketRowsMother;
+use Flow\Parquet\Option;
+use Flow\Parquet\Options;
+use function Flow\ETL\Adapter\Parquet\from_parquet;
+use function Flow\ETL\Adapter\Parquet\to_parquet;
use function Flow\ETL\Adapter\Text\from_text;
use function Flow\ETL\DSL\analyze;
use function Flow\ETL\DSL\config_builder;
use function Flow\ETL\DSL\df;
use function Flow\ETL\DSL\from_array;
+use function Flow\ETL\DSL\from_sequence_number;
use function Flow\ETL\DSL\infer_schema;
use function Flow\ETL\DSL\int_schema;
use function Flow\ETL\DSL\join_on;
@@ -31,6 +36,8 @@
use function Flow\ETL\DSL\ref;
use function Flow\ETL\DSL\schema;
use function Flow\ETL\DSL\str_schema;
+use function Flow\Filesystem\DSL\memory_filesystem;
+use function Flow\Filesystem\DSL\path;
final class AnalyzeTest extends FlowIntegrationTestCase
{
@@ -216,6 +223,35 @@ public function test_a_source_read_under_a_pushed_limit_reports_no_rows_error():
static::assertNull($sources[0]->rowsError());
}
+ public function test_a_source_read_under_a_pushed_filter_reports_no_rows_error(): void
+ {
+ $memory = memory_filesystem();
+ df()
+ ->read(from_sequence_number('id', 1, 3000))
+ ->write(to_parquet(
+ path('memory://analyze/ids.parquet'),
+ Options::default()->set(Option::ROW_GROUP_SIZE_BYTES, 1)->set(
+ Option::ROW_GROUP_SIZE_CHECK_INTERVAL,
+ 1000,
+ ),
+ filesystem: $memory,
+ ))
+ ->run();
+
+ $report = df()
+ ->read(from_parquet(path('memory://analyze/ids.parquet'), filesystem: $memory))
+ ->filter(ref('id')->greaterThan(lit(2500)))
+ ->run(analyze: analyze()->withSourceStatistics());
+
+ static::assertNotNull($report);
+ $sources = $report->sources();
+ static::assertNotNull($sources);
+ static::assertCount(1, $sources);
+ static::assertFalse($sources[0]->complete);
+ static::assertLessThan(3000, $sources[0]->rows);
+ static::assertNull($sources[0]->rowsError());
+ }
+
public function test_run_explicit_analyze_overrides_config(): void
{
$config = config_builder()->analyze(analyze())->build();
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/FilterTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/FilterTest.php
index 48082a3f72..4006314426 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/FilterTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/FilterTest.php
@@ -6,8 +6,11 @@
use Flow\ETL\Tests\FlowIntegrationTestCase;
+use function Flow\ETL\Adapter\CSV\from_csv;
+use function Flow\ETL\Adapter\CSV\to_csv;
use function Flow\ETL\DSL\df;
use function Flow\ETL\DSL\from_array;
+use function Flow\ETL\DSL\lit;
use function Flow\ETL\DSL\ref;
final class FilterTest extends FlowIntegrationTestCase
@@ -58,4 +61,22 @@ public function test_single_filter(): void
$rows->toArray(),
);
}
+
+ public function test_a_filter_over_a_file_source_guards_the_filter_above_it(): void
+ {
+ $file = $this->cacheDir->path() . '/guarded-filters.csv';
+
+ df()
+ ->read(from_array([['a' => 1, 'b' => 0], ['a' => 2, 'b' => 2]]))
+ ->write(to_csv($file))
+ ->run();
+
+ $rows = df()
+ ->read(from_csv($file))
+ ->filter(ref('b')->notEquals(lit(0)))
+ ->filter(ref('a')->divide(ref('b'))->greaterThan(lit(0)))
+ ->fetch();
+
+ static::assertSame([['a' => 2, 'b' => 2]], $rows->toArray());
+ }
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/LimitTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/LimitTest.php
index 9e9e1003c1..96f93dc591 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/LimitTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/LimitTest.php
@@ -448,6 +448,7 @@ public function test_a_limit_over_a_read_frame_reaches_that_frames_source(bool $
$rows = df()->read($nested)->offset(1)->limit(2)->fetch();
static::assertSame([['id' => 2], ['id' => 3]], $rows->toArray());
- static::assertSame([3], $source->limits);
+ static::assertCount(1, $source->scans);
+ static::assertSame(3, $source->scans[0]->limit);
}
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinglePlanTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinglePlanTest.php
index e2f12340e5..794800a9eb 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinglePlanTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinglePlanTest.php
@@ -5,6 +5,7 @@
namespace Flow\ETL\Tests\Integration\DataFrame;
use Flow\ETL\ErrorHandler\IgnoreError;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Memory\ArrayMemory;
use Flow\ETL\Optimizer;
use Flow\ETL\Tests\Double\RecordingFileExtractor;
@@ -13,7 +14,6 @@
use Flow\ETL\Tests\Mother\PartitionedSourceMother;
use Flow\ETL\Transformation\AddRowIndex\StartFrom;
use Flow\ETL\Transformer\AddRowIndexTransformer;
-use Flow\Filesystem\Path\Filter;
use RuntimeException;
use function array_filter;
@@ -97,7 +97,8 @@ public function test_a_joined_frame_is_optimized_with_the_outer_frames_optimizer
->join($right, join_on(['id' => 'id'], 'r_'))
->fetch();
- static::assertSame([1], $source->limits);
+ static::assertCount(1, $source->scans);
+ static::assertSame(1, $source->scans[0]->limit);
}
public function test_a_joined_frames_own_error_handler_is_ignored(): void
@@ -145,11 +146,11 @@ public function test_a_filter_pushed_into_a_source_both_sides_share_leaves_the_j
->crossJoin($joined, 'j_')
->fetch();
- $accepts2024 = static fn(Filter $filter): bool => $filter->accept(PartitionedSourceMother::file(
+ $accepts2024 = static fn(FileScan $scan): bool => $scan->pathFilter->accept(PartitionedSourceMother::file(
'year=2024/month=08',
));
- static::assertCount(2, $source->pathFilters);
- static::assertCount(1, array_filter($source->pathFilters, $accepts2024));
+ static::assertCount(2, $source->scans);
+ static::assertCount(1, array_filter($source->scans, $accepts2024));
}
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootTest.php
index 6c3b61c2a4..726a8895fd 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootTest.php
@@ -414,7 +414,7 @@ public function test_a_later_limit_does_not_narrow_an_earlier_sink(): void
$rows = df()->read($extractor)->write($spy)->limit(2)->fetch();
static::assertCount(2, $rows);
- static::assertNull($extractor->limits[0]);
+ static::assertNull($extractor->scans[0]->limit);
static::assertSame([3], $spy->loadedRowCounts());
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/FileExtractorScanTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/FileExtractorScanTest.php
new file mode 100644
index 0000000000..2e664ad1ab
--- /dev/null
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/FileExtractorScanTest.php
@@ -0,0 +1,155 @@
+cacheDir->path() . '/scan-' . $family;
+ $uri = path($directory . '/rows.' . $family);
+ $xlsx = path($directory . '/rows.xlsx');
+
+ [$sink, $extractor] = match ($family) {
+ 'csv' => [to_csv($uri), from_csv($uri)],
+ 'json' => [to_json($uri), from_json($uri)],
+ 'text' => [to_text($uri), from_text($uri)],
+ 'xml' => [to_xml($uri), from_xml($uri)],
+ 'excel' => [to_excel($xlsx), from_excel($xlsx)],
+ 'parquet' => [to_parquet($uri), from_parquet($uri)],
+ 'floe' => [to_floe($uri), from_floe($uri)],
+ 'files' => [to_csv($uri), files($directory . '/*')],
+ default => [to_csv(path($directory . '/id=a/rows.csv')), from_path_partitions($directory . '/*/*.csv')],
+ };
+
+ df()
+ ->read(from_array([['id' => 'a'], ['id' => 'b'], ['id' => 'c']]))
+ ->write($sink)
+ ->run();
+
+ $context = flow_context();
+ $scanned = rows(schema());
+
+ foreach ($extractor->scan($context, new FileScan(limit: 2)) as $batch) {
+ $scanned = $scanned->isEmpty() ? $batch : $scanned->concat($context->backend(), $batch);
+ }
+
+ static::assertNotSame([], $scanned->toArray());
+ static::assertEquals(ExtractedRows::of($extractor, $context, new FileScan(2))->toArray(), $scanned->toArray());
+ }
+
+ #[TestWith(['csv'])]
+ #[TestWith(['json'])]
+ #[TestWith(['text'])]
+ #[TestWith(['xml'])]
+ #[TestWith(['excel'])]
+ #[TestWith(['parquet'])]
+ #[TestWith(['floe'])]
+ #[TestWith(['files'])]
+ #[TestWith(['path_partitions'])]
+ public function test_a_pushed_filter_returns_what_the_filter_node_returns(string $family): void
+ {
+ $directory = $this->cacheDir->path() . '/pushed-' . $family;
+ $uri = path($directory . '/rows.' . $family);
+ $xlsx = path($directory . '/rows.xlsx');
+
+ [$sink, $extractor, $predicate] = match ($family) {
+ 'csv' => [to_csv($uri), static fn(): FileExtractor => from_csv($uri), ref('id')->notEquals(lit('b'))],
+ 'json' => [to_json($uri), static fn(): FileExtractor => from_json($uri), ref('id')->notEquals(lit('b'))],
+ 'text' => [to_text($uri), static fn(): FileExtractor => from_text($uri), ref('text')->notEquals(lit('b'))],
+ 'xml' => [
+ to_xml($uri),
+ static fn(): FileExtractor => from_xml($uri, 'rows/row'),
+ ref('node')->xpath('id')->domElementValue()->notEquals(lit('b')),
+ ],
+ 'excel' => [
+ to_excel($xlsx),
+ static fn(): FileExtractor => from_excel($xlsx),
+ ref('id')->notEquals(lit('b')),
+ ],
+ 'parquet' => [
+ to_parquet($uri),
+ static fn(): FileExtractor => from_parquet($uri),
+ ref('id')->notEquals(lit('b')),
+ ],
+ 'floe' => [to_floe($uri), static fn(): FileExtractor => from_floe($uri), ref('id')->notEquals(lit('b'))],
+ 'files' => [null, static fn(): FileExtractor => files($directory . '/*/*'), ref('id')->notEquals(lit('b'))],
+ default => [
+ null,
+ static fn(): FileExtractor => from_path_partitions($directory . '/*/*'),
+ ref('id')->notEquals(lit('b')),
+ ],
+ };
+
+ if ($sink === null) {
+ foreach (['a', 'b', 'c'] as $id) {
+ df()
+ ->read(from_array([['id' => $id]]))
+ ->write(to_csv(path($directory . '/id=' . $id . '/rows.csv')))
+ ->run();
+ }
+ } else {
+ df()
+ ->read(from_array([['id' => 'a'], ['id' => 'b'], ['id' => 'c']]))
+ ->write($sink)
+ ->run();
+ }
+
+ $pushed = df()->read($extractor())->filter($predicate)->fetch();
+ $plain = df(config_builder()->optimizer(Optimizer::default()->without(PushFilterIntoSource::class)))
+ ->read($extractor())
+ ->filter($predicate)
+ ->fetch();
+
+ static::assertCount(2, $plain);
+ static::assertEquals($plain->toArray(), $pushed->toArray());
+ }
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php
index f048299c3a..4a3aefbd36 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php
@@ -6,6 +6,7 @@
use Flow\ETL\Cardinality;
use Flow\ETL\Column\PhpBackend;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\PathPartitionsExtractor;
use Flow\ETL\Rows;
use Flow\ETL\Tests\Context\ExtractedRows;
@@ -16,7 +17,9 @@
use function Flow\ETL\DSL\flow_context;
use function Flow\ETL\DSL\from_path_partitions;
+use function Flow\ETL\DSL\lit;
use function Flow\ETL\DSL\map_schema;
+use function Flow\ETL\DSL\ref;
use function Flow\ETL\DSL\rows;
use function Flow\ETL\DSL\schema;
use function Flow\ETL\DSL\str_schema;
@@ -243,4 +246,20 @@ public function test_path_partitions_extractor_honours_the_batch_contract(): voi
),
);
}
+
+ public function test_a_scan_returns_only_the_files_matching_its_filter(): void
+ {
+ $batches = iterator_to_array(from_path_partitions('memory://dir/**/*.txt', MemoryFiles::with([
+ 'memory://dir/year=2021/a.txt' => 'a',
+ 'memory://dir/year=2022/b.txt' => 'b',
+ 'memory://dir/year=2023/c.txt' => 'c',
+ 'memory://dir/year=2024/d.txt' => 'd',
+ 'memory://dir/year=2025/e.txt' => 'e',
+ ]))
+ ->withBatchSize(2)
+ ->scan(flow_context(), new FileScan(conjuncts: [ref('year')->isIn(lit(['2023', '2024']))])), false);
+
+ static::assertCount(1, $batches);
+ static::assertSame(['2023', '2024'], $batches[0]->column('year')->values());
+ }
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/PartitionedSourceMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/PartitionedSourceMother.php
index a6d27a9281..bbdbb81314 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Mother/PartitionedSourceMother.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/PartitionedSourceMother.php
@@ -22,8 +22,8 @@ public static function partitions(): Schema
}
/**
- * One row per partition. The double ignores the pushed path filter, so only a Filter node the planner kept can
- * drop the other partition's row.
+ * One row per partition. The double ignores the pushed path filter, so only the scan's filter or a Filter node
+ * the planner kept can drop the other partition's row.
*/
public static function yearMonth(): RecordingFileExtractor
{
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalForTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalForTest.php
index 42ee0b927d..77dcd65dd9 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalForTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalForTest.php
@@ -130,7 +130,7 @@ public static function supported(): Generator
foreach (self::types() as $name => [$type]) {
yield $name => [
$type,
- !in_array($name, ['positive integer', 'non-empty string', 'numeric string', 'class string'], true),
+ !in_array($name, ['positive integer', 'non-empty string', 'class string'], true),
];
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Telemetry/TelemetryContextTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Telemetry/TelemetryContextTest.php
index 83d3cdcb0c..06d2628298 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Telemetry/TelemetryContextTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Telemetry/TelemetryContextTest.php
@@ -990,8 +990,8 @@ public function test_optimizer_rules_are_logged(): void
[
CombineLimits::class,
CombineSortAndLimit::class,
- PushLimitIntoSource::class,
PushFilterIntoSource::class,
+ PushLimitIntoSource::class,
CountFromStatistics::class,
PushProjectionIntoExpand::class,
],
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/DataFrameTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/DataFrameTest.php
index e9e50824f2..9a7205e89f 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/DataFrameTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/DataFrameTest.php
@@ -958,9 +958,9 @@ public function test_a_pushed_limit_reaches_extract_and_leaves_the_users_extract
static::assertSame([['id' => 1, 'doubled' => 2]], $frame->fetch()->toArray());
static::assertSame([['id' => 1, 'doubled' => 2]], $frame->fetch()->toArray());
- static::assertCount(2, $extractor->limits);
- static::assertSame(1, $extractor->limits[0]);
- static::assertSame(1, $extractor->limits[1]);
+ static::assertCount(2, $extractor->scans);
+ static::assertSame(1, $extractor->scans[0]->limit);
+ static::assertSame(1, $extractor->scans[1]->limit);
static::assertStringEndsWith(
"#1 Read\n Extractor: RecordingFileExtractor\n Source: file:///dev/null",
$frame->explain()->toString(Stage::unoptimized),
@@ -985,7 +985,7 @@ public function test_a_limit_inside_a_joins_right_side_is_pushed_into_that_sourc
->join($right, join_on(['id' => 'id'], 'r_'))
->fetch();
- static::assertSame(1, $extractor->limits[0]);
+ static::assertSame(1, $extractor->scans[0]->limit);
}
public function test_an_empty_fetch_plans_once(): void
@@ -1112,7 +1112,7 @@ public function test_a_limit_inside_a_read_frame_is_pushed_into_its_source(): vo
$inner = df()->read($extractor)->limit(1);
static::assertSame([['id' => 1]], df()->read(from_data_frame($inner))->fetch()->toArray());
- static::assertSame(1, $extractor->limits[0]);
+ static::assertSame(1, $extractor->scans[0]->limit);
}
public function test_explain_without_sinks_prints_one_result_over_the_chain(): void
@@ -1122,8 +1122,8 @@ public function test_explain_without_sinks_prints_one_result_over_the_chain(): v
static::assertSame(<<<'PLAN'
#3 Result preserving · transparent · streaming
│ Rows this plan hands out: to the trigger, or to the node reading it
- └─ #2 Select preserving · transparent · streaming
- │ Columns: id
+ └─ #2 Select preserving · transparent · streaming · redefines all but id
+ │ Keeps only: id
└─ #1 Read source · transparent · streaming
Extractor: ArrayExtractor
PLAN, $dataFrame->explain()->toString(format: Format::declarations));
@@ -1140,8 +1140,8 @@ public function test_explain_with_a_sink_prints_the_outputs_sharing_the_chain():
Outputs preserving · opaque · streaming
├─ #3 Result preserving · transparent · streaming
│ │ Rows this plan hands out: to the trigger, or to the node reading it
- │ └─ #2 Select preserving · transparent · streaming
- │ │ Columns: id
+ │ └─ #2 Select preserving · transparent · streaming · redefines all but id
+ │ │ Keeps only: id
│ └─ #1 Read source · transparent · streaming
│ Extractor: ArrayExtractor
└─ #4 Write preserving · opaque · streaming
@@ -1173,8 +1173,8 @@ public function test_explain_of_the_run_trigger_keeps_a_result_over_a_chain_end_
Outputs preserving · opaque · streaming
├─ #3 Result preserving · transparent · streaming
│ │ Rows this plan hands out: to the trigger, or to the node reading it
- │ └─ #2 Select preserving · transparent · streaming
- │ │ Columns: id
+ │ └─ #2 Select preserving · transparent · streaming · redefines all but id
+ │ │ Keeps only: id
│ └─ #1 Read source · transparent · streaming
│ Extractor: ArrayExtractor
└─ #4 Write preserving · opaque · streaming
@@ -1203,15 +1203,14 @@ public function test_explain_prints_the_logical_plan_without_reading_a_row(): vo
$dataFrame = df()->read($extractor)->filter(ref('id')->isNotNull())->write(to_memory(new ArrayMemory()));
static::assertSame(<<<'PLAN'
- #3 Write preserving · opaque · streaming
+ #2 Write preserving · opaque · streaming
│ Loader: MemoryLoader
- └─ #2 Filter reducing · transparent · streaming
- │ Condition: IsNotNull
- └─ #1 Read source · transparent · streaming
- Extractor: RecordingFileExtractor
- Source: file:///dev/null
+ └─ #1 Read source · transparent · streaming
+ Extractor: RecordingFileExtractor
+ Source: file:///dev/null
+ Filter: IsNotNull
PLAN, $dataFrame->explain()->toString(format: Format::declarations));
- static::assertSame([], $extractor->limits);
+ static::assertSame([], $extractor->scans);
}
public function test_from_data_frame_reads_the_frame_as_a_source(): void
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/PipelineTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/PipelineTest.php
index c7da1b6421..21a448113d 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/PipelineTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/PipelineTest.php
@@ -7,9 +7,9 @@
use Flow\ETL\Executor\Pipeline;
use Flow\ETL\Executor\Segments;
use Flow\ETL\Executor\SourceRows;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Tests\FlowTestCase;
use Flow\ETL\Tests\Mother\NodeMother;
-use Flow\Filesystem\Path\Filter\OnlyFiles;
use Flow\Filesystem\Tests\Double\RejectingFilter;
use function Flow\ETL\DSL\from_array;
@@ -21,18 +21,16 @@ public function test_accessors_return_the_values_it_was_built_with(): void
$segments = new Segments();
$context = NodeMother::context();
$input = new Pipeline(0, new Segments(from_array([['id' => 1]])), $context);
- $filter = new RejectingFilter();
-
+ $scan = new FileScan(3, new RejectingFilter());
$sources = new SourceRows();
- $pipeline = new Pipeline(1, $segments, $context, $input, 3, $filter, $sources);
+ $pipeline = new Pipeline(1, $segments, $context, $input, $scan, $sources);
static::assertSame(1, $pipeline->id);
static::assertSame($segments, $pipeline->segments());
static::assertSame($context, $pipeline->context());
static::assertSame($input, $pipeline->input());
- static::assertSame(3, $pipeline->limit());
- static::assertSame($filter, $pipeline->pathFilter());
+ static::assertSame($scan, $pipeline->scan());
static::assertSame($sources, $pipeline->sources());
}
@@ -41,8 +39,7 @@ public function test_a_leaf_pipeline_defaults_to_no_input_no_limit_and_only_file
$pipeline = new Pipeline(0, new Segments(from_array([['id' => 1]])), NodeMother::context());
static::assertNull($pipeline->input());
- static::assertNull($pipeline->limit());
- static::assertInstanceOf(OnlyFiles::class, $pipeline->pathFilter());
+ static::assertEquals(new FileScan(), $pipeline->scan());
static::assertNull($pipeline->sources());
}
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/ExecutorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/ExecutorTest.php
index efeb84abb4..2b4f76aa06 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/ExecutorTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/ExecutorTest.php
@@ -12,6 +12,7 @@
use Flow\ETL\Executor\Pipeline;
use Flow\ETL\Executor\Raw;
use Flow\ETL\Executor\Segments;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Optimizer;
use Flow\ETL\Plan\Node\Transform;
use Flow\ETL\Processor\BatchingProcessor;
@@ -186,17 +187,16 @@ public function test_a_pipeline_without_a_source_throws(): void
iterator_to_array((new Executor())->executePipeline(new Pipeline(7, new Segments(), NodeMother::context())));
}
- public function test_a_file_source_receives_the_pipelines_limit_and_path_filter(): void
+ public function test_a_file_source_receives_the_pipelines_scan(): void
{
$extractor = new RecordingFileExtractor(schema(int_schema('id')), RowsMother::sequentialIds(1));
- $filter = new RejectingFilter();
+ $scan = new FileScan(7, new RejectingFilter());
iterator_to_array((new Executor())->executePipeline(
- new Pipeline(0, new Segments($extractor), NodeMother::context(), limit: 7, pathFilter: $filter),
+ new Pipeline(0, new Segments($extractor), NodeMother::context(), scan: $scan),
));
- static::assertSame([7], $extractor->limits);
- static::assertSame([$filter], $extractor->pathFilters);
+ static::assertSame([$scan], $extractor->scans);
}
public function test_a_source_that_lists_no_files_receives_the_pipelines_limit(): void
@@ -204,7 +204,7 @@ public function test_a_source_that_lists_no_files_receives_the_pipelines_limit()
$extractor = new RecordingExtractor(schema(int_schema('id')), RowsMother::sequentialIds(1));
iterator_to_array((new Executor())->executePipeline(
- new Pipeline(0, new Segments($extractor), NodeMother::context(), limit: 7),
+ new Pipeline(0, new Segments($extractor), NodeMother::context(), scan: new FileScan(7)),
));
static::assertSame([7], $extractor->limits);
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/DataFrameExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/DataFrameExtractorTest.php
index dc607111f5..4bc0922602 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/DataFrameExtractorTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/DataFrameExtractorTest.php
@@ -162,7 +162,8 @@ public function test_a_limit_given_to_extract_reaches_the_frames_source(): void
$batches = iterator_to_array(from_data_frame(df()->read($source))->extract(flow_context(config()), 2), false);
- static::assertSame([2], $source->limits);
+ static::assertCount(1, $source->scans);
+ static::assertSame(2, $source->scans[0]->limit);
static::assertSame(
[['id' => 1], ['id' => 2]],
array_merge(...array_map(static fn($rows) => $rows->toArray(), $batches)),
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileMetadataRowTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileMetadataRowTest.php
new file mode 100644
index 0000000000..a29fd6bfd2
--- /dev/null
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileMetadataRowTest.php
@@ -0,0 +1,35 @@
+ '/dir/a.csv',
+ 'protocol' => 'memory',
+ 'file_name' => 'a',
+ 'base_name' => 'a.csv',
+ 'is_file' => true,
+ 'is_dir' => false,
+ 'extension' => 'csv',
+ ],
+ (new FileMetadataRow())->row(new FileStatus(path('memory://dir/a.csv'), true)),
+ );
+ }
+
+ public function test_a_file_without_an_extension_has_a_null_extension(): void
+ {
+ static::assertNull((new FileMetadataRow())->row(new FileStatus(path('memory://dir/a'), true))['extension']);
+ }
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileReadLoopTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileReadLoopTest.php
index 7357642e5f..5a22c67cd2 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileReadLoopTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileReadLoopTest.php
@@ -4,7 +4,6 @@
namespace Flow\ETL\Tests\Unit\Extractor\File;
-use Flow\ETL\Column\PhpBackend;
use Flow\ETL\Extractor\File\FileReadLoop;
use Flow\ETL\Extractor\File\ReadWindow;
use Flow\ETL\Extractor\File\SourceFile;
@@ -17,7 +16,10 @@
use function array_keys;
use function array_map;
+use function Flow\ETL\DSL\flow_context;
use function Flow\ETL\DSL\int_schema;
+use function Flow\ETL\DSL\lit;
+use function Flow\ETL\DSL\ref;
use function Flow\ETL\DSL\schema;
use function Flow\ETL\DSL\str_schema;
use function Flow\Filesystem\DSL\path;
@@ -40,7 +42,7 @@ public function test_every_batch_carries_its_files_constants(): void
],
$batches,
10,
- new PhpBackend(),
+ flow_context(),
new ReadWindow(),
));
@@ -56,7 +58,7 @@ public function test_stop_after_the_first_batch_ends_the_read(): void
[new SourceFile(path('memory://a.csv')), new SourceFile(path('memory://b.csv'))],
new InMemoryFileBatches(['memory://a.csv' => [1, 2, 3], 'memory://b.csv' => [4]]),
1,
- new PhpBackend(),
+ flow_context(),
new ReadWindow(),
);
@@ -78,7 +80,7 @@ public function test_a_limit_inside_a_file_ends_the_read_and_is_pushed_down(): v
[new SourceFile(path('memory://a.csv')), new SourceFile(path('memory://b.csv'))],
$batches,
10,
- new PhpBackend(),
+ flow_context(),
new ReadWindow(limit: 2),
));
@@ -100,7 +102,7 @@ public function test_a_limit_at_a_file_boundary_opens_no_further_file(): void
[new SourceFile(path('memory://a.csv')), new SourceFile(path('memory://b.csv'))],
$batches,
1,
- new PhpBackend(),
+ flow_context(),
new ReadWindow(limit: 2),
));
@@ -127,7 +129,7 @@ public function test_the_offset_a_file_leaves_and_the_rows_still_wanted_reach_th
],
$batches,
10,
- new PhpBackend(),
+ flow_context(),
new ReadWindow(offset: 3, limit: 2),
));
@@ -152,7 +154,7 @@ public function test_no_sources_read_nothing(): void
iterator_to_array((new FileReadLoop(
FileColumnsContext::discovering(names: []),
schema(int_schema('id')),
- ))->read([], new InMemoryFileBatches([]), 10, new PhpBackend(), new ReadWindow())),
+ ))->read([], new InMemoryFileBatches([]), 10, flow_context(), new ReadWindow())),
);
}
@@ -173,7 +175,7 @@ public function test_the_loop_slices_the_offset_a_format_cannot_skip_across_file
],
$batches,
10,
- new PhpBackend(),
+ flow_context(),
new ReadWindow(offset: 3, limit: 2),
),
false,
@@ -202,7 +204,7 @@ public function test_the_loop_skips_the_offset_and_trims_to_the_limit_for_a_form
[new SourceFile(path('memory://a.csv'))],
$batches,
10,
- new PhpBackend(),
+ flow_context(),
new ReadWindow(offset: 3, limit: 2),
),
false,
@@ -214,4 +216,127 @@ public function test_the_loop_skips_the_offset_and_trims_to_the_limit_for_a_form
);
static::assertEquals(['memory://a.csv' => [new ReadWindow(0, 5)]], $batches->windows);
}
+
+ public function test_a_filter_drops_the_rows_it_does_not_match_and_skips_a_batch_it_empties(): void
+ {
+ $rows = iterator_to_array(
+ (new FileReadLoop(FileColumnsContext::discovering(names: []), schema(int_schema('id')), [ref(
+ 'id',
+ )->greaterThan(lit(
+ 2,
+ ))]))->read([new SourceFile(path('memory://a.csv'))], new InMemoryFileBatches(['memory://a.csv' => [1, 2, 3, 4, 5]]), 2, flow_context(), new ReadWindow()),
+ false,
+ );
+
+ static::assertSame(
+ [[['id' => 3], ['id' => 4]], [['id' => 5]]],
+ array_map(static fn(Rows $batch): array => $batch->toArray(), $rows),
+ );
+ }
+
+ public function test_a_filter_mixing_a_body_and_a_partition_column_sees_the_partition_values(): void
+ {
+ $rows = iterator_to_array(
+ (new FileReadLoop(FileColumnsContext::discovering(), schema(int_schema('id'), str_schema('year')), [
+ ref('year')->equals(lit('2024')),
+ ref('id')->greaterThan(lit(3)),
+ ]))->read([
+ new SourceFile(path('memory://orders/year=2023/a.csv')),
+ new SourceFile(path('memory://orders/year=2024/b.csv')),
+ ], new InMemoryFileBatches([
+ 'memory://orders/year=2023/a.csv' => [4, 5],
+ 'memory://orders/year=2024/b.csv' => [3, 4],
+ ]), 10, flow_context(), new ReadWindow()),
+ false,
+ );
+
+ static::assertSame(
+ [[['id' => 4, 'year' => '2024']]],
+ array_map(static fn(Rows $batch): array => $batch->toArray(), $rows),
+ );
+ }
+
+ public function test_the_offset_counts_raw_rows_and_the_limit_counts_filtered_ones(): void
+ {
+ $batches = new InMemoryFileBatches(['memory://a.csv' => [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]]);
+
+ $rows = iterator_to_array(
+ (new FileReadLoop(FileColumnsContext::discovering(names: []), schema(int_schema('id')), [ref(
+ 'id',
+ )->greaterThan(lit(
+ 5,
+ ))]))->read([new SourceFile(path('memory://a.csv'))], $batches, 10, flow_context(), new ReadWindow(offset: 6, limit: 2)),
+ false,
+ );
+
+ static::assertSame(
+ [[['id' => 7], ['id' => 8]]],
+ array_map(static fn(Rows $batch): array => $batch->toArray(), $rows),
+ );
+ static::assertEquals(['memory://a.csv' => [new ReadWindow(6)]], $batches->windows);
+ }
+
+ public function test_the_loop_drops_the_raw_offset_before_the_filter_for_a_format_that_cannot_skip(): void
+ {
+ $batches = new InMemoryUnskippableFileBatches(['memory://a.csv' => [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]]);
+
+ $rows = iterator_to_array(
+ (new FileReadLoop(FileColumnsContext::discovering(names: []), schema(int_schema('id')), [ref(
+ 'id',
+ )->greaterThan(lit(
+ 5,
+ ))]))->read([new SourceFile(path('memory://a.csv'))], $batches, 10, flow_context(), new ReadWindow(offset: 6, limit: 2)),
+ false,
+ );
+
+ static::assertSame(
+ [[['id' => 7], ['id' => 8]]],
+ array_map(static fn(Rows $batch): array => $batch->toArray(), $rows),
+ );
+ static::assertEquals(['memory://a.csv' => [new ReadWindow()]], $batches->windows);
+ }
+
+ public function test_a_limit_beside_a_filter_counts_matching_rows_across_files_and_opens_no_further_file(): void
+ {
+ $batches = new InMemoryFileBatches([
+ 'memory://a.csv' => [1, 2],
+ 'memory://b.csv' => [3, 4],
+ 'memory://c.csv' => [5],
+ ]);
+
+ $rows = iterator_to_array(
+ (new FileReadLoop(FileColumnsContext::discovering(names: []), schema(int_schema('id')), [ref(
+ 'id',
+ )->greaterThan(lit(
+ 1,
+ ))]))->read([
+ new SourceFile(path('memory://a.csv')),
+ new SourceFile(path('memory://b.csv')),
+ new SourceFile(path('memory://c.csv')),
+ ], $batches, 10, flow_context(), new ReadWindow(limit: 2)),
+ false,
+ );
+
+ static::assertSame(
+ [[['id' => 2]], [['id' => 3]]],
+ array_map(static fn(Rows $batch): array => $batch->toArray(), $rows),
+ );
+ static::assertEquals(
+ ['memory://a.csv' => [new ReadWindow()], 'memory://b.csv' => [new ReadWindow()]],
+ $batches->windows,
+ );
+ }
+
+ public function test_each_conjunct_of_the_filter_sees_only_the_rows_the_ones_before_it_kept(): void
+ {
+ $rows = iterator_to_array(
+ (new FileReadLoop(FileColumnsContext::discovering(names: []), schema(int_schema('id')), [
+ ref('id')->notEquals(lit(0)),
+ lit(10)->divide(ref('id'))->greaterThan(lit(1)),
+ ]))->read([new SourceFile(path('memory://a.csv'))], new InMemoryFileBatches(['memory://a.csv' => [0, 2, 10]]), 10, flow_context(), new ReadWindow()),
+ false,
+ );
+
+ static::assertSame([[['id' => 2]]], array_map(static fn(Rows $batch): array => $batch->toArray(), $rows));
+ }
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileScanTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileScanTest.php
new file mode 100644
index 0000000000..8a6a115393
--- /dev/null
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileScanTest.php
@@ -0,0 +1,91 @@
+limit);
+ static::assertInstanceOf(OnlyFiles::class, $scan->pathFilter);
+ static::assertSame([], $scan->conjuncts);
+ static::assertFalse($scan->narrows());
+ }
+
+ #[TestWith([0])]
+ #[TestWith([-1])]
+ public function test_a_limit_below_one_is_refused(int $limit): void
+ {
+ $this->expectException(InvalidArgumentException::class);
+ $this->expectExceptionMessage('Limit must be greater than 0');
+
+ new FileScan($limit);
+ }
+
+ public function test_a_filter_appends_its_conjuncts_in_push_order(): void
+ {
+ $first = ref('id')->greaterThan(lit(1));
+ $second = ref('id')->lessThan(lit(10));
+ $third = ref('id')->notEquals(lit(5));
+
+ $scan = (new FileScan(5))
+ ->withFilter($first)
+ ->withFilter($second->and($third));
+
+ static::assertSame([$first, $second, $third], $scan->conjuncts);
+ static::assertSame(5, $scan->limit);
+ static::assertTrue($scan->narrows());
+ }
+
+ public function test_a_second_limit_only_lowers_the_first_and_keeps_the_conjuncts(): void
+ {
+ $filter = ref('id')->greaterThan(lit(1));
+ $scan = (new FileScan())
+ ->withFilter($filter)
+ ->withLimit(5);
+
+ static::assertSame(3, $scan->withLimit(3)->limit);
+ static::assertSame(5, $scan->withLimit(8)->limit);
+ static::assertSame([$filter], $scan->withLimit(8)->conjuncts);
+ static::assertTrue(
+ (new FileScan())
+ ->withLimit(1)
+ ->narrows(),
+ );
+ }
+
+ public function test_path_filters_compose_and_keep_the_conjuncts(): void
+ {
+ $filter = ref('id')->greaterThan(lit(1));
+ $first = new RejectingFilter();
+ $second = new RejectingFilter();
+
+ $scan = (new FileScan())
+ ->withFilter($filter)
+ ->withPathFilter($first)
+ ->withPathFilter($second);
+
+ static::assertEquals(new Filters(new OnlyFiles(), $first, $second), $scan->pathFilter);
+ static::assertSame([$filter], $scan->conjuncts);
+ static::assertTrue(
+ (new FileScan())
+ ->withPathFilter($first)
+ ->narrows(),
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/ListingScanTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/ListingScanTest.php
new file mode 100644
index 0000000000..fa5ffc405f
--- /dev/null
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/ListingScanTest.php
@@ -0,0 +1,154 @@
+ 'a', 'memory://dir/b' => 'b', 'memory://dir/c' => 'c']);
+
+ static::assertSame(
+ [['memory://dir/a', 'memory://dir/b'], ['memory://dir/c']],
+ array_map(
+ static fn(Rows $rows): array => $rows->reduceToArray('path'),
+ iterator_to_array(
+ (new ListingScan(
+ $filesystem,
+ path('memory://dir/*'),
+ new FileColumns(new PartitionColumns($filesystem), [], new PartitionTypes(), false),
+ new PathPartitionsRow(),
+ ))->scan(
+ flow_context(),
+ new FileScan(),
+ schema(str_schema('path'), map_schema('partitions', type_map(type_string(), type_string()))),
+ 2,
+ ),
+ false,
+ ),
+ ),
+ );
+ }
+
+ public function test_a_batch_holds_only_the_files_every_conjunct_keeps_and_an_emptied_batch_is_skipped(): void
+ {
+ $filesystem = MemoryFiles::with(['memory://dir/a' => 'a', 'memory://dir/b' => 'b', 'memory://dir/c' => 'c']);
+
+ static::assertSame(
+ [['memory://dir/c']],
+ array_map(
+ static fn(Rows $rows): array => $rows->reduceToArray('path'),
+ iterator_to_array(
+ (new ListingScan(
+ $filesystem,
+ path('memory://dir/*'),
+ new FileColumns(new PartitionColumns($filesystem), [], new PartitionTypes(), false),
+ new PathPartitionsRow(),
+ ))->scan(
+ flow_context(),
+ new FileScan(conjuncts: [
+ ref('path')->notEquals(lit('memory://dir/a')),
+ ref('path')->notEquals(lit('memory://dir/b')),
+ ]),
+ schema(str_schema('path'), map_schema('partitions', type_map(type_string(), type_string()))),
+ 2,
+ ),
+ false,
+ ),
+ ),
+ );
+ }
+
+ public function test_the_scan_stops_once_the_limit_of_rows_is_handed_out(): void
+ {
+ $filesystem = MemoryFiles::with(['memory://dir/a' => 'a', 'memory://dir/b' => 'b', 'memory://dir/c' => 'c']);
+
+ static::assertCount(1, iterator_to_array(
+ (new ListingScan(
+ $filesystem,
+ path('memory://dir/*'),
+ new FileColumns(new PartitionColumns($filesystem), [], new PartitionTypes(), false),
+ new PathPartitionsRow(),
+ ))->scan(
+ flow_context(),
+ new FileScan(limit: 1),
+ schema(str_schema('path'), map_schema('partitions', type_map(type_string(), type_string()))),
+ 1,
+ ),
+ false,
+ ));
+ }
+
+ public function test_a_batch_past_the_limit_is_trimmed_to_it(): void
+ {
+ $filesystem = MemoryFiles::with(['memory://dir/a' => 'a', 'memory://dir/b' => 'b', 'memory://dir/c' => 'c']);
+
+ static::assertSame(
+ [['memory://dir/a', 'memory://dir/b']],
+ array_map(
+ static fn(Rows $rows): array => $rows->reduceToArray('path'),
+ iterator_to_array(
+ (new ListingScan(
+ $filesystem,
+ path('memory://dir/*'),
+ new FileColumns(new PartitionColumns($filesystem), [], new PartitionTypes(), false),
+ new PathPartitionsRow(),
+ ))->scan(
+ flow_context(),
+ new FileScan(limit: 2),
+ schema(str_schema('path'), map_schema('partitions', type_map(type_string(), type_string()))),
+ 3,
+ ),
+ false,
+ ),
+ ),
+ );
+ }
+
+ public function test_a_stop_signal_ends_the_scan(): void
+ {
+ $filesystem = MemoryFiles::with(['memory://dir/a' => 'a', 'memory://dir/b' => 'b']);
+ $scan = (new ListingScan(
+ $filesystem,
+ path('memory://dir/*'),
+ new FileColumns(new PartitionColumns($filesystem), [], new PartitionTypes(), false),
+ new PathPartitionsRow(),
+ ))->scan(
+ flow_context(),
+ new FileScan(),
+ schema(str_schema('path'), map_schema('partitions', type_map(type_string(), type_string()))),
+ 1,
+ );
+
+ static::assertSame(['memory://dir/a'], $scan->current()->reduceToArray('path'));
+
+ $scan->send(Signal::STOP);
+
+ static::assertFalse($scan->valid());
+ }
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/PathPartitionsRowTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/PathPartitionsRowTest.php
new file mode 100644
index 0000000000..f302978165
--- /dev/null
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/PathPartitionsRowTest.php
@@ -0,0 +1,30 @@
+ 'memory://dir/year=2024/month=01/a.csv', 'partitions' => ['year' => '2024', 'month' => '01']],
+ (new PathPartitionsRow())->row(new FileStatus(path('memory://dir/year=2024/month=01/a.csv'), true)),
+ );
+ }
+
+ public function test_a_file_outside_partitions_has_none(): void
+ {
+ static::assertSame(
+ [],
+ (new PathPartitionsRow())->row(new FileStatus(path('memory://dir/a.csv'), true))['partitions'],
+ );
+ }
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FilesExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FilesExtractorTest.php
index 62e717d70f..b9308f80b8 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FilesExtractorTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FilesExtractorTest.php
@@ -5,6 +5,7 @@
namespace Flow\ETL\Tests\Unit\Extractor;
use Flow\ETL\Cardinality;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\FilesExtractor;
use Flow\ETL\Extractor\Signal;
use Flow\ETL\Schema;
@@ -16,6 +17,8 @@
use function Flow\ETL\DSL\files;
use function Flow\ETL\DSL\flow_context;
+use function Flow\ETL\DSL\lit;
+use function Flow\ETL\DSL\ref;
use function Flow\ETL\DSL\schema;
use function Flow\ETL\DSL\str_schema;
use function iterator_to_array;
@@ -139,7 +142,7 @@ public function test_extracting_files_from_directory_with_limit(): void
{
$extractor = files(__DIR__ . '/Fixtures/FileListExtractor/**/*')->withBatchSize(1);
- self::assertExtractedRowsCount(2, $extractor, limit: 2);
+ self::assertExtractedRowsCount(2, $extractor, scan: new FileScan(2));
}
public function test_is_repeatable(): void
@@ -202,4 +205,20 @@ public function test_files_extractor_honours_the_batch_contract(): void
ExtractedRows::of(files(__DIR__ . '/Fixtures/FileListExtractor/**/*')->withBatchSize(1)),
);
}
+
+ public function test_a_scan_returns_only_the_files_matching_its_filter(): void
+ {
+ $batches = iterator_to_array(files('memory://dir/*.txt', MemoryFiles::with([
+ 'memory://dir/a.txt' => 'a',
+ 'memory://dir/b.txt' => 'b',
+ 'memory://dir/c.txt' => 'c',
+ 'memory://dir/d.txt' => 'd',
+ 'memory://dir/e.txt' => 'e',
+ ]))
+ ->withBatchSize(2)
+ ->scan(flow_context(), new FileScan(conjuncts: [ref('base_name')->isIn(lit(['c.txt', 'd.txt']))])), false);
+
+ static::assertCount(1, $batches);
+ static::assertSame(['c.txt', 'd.txt'], $batches[0]->column('base_name')->values());
+ }
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowPhpExtensionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowPhpExtensionTest.php
index 1ecd4d604a..25f21489dc 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowPhpExtensionTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowPhpExtensionTest.php
@@ -26,7 +26,7 @@ public function test_a_loaded_extension_of_another_abi_is_refused(): void
{
$this->expectException(RuntimeException::class);
$this->expectExceptionMessage(
- 'The loaded flow_php extension (0.46.0, ABI 2) does not match flow-php/etl (ABI 1). Rebuild or upgrade '
+ 'The loaded flow_php extension (0.46.0, ABI 3) does not match flow-php/etl (ABI 2). Rebuild or upgrade '
. 'flow_php to the version of flow-php/etl, or stop loading it to use the PHP implementation.',
);
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BetweenTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BetweenTest.php
index 17cf7fa38c..94c23fa6fb 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BetweenTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BetweenTest.php
@@ -12,11 +12,14 @@
use Flow\Types\Exception\InvalidArgumentException as TypesInvalidArgumentException;
use function Flow\ETL\DSL\between;
+use function Flow\ETL\DSL\definition_from_type;
use function Flow\ETL\DSL\flow_context;
use function Flow\ETL\DSL\int_schema;
use function Flow\ETL\DSL\lit;
use function Flow\ETL\DSL\ref;
use function Flow\ETL\DSL\schema;
+use function Flow\ETL\DSL\str_schema;
+use function Flow\Types\DSL\type_numeric_string;
final class BetweenTest extends FlowTestCase
{
@@ -165,4 +168,34 @@ public function test_between_refuses_incomparable_bounds_at_bind(): void
->between(lit(1), lit(new DateTimeImmutable('2024-01-01')))
->returns();
}
+
+ public function test_between_orders_strings_by_bytes(): void
+ {
+ static::assertTrue((new FunctionContext(flow_context()))->eval(
+ ref('a')->between(lit('10'), lit('9')),
+ ['a' => '100'],
+ schema(str_schema('a')),
+ ));
+ }
+
+ public function test_between_orders_a_numeric_string_column_numerically(): void
+ {
+ $context = flow_context();
+
+ static::assertFalse((new FunctionContext($context))->eval(
+ ref('a')->between(lit('10'), lit('9')),
+ ['a' => '100'],
+ schema(definition_from_type('a', type_numeric_string())),
+ ));
+ static::assertTrue((new FunctionContext($context))->eval(
+ ref('a')->between(lit('9'), lit('18446744073709551615'), lit(Boundary::INCLUSIVE)),
+ ['a' => '18446744073709551615'],
+ schema(definition_from_type('a', type_numeric_string())),
+ ));
+ static::assertFalse((new FunctionContext($context))->eval(
+ ref('a')->between(lit('9'), lit('18446744073709551614'), lit(Boundary::INCLUSIVE)),
+ ['a' => '18446744073709551615'],
+ schema(definition_from_type('a', type_numeric_string())),
+ ));
+ }
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ConjunctsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ConjunctsTest.php
new file mode 100644
index 0000000000..e96211fbf3
--- /dev/null
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ConjunctsTest.php
@@ -0,0 +1,41 @@
+greaterThan(lit(1));
+
+ static::assertSame([$function], (new Conjuncts())->of($function));
+ }
+
+ public function test_a_nested_and_is_flattened(): void
+ {
+ $a = ref('a')->equals(lit(1));
+ $b = ref('b')->equals(lit(2));
+ $c = ref('c')->equals(lit(3));
+ $d = ref('d')->equals(lit(4));
+
+ static::assertSame([$a, $b, $c, $d], (new Conjuncts())->of(new All($a, new All($b, new All($c)), $d)));
+ }
+
+ public function test_an_or_is_kept_whole(): void
+ {
+ $a = ref('a')->equals(lit(1));
+ $or = new Any(ref('b')->equals(lit(2)), new All(ref('c')->equals(lit(3)), ref('d')->equals(lit(4))));
+
+ static::assertSame([$a, $or], (new Conjuncts())->of(new All($a, $or)));
+ }
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/FilterTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/FilterTest.php
new file mode 100644
index 0000000000..c46224a20a
--- /dev/null
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/FilterTest.php
@@ -0,0 +1,110 @@
+ 1], ['a' => 2]], schema(int_schema('a')));
+
+ static::assertSame($rows, Filter::bind($rows->schema())->filter($rows, flow_context(config())));
+ }
+
+ public function test_empty_rows_stay_the_same_rows(): void
+ {
+ $rows = array_to_rows([], schema(int_schema('a')));
+
+ static::assertSame($rows, Filter::bind($rows->schema(), ref('a')->greaterThan(lit(0)))->filter(
+ $rows,
+ flow_context(config()),
+ ));
+ }
+
+ public function test_a_predicate_every_row_matches_keeps_the_same_rows(): void
+ {
+ $rows = array_to_rows([['a' => 1], ['a' => 2]], schema(int_schema('a')));
+
+ static::assertSame($rows, Filter::bind($rows->schema(), ref('a')->greaterThan(lit(0)))->filter(
+ $rows,
+ flow_context(config()),
+ ));
+ }
+
+ public function test_a_predicate_no_row_matches_keeps_no_row(): void
+ {
+ $rows = array_to_rows([['a' => 1], ['a' => 2]], schema(int_schema('a')));
+
+ static::assertSame(
+ 0,
+ Filter::bind($rows->schema(), ref('a')->greaterThan(lit(5)))
+ ->filter($rows, flow_context(config()))
+ ->count(),
+ );
+ }
+
+ public function test_a_null_result_drops_the_row_as_false_does(): void
+ {
+ $rows = array_to_rows([['a' => 1], ['a' => null], ['a' => 3]], schema(int_schema('a', nullable: true)));
+
+ static::assertSame(
+ [['a' => 3]],
+ Filter::bind($rows->schema(), ref('a')->greaterThan(lit(1)))
+ ->filter($rows, flow_context(config()))
+ ->toArray(),
+ );
+ }
+
+ public function test_a_predicate_is_evaluated_only_on_the_rows_the_ones_before_it_kept(): void
+ {
+ $rows = array_to_rows([['a' => 0], ['a' => 2], ['a' => 10]], schema(int_schema('a')));
+
+ static::assertSame(
+ [['a' => 2]],
+ Filter::bind($rows->schema(), ref('a')->notEquals(lit(0)), lit(10)->divide(ref('a'))->greaterThan(lit(1)))
+ ->filter($rows, flow_context(config()))
+ ->toArray(),
+ );
+ }
+
+ public function test_a_reference_the_schema_lacks_is_refused(): void
+ {
+ $this->expectException(SchemaDefinitionNotFoundException::class);
+
+ Filter::bind(schema(int_schema('a')), ref('missing')->equals(lit(1)));
+ }
+
+ public function test_an_expanding_predicate_is_refused(): void
+ {
+ $this->expectException(InvalidArgumentException::class);
+ $this->expectExceptionMessage(
+ 'filter() cannot contain array_expand(), it turns one row into many rows. Expand with withEntry() first, then use the new column.',
+ );
+
+ Filter::bind(ListColumnsMother::schema(), ref('tags')->expand()->equals(lit('x')));
+ }
+
+ public function test_a_predicate_returning_no_boolean_is_refused(): void
+ {
+ $this->expectException(InvalidArgumentException::class);
+ $this->expectExceptionMessage('filter() requires a predicate returning boolean');
+
+ Filter::bind(schema(int_schema('a')), ref('a'));
+ }
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanEqualTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanEqualTest.php
index 80daa88ae0..3cc7299f6b 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanEqualTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanEqualTest.php
@@ -11,6 +11,7 @@
use Flow\Types\Exception\InvalidArgumentException;
use function Flow\ETL\DSL\datetime_schema;
+use function Flow\ETL\DSL\definition_from_type;
use function Flow\ETL\DSL\float_schema;
use function Flow\ETL\DSL\flow_context;
use function Flow\ETL\DSL\int_schema;
@@ -22,6 +23,9 @@
use function Flow\ETL\DSL\time_schema;
use function Flow\Types\DSL\type_integer;
use function Flow\Types\DSL\type_list;
+use function Flow\Types\DSL\type_numeric_string;
+
+use const NAN;
final class GreaterThanEqualTest extends FlowTestCase
{
@@ -217,4 +221,45 @@ public function test_greater_than_equal_time_intervals(): void
schema(time_schema('v')),
));
}
+
+ public function test_greater_than_equal_orders_strings_by_bytes(): void
+ {
+ static::assertTrue((new FunctionContext(flow_context()))->eval(
+ ref('a')->greaterThanEqual(lit('10')),
+ ['a' => '9'],
+ schema(str_schema('a')),
+ ));
+ }
+
+ public function test_greater_than_equal_orders_a_numeric_string_column_numerically(): void
+ {
+ $context = flow_context();
+
+ static::assertFalse((new FunctionContext($context))->eval(
+ ref('a')->greaterThanEqual(lit('10')),
+ ['a' => '9'],
+ schema(definition_from_type('a', type_numeric_string())),
+ ));
+ static::assertTrue((new FunctionContext($context))->eval(
+ ref('a')->greaterThanEqual(lit('18446744073709551614')),
+ ['a' => '18446744073709551615'],
+ schema(definition_from_type('a', type_numeric_string())),
+ ));
+ }
+
+ public function test_greater_than_equal_is_false_against_nan_in_either_direction(): void
+ {
+ $context = flow_context();
+
+ static::assertFalse((new FunctionContext($context))->eval(
+ ref('a')->greaterThanEqual(lit(1.0)),
+ ['a' => NAN],
+ schema(float_schema('a')),
+ ));
+ static::assertFalse((new FunctionContext($context))->eval(
+ lit(1.0)->greaterThanEqual(ref('a')),
+ ['a' => NAN],
+ schema(float_schema('a')),
+ ));
+ }
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanTest.php
index a5d92c5ab2..d66561daa4 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanTest.php
@@ -11,6 +11,7 @@
use Flow\Types\Exception\InvalidArgumentException;
use function Flow\ETL\DSL\datetime_schema;
+use function Flow\ETL\DSL\definition_from_type;
use function Flow\ETL\DSL\float_schema;
use function Flow\ETL\DSL\flow_context;
use function Flow\ETL\DSL\int_schema;
@@ -22,6 +23,9 @@
use function Flow\ETL\DSL\time_schema;
use function Flow\Types\DSL\type_integer;
use function Flow\Types\DSL\type_list;
+use function Flow\Types\DSL\type_numeric_string;
+
+use const NAN;
final class GreaterThanTest extends FlowTestCase
{
@@ -227,4 +231,45 @@ public function test_greater_than_time_intervals(): void
schema(time_schema('v')),
));
}
+
+ public function test_greater_than_orders_strings_by_bytes(): void
+ {
+ static::assertTrue((new FunctionContext(flow_context()))->eval(
+ ref('a')->greaterThan(lit('10')),
+ ['a' => '9'],
+ schema(str_schema('a')),
+ ));
+ }
+
+ public function test_greater_than_orders_a_numeric_string_column_numerically(): void
+ {
+ $context = flow_context();
+
+ static::assertFalse((new FunctionContext($context))->eval(
+ ref('a')->greaterThan(lit('10')),
+ ['a' => '9'],
+ schema(definition_from_type('a', type_numeric_string())),
+ ));
+ static::assertTrue((new FunctionContext($context))->eval(
+ ref('a')->greaterThan(lit('18446744073709551614')),
+ ['a' => '18446744073709551615'],
+ schema(definition_from_type('a', type_numeric_string())),
+ ));
+ }
+
+ public function test_greater_than_is_false_against_nan_in_either_direction(): void
+ {
+ $context = flow_context();
+
+ static::assertFalse((new FunctionContext($context))->eval(
+ ref('a')->greaterThan(lit(1.0)),
+ ['a' => NAN],
+ schema(float_schema('a')),
+ ));
+ static::assertFalse((new FunctionContext($context))->eval(
+ lit(1.0)->greaterThan(ref('a')),
+ ['a' => NAN],
+ schema(float_schema('a')),
+ ));
+ }
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreatestTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreatestTest.php
index 8bf15b7382..81a1383fef 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreatestTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreatestTest.php
@@ -10,12 +10,15 @@
use Flow\ETL\Tests\Context\FunctionContext;
use Flow\ETL\Tests\FlowTestCase;
+use function Flow\ETL\DSL\definition_from_type;
use function Flow\ETL\DSL\flow_context;
use function Flow\ETL\DSL\greatest;
use function Flow\ETL\DSL\int_schema;
use function Flow\ETL\DSL\lit;
use function Flow\ETL\DSL\ref;
use function Flow\ETL\DSL\schema;
+use function Flow\ETL\DSL\str_schema;
+use function Flow\Types\DSL\type_numeric_string;
final class GreatestTest extends FlowTestCase
{
@@ -89,4 +92,22 @@ public function test_greatest_with_null(): void
schema(int_schema('int')),
));
}
+
+ public function test_greatest_orders_strings_by_bytes(): void
+ {
+ static::assertSame('9', (new FunctionContext(flow_context()))->eval(
+ greatest(ref('a'), lit('10'), lit('9')),
+ ['a' => null],
+ schema(str_schema('a', nullable: true)),
+ ));
+ }
+
+ public function test_greatest_orders_a_numeric_string_column_numerically(): void
+ {
+ static::assertSame('10', (new FunctionContext(flow_context()))->eval(
+ greatest(ref('a'), ref('b')),
+ ['a' => '9', 'b' => '10'],
+ schema(definition_from_type('a', type_numeric_string()), definition_from_type('b', type_numeric_string())),
+ ));
+ }
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LeastTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LeastTest.php
index dab159f076..63729cfb2d 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LeastTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LeastTest.php
@@ -10,12 +10,15 @@
use Flow\ETL\Tests\Context\FunctionContext;
use Flow\ETL\Tests\FlowTestCase;
+use function Flow\ETL\DSL\definition_from_type;
use function Flow\ETL\DSL\flow_context;
use function Flow\ETL\DSL\int_schema;
use function Flow\ETL\DSL\least;
use function Flow\ETL\DSL\lit;
use function Flow\ETL\DSL\ref;
use function Flow\ETL\DSL\schema;
+use function Flow\ETL\DSL\str_schema;
+use function Flow\Types\DSL\type_numeric_string;
final class LeastTest extends FlowTestCase
{
@@ -76,4 +79,22 @@ public function test_least_with_null(): void
schema(int_schema('int')),
));
}
+
+ public function test_least_orders_strings_by_bytes(): void
+ {
+ static::assertSame('10', (new FunctionContext(flow_context()))->eval(
+ least(ref('a'), lit('10'), lit('9')),
+ ['a' => null],
+ schema(str_schema('a', nullable: true)),
+ ));
+ }
+
+ public function test_least_orders_a_numeric_string_column_numerically(): void
+ {
+ static::assertSame('9', (new FunctionContext(flow_context()))->eval(
+ least(ref('a'), ref('b')),
+ ['a' => '9', 'b' => '10'],
+ schema(definition_from_type('a', type_numeric_string()), definition_from_type('b', type_numeric_string())),
+ ));
+ }
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanEqualTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanEqualTest.php
index 7f3b0d1620..2670ac50e0 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanEqualTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanEqualTest.php
@@ -11,6 +11,7 @@
use Flow\Types\Exception\InvalidArgumentException;
use function Flow\ETL\DSL\datetime_schema;
+use function Flow\ETL\DSL\definition_from_type;
use function Flow\ETL\DSL\float_schema;
use function Flow\ETL\DSL\flow_context;
use function Flow\ETL\DSL\int_schema;
@@ -22,6 +23,9 @@
use function Flow\ETL\DSL\time_schema;
use function Flow\Types\DSL\type_integer;
use function Flow\Types\DSL\type_list;
+use function Flow\Types\DSL\type_numeric_string;
+
+use const NAN;
final class LessThanEqualTest extends FlowTestCase
{
@@ -217,4 +221,45 @@ public function test_less_than_equal_time_intervals(): void
schema(time_schema('v')),
));
}
+
+ public function test_less_than_equal_orders_strings_by_bytes(): void
+ {
+ static::assertFalse((new FunctionContext(flow_context()))->eval(
+ ref('a')->lessThanEqual(lit('10')),
+ ['a' => '9'],
+ schema(str_schema('a')),
+ ));
+ }
+
+ public function test_less_than_equal_orders_a_numeric_string_column_numerically(): void
+ {
+ $context = flow_context();
+
+ static::assertTrue((new FunctionContext($context))->eval(
+ ref('a')->lessThanEqual(lit('10')),
+ ['a' => '9'],
+ schema(definition_from_type('a', type_numeric_string())),
+ ));
+ static::assertFalse((new FunctionContext($context))->eval(
+ ref('a')->lessThanEqual(lit('18446744073709551614')),
+ ['a' => '18446744073709551615'],
+ schema(definition_from_type('a', type_numeric_string())),
+ ));
+ }
+
+ public function test_less_than_equal_is_false_against_nan_in_either_direction(): void
+ {
+ $context = flow_context();
+
+ static::assertFalse((new FunctionContext($context))->eval(
+ ref('a')->lessThanEqual(lit(1.0)),
+ ['a' => NAN],
+ schema(float_schema('a')),
+ ));
+ static::assertFalse((new FunctionContext($context))->eval(
+ lit(1.0)->lessThanEqual(ref('a')),
+ ['a' => NAN],
+ schema(float_schema('a')),
+ ));
+ }
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanTest.php
index e24c88e4c0..d1c19da8f6 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanTest.php
@@ -11,6 +11,7 @@
use Flow\Types\Exception\InvalidArgumentException;
use function Flow\ETL\DSL\datetime_schema;
+use function Flow\ETL\DSL\definition_from_type;
use function Flow\ETL\DSL\float_schema;
use function Flow\ETL\DSL\flow_context;
use function Flow\ETL\DSL\int_schema;
@@ -22,6 +23,9 @@
use function Flow\ETL\DSL\time_schema;
use function Flow\Types\DSL\type_integer;
use function Flow\Types\DSL\type_list;
+use function Flow\Types\DSL\type_numeric_string;
+
+use const NAN;
final class LessThanTest extends FlowTestCase
{
@@ -217,4 +221,45 @@ public function test_less_than_time_intervals(): void
schema(time_schema('v')),
));
}
+
+ public function test_less_than_orders_strings_by_bytes(): void
+ {
+ static::assertFalse((new FunctionContext(flow_context()))->eval(
+ ref('a')->lessThan(lit('10')),
+ ['a' => '9'],
+ schema(str_schema('a')),
+ ));
+ }
+
+ public function test_less_than_orders_a_numeric_string_column_numerically(): void
+ {
+ $context = flow_context();
+
+ static::assertTrue((new FunctionContext($context))->eval(
+ ref('a')->lessThan(lit('10')),
+ ['a' => '9'],
+ schema(definition_from_type('a', type_numeric_string())),
+ ));
+ static::assertFalse((new FunctionContext($context))->eval(
+ ref('a')->lessThan(lit('18446744073709551614')),
+ ['a' => '18446744073709551615'],
+ schema(definition_from_type('a', type_numeric_string())),
+ ));
+ }
+
+ public function test_less_than_is_false_against_nan_in_either_direction(): void
+ {
+ $context = flow_context();
+
+ static::assertFalse((new FunctionContext($context))->eval(
+ ref('a')->lessThan(lit(1.0)),
+ ['a' => NAN],
+ schema(float_schema('a')),
+ ));
+ static::assertFalse((new FunctionContext($context))->eval(
+ lit(1.0)->lessThan(ref('a')),
+ ['a' => NAN],
+ schema(float_schema('a')),
+ ));
+ }
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ValueOrderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ValueOrderTest.php
new file mode 100644
index 0000000000..2c63dd6110
--- /dev/null
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ValueOrderTest.php
@@ -0,0 +1,81 @@
+compare($left, $right));
+ }
+
+ #[TestWith(['9', '10', -1])]
+ #[TestWith(['18446744073709551615', '18446744073709551614', 1])]
+ #[TestWith(['100', '100', 0])]
+ public function test_a_numeric_string_operand_orders_strings_numerically(
+ string $left,
+ string $right,
+ int $expected,
+ ): void {
+ static::assertSame($expected, (new ValueOrder(type_optional(type_numeric_string()), type_string()))->compare(
+ $left,
+ $right,
+ ));
+ }
+
+ public function test_an_order_without_operand_types_orders_strings_by_bytes(): void
+ {
+ static::assertSame(1, (new ValueOrder())->compare('9', '10'));
+ }
+
+ public function test_numbers_order_as_php_compares_them(): void
+ {
+ $order = new ValueOrder();
+
+ static::assertSame(-1, $order->compare(1, 1.5));
+ static::assertSame(0, $order->compare(2, 2.0));
+ static::assertSame(1, $order->compare('5', 3));
+ }
+
+ public function test_nan_is_never_below_anything_in_either_direction(): void
+ {
+ $order = new ValueOrder();
+
+ static::assertSame(1, $order->compare(NAN, 1.0));
+ static::assertSame(1, $order->compare(1.0, NAN));
+ }
+
+ public function test_date_times_order_by_instant(): void
+ {
+ static::assertSame(-1, (new ValueOrder())->compare(
+ new DateTimeImmutable('2024-01-01 10:00:00+00:00'),
+ new DateTimeImmutable('2024-01-01 11:30:00+01:00'),
+ ));
+ }
+
+ public function test_intervals_order_by_length(): void
+ {
+ $order = new ValueOrder();
+
+ static::assertSame(-1, $order->compare(new DateInterval('PT59M'), new DateInterval('PT1H')));
+ static::assertSame(0, $order->compare(new DateInterval('PT60M'), new DateInterval('PT1H')));
+ }
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/FilterWalkTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/FilterWalkTest.php
index b1f9030bbf..1846332960 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/FilterWalkTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/FilterWalkTest.php
@@ -61,7 +61,7 @@ public function test_a_transparent_preserving_chain_reaches_the_leaf(): void
$read = NodeMother::read();
static::assertNotNull((new FilterWalk())->predicateAtLeaf(
- new Filter(NodeMother::select($read), lit(true)),
+ new Filter(NodeMother::select($read, 'year'), lit(true)),
$read,
ref('year')->isNotNull(),
));
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/JoinSidesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/JoinSidesTest.php
index 3da06e64ad..9cfc78dda3 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/JoinSidesTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/JoinSidesTest.php
@@ -18,17 +18,17 @@ public function test_a_node_that_is_not_a_join_is_returned_as_it_is(): void
{
$limit = NodeMother::limit(NodeMother::read(), 5);
- static::assertSame($limit, (new JoinSides(Optimizer::default(), NodeMother::context()))->of($limit));
+ static::assertSame($limit, (new JoinSides(Optimizer::default(), NodeMother::context()))->of($limit, $limit));
}
public function test_a_joins_right_side_is_optimized_as_its_own_plan(): void
{
$join = NodeMother::join(NodeMother::read(), NodeMother::plan(NodeMother::limit(NodeMother::read(), 3))->root);
- $rewritten = (new JoinSides(Optimizer::default(), NodeMother::context()))->of($join);
+ $rewritten = (new JoinSides(Optimizer::default(), NodeMother::context()))->of($join, $join);
static::assertInstanceOf(Join::class, $rewritten);
- static::assertSame(3, (new LogicalPlan($rewritten->right()))->source()->limit());
+ static::assertSame(3, (new LogicalPlan($rewritten->right()))->source()->scan()->limit);
static::assertSame($join->children()[0], $rewritten->children()[0]);
}
@@ -39,16 +39,16 @@ public function test_a_cross_joins_right_side_is_optimized_as_its_own_plan(): vo
NodeMother::plan(NodeMother::limit(NodeMother::read(), 3))->root,
);
- $rewritten = (new JoinSides(Optimizer::default(), NodeMother::context()))->of($join);
+ $rewritten = (new JoinSides(Optimizer::default(), NodeMother::context()))->of($join, $join);
static::assertInstanceOf(CrossJoin::class, $rewritten);
- static::assertSame(3, (new LogicalPlan($rewritten->right()))->source()->limit());
+ static::assertSame(3, (new LogicalPlan($rewritten->right()))->source()->scan()->limit);
}
public function test_a_join_whose_right_side_does_not_change_keeps_its_identity(): void
{
$join = NodeMother::crossJoin(NodeMother::read(), NodeMother::plan(NodeMother::read())->root);
- static::assertSame($join, (new JoinSides(new Optimizer(), NodeMother::context()))->of($join));
+ static::assertSame($join, (new JoinSides(new Optimizer(), NodeMother::context()))->of($join, $join));
}
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CountFromStatisticsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CountFromStatisticsTest.php
index 1c8079b83b..9e4cafef44 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CountFromStatisticsTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CountFromStatisticsTest.php
@@ -9,6 +9,7 @@
use Flow\ETL\Extractor\Statistics;
use Flow\ETL\Memory\ArrayMemory;
use Flow\ETL\Optimizer\Rule\CountFromStatistics;
+use Flow\ETL\Plan\Node\Read;
use Flow\ETL\Plan\Trigger;
use Flow\ETL\Tests\Double\DeclaringExtractor;
use Flow\ETL\Tests\FlowTestCase;
@@ -130,10 +131,62 @@ public function test_a_read_with_a_pushed_limit_is_counted_by_running(): void
static::assertSame($plan, (new CountFromStatistics())->apply($plan, NodeMother::context()));
}
- public function test_a_read_with_a_pushed_partition_filter_is_counted_by_running(): void
+ public function test_a_parquet_read_of_exact_statistics_is_counted_from_them(): void
{
- $plan = Trigger::count->plan(NodeMother::read()->withPathFilter(new RejectingFilter()));
+ $memory = memory_filesystem();
+ df()
+ ->read(from_sequence_number('id', 1, 25))
+ ->write(to_parquet(path('memory://count/file.parquet'), filesystem: $memory))
+ ->run();
+ $plan = Trigger::count->plan(new Read(from_parquet(path('memory://count/file.parquet'), filesystem: $memory)));
+
+ static::assertNotSame($plan, (new CountFromStatistics())->apply($plan, NodeMother::context()));
+ }
+
+ public function test_a_read_with_a_pushed_path_filter_is_counted_by_running(): void
+ {
+ $memory = memory_filesystem();
+ df()
+ ->read(from_sequence_number('id', 1, 25))
+ ->write(to_parquet(path('memory://count/file.parquet'), filesystem: $memory))
+ ->run();
+ $plan = Trigger::count->plan((new Read(from_parquet(
+ path('memory://count/file.parquet'),
+ filesystem: $memory,
+ )))->withPathFilter(new RejectingFilter()));
+
+ static::assertSame($plan, (new CountFromStatistics())->apply($plan, NodeMother::context()));
+ }
+
+ public function test_a_read_with_a_pushed_filter_is_counted_by_running(): void
+ {
+ $memory = memory_filesystem();
+ df()
+ ->read(from_sequence_number('id', 1, 25))
+ ->write(to_parquet(path('memory://count/file.parquet'), filesystem: $memory))
+ ->run();
+ $plan = Trigger::count->plan((new Read(from_parquet(
+ path('memory://count/file.parquet'),
+ filesystem: $memory,
+ )))->withFilter(ref('id')->greaterThan(lit(20))));
+
+ static::assertSame($plan, (new CountFromStatistics())->apply($plan, NodeMother::context()));
+ }
+
+ public function test_a_count_over_a_filtered_parquet_read_is_counted_by_running(): void
+ {
+ $memory = memory_filesystem();
+ df()
+ ->read(from_sequence_number('id', 1, 25))
+ ->write(to_parquet(path('memory://count/file.parquet'), filesystem: $memory))
+ ->run();
+ $frame = df()
+ ->read(from_parquet(path('memory://count/file.parquet'), filesystem: $memory))
+ ->filter(ref('id')->greaterThan(lit(20)));
+
+ $plan = $frame->explain(Trigger::count)->logical;
+ static::assertSame(5, $frame->count());
static::assertSame($plan, (new CountFromStatistics())->apply($plan, NodeMother::context()));
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushFilterIntoSourceTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushFilterIntoSourceTest.php
index 7c55ab75b5..7849c68f54 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushFilterIntoSourceTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushFilterIntoSourceTest.php
@@ -6,9 +6,12 @@
use DateTimeImmutable;
use Flow\ETL\Exception\EvaluationException;
+use Flow\ETL\Exception\SchemaDefinitionNotFoundException;
+use Flow\ETL\Function\All;
use Flow\ETL\Memory\ArrayMemory;
use Flow\ETL\Optimizer;
use Flow\ETL\Optimizer\Rule\PushFilterIntoSource;
+use Flow\ETL\Plan\Node;
use Flow\ETL\Plan\Node\Filter;
use Flow\ETL\Plan\Node\Read;
use Flow\ETL\Plan\Stage;
@@ -21,6 +24,7 @@
use Flow\ETL\WithEntry;
use Flow\Filesystem\Path\Filter\Filters;
use Flow\Filesystem\Path\Filter\OnlyFiles;
+use PHPUnit\Framework\Attributes\DataProvider;
use function Flow\ETL\DSL\array_to_rows;
use function Flow\ETL\DSL\df;
@@ -45,11 +49,11 @@ public function test_a_partition_only_predicate_is_pushed_into_the_source(): voi
->filter(ref('year')->equals(lit(2023)))
->fetch();
- $pathFilter = $extractor->pathFilters[0];
+ $pathFilter = $extractor->scans[0]->pathFilter;
static::assertInstanceOf(Filters::class, $pathFilter);
static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07')));
static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07')));
- // the double reads every partition anyway, so only the Filter node the rule kept drops 2024
+ // the double lists every partition, so only the scan's filter drops 2024
static::assertSame([2023], $rows->reduceToArray('year'));
}
@@ -63,7 +67,7 @@ public function test_two_stacked_partition_filters_are_both_pushed(): void
->filter(ref('month')->equals(lit('07')))
->fetch();
- $pathFilter = $extractor->pathFilters[0];
+ $pathFilter = $extractor->scans[0]->pathFilter;
static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07')));
static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07')));
static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=08')));
@@ -80,7 +84,7 @@ public function test_three_stacked_partition_filters_all_reach_the_source(): voi
->filter(ref('month')->equals(lit('07')))
->fetch();
- $pathFilter = $extractor->pathFilters[0];
+ $pathFilter = $extractor->scans[0]->pathFilter;
static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07')));
static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2022/month=07')));
static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07')));
@@ -97,38 +101,53 @@ public function test_a_body_predicate_below_a_partition_filter_does_not_block_it
->filter(ref('year')->equals(lit(2023)))
->fetch();
- $pathFilter = $extractor->pathFilters[0];
+ $pathFilter = $extractor->scans[0]->pathFilter;
static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07')));
static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07')));
static::assertSame(['a'], $rows->reduceToArray('value'));
}
- public function test_a_body_column_predicate_is_not_pushed(): void
+ public function test_a_body_column_predicate_is_pushed_as_a_filter_and_never_as_a_path_filter(): void
{
$extractor = PartitionedSourceMother::yearMonth();
df()
->read($extractor)
- ->filter(ref('value')->equals(lit('a')))
+ ->filter($predicate = ref('value')->equals(lit('a')))
+ ->fetch();
+
+ static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter);
+ static::assertSame([$predicate], $extractor->scans[0]->conjuncts);
+ }
+
+ public function test_a_predicate_referencing_no_column_is_pushed_as_a_filter_and_never_as_a_path_filter(): void
+ {
+ $extractor = PartitionedSourceMother::yearMonth();
+
+ df()
+ ->read($extractor)
+ ->filter($predicate = lit(1)->equals(lit(1)))
->fetch();
- static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]);
+ static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter);
+ static::assertSame([$predicate], $extractor->scans[0]->conjuncts);
}
- public function test_a_mixed_predicate_pushes_only_its_partition_conjunct(): void
+ public function test_a_mixed_predicate_pushes_both_conjuncts_and_only_the_partition_one_as_a_path_filter(): void
{
$extractor = PartitionedSourceMother::yearMonth();
$rows = df()
->read($extractor)
- ->filter(ref('year')->equals(lit(2023))->and(ref('value')->notEquals(lit('a'))))
+ ->filter(($year = ref('year')->equals(lit(2023)))->and($value = ref('value')->notEquals(lit('a'))))
->fetch();
- $pathFilter = $extractor->pathFilters[0];
+ $pathFilter = $extractor->scans[0]->pathFilter;
static::assertInstanceOf(Filters::class, $pathFilter);
static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07')));
static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07')));
- // the body conjunct still runs in the Filter node: the only 2023 row has value "a"
+ static::assertEquals([$year, $value], $extractor->scans[0]->conjuncts);
+ // the scan applies the body conjunct too: the only 2023 row has value "a"
static::assertCount(0, $rows);
}
@@ -146,7 +165,7 @@ public function test_a_mixed_predicate_with_two_partition_conjuncts_pushes_both(
)
->fetch();
- $pathFilter = $extractor->pathFilters[0];
+ $pathFilter = $extractor->scans[0]->pathFilter;
static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07')));
static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07')));
static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=08')));
@@ -165,7 +184,7 @@ public function test_a_conjunct_that_is_not_deterministic_is_skipped_and_the_res
)
->fetch();
- $pathFilter = $extractor->pathFilters[0];
+ $pathFilter = $extractor->scans[0]->pathFilter;
static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07')));
static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07')));
}
@@ -179,22 +198,23 @@ public function test_an_or_over_partition_columns_is_pushed_whole(): void
->filter(ref('year')->equals(lit(2023))->or(ref('year')->equals(lit(2024))))
->fetch();
- $pathFilter = $extractor->pathFilters[0];
+ $pathFilter = $extractor->scans[0]->pathFilter;
static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07')));
static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07')));
static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2025/month=07')));
}
- public function test_an_or_mixing_a_body_column_is_not_pushed(): void
+ public function test_an_or_mixing_a_body_column_is_pushed_whole_as_a_filter_and_never_as_a_path_filter(): void
{
$extractor = PartitionedSourceMother::yearMonth();
df()
->read($extractor)
- ->filter(ref('year')->equals(lit(2023))->or(ref('value')->equals(lit('x'))))
+ ->filter($predicate = ref('year')->equals(lit(2023))->or(ref('value')->equals(lit('x'))))
->fetch();
- static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]);
+ static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter);
+ static::assertSame([$predicate], $extractor->scans[0]->conjuncts);
}
public function test_a_nested_and_inside_an_and_pushes_the_partition_conjuncts(): void
@@ -211,7 +231,7 @@ public function test_a_nested_and_inside_an_and_pushes_the_partition_conjuncts()
)
->fetch();
- $pathFilter = $extractor->pathFilters[0];
+ $pathFilter = $extractor->scans[0]->pathFilter;
static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07')));
static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07')));
static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=08')));
@@ -228,7 +248,7 @@ public function test_a_limit_below_the_filter_blocks_the_push(): void
->filter(ref('year')->equals(lit(2023)))
->fetch();
- static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]);
+ static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter);
}
public function test_a_with_entry_redefining_the_partition_column_blocks_the_push(): void
@@ -241,7 +261,7 @@ public function test_a_with_entry_redefining_the_partition_column_blocks_the_pus
->filter(ref('year')->equals(lit(2023)))
->fetch();
- static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]);
+ static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter);
}
public function test_a_duplicate_row_defining_the_partition_column_blocks_the_push(): void
@@ -254,7 +274,7 @@ public function test_a_duplicate_row_defining_the_partition_column_blocks_the_pu
->filter(ref('year')->equals(lit(2023)))
->fetch();
- static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]);
+ static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter);
// the 2024 row's duplicate carries year=2023: pruning its partition would lose it
static::assertCount(3, $rows);
}
@@ -269,7 +289,7 @@ public function test_a_duplicate_row_defining_another_column_does_not_block_the_
->filter(ref('year')->equals(lit(2023)))
->fetch();
- static::assertInstanceOf(Filters::class, $extractor->pathFilters[0]);
+ static::assertInstanceOf(Filters::class, $extractor->scans[0]->pathFilter);
}
public function test_a_filter_a_sink_root_does_not_consume_through_is_not_pushed(): void
@@ -282,7 +302,7 @@ public function test_a_filter_a_sink_root_does_not_consume_through_is_not_pushed
->filter(ref('year')->equals(lit(2023)))
->fetch();
- static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]);
+ static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter);
}
public function test_a_non_file_extractor_is_never_pushed_into(): void
@@ -294,7 +314,7 @@ public function test_a_non_file_extractor_is_never_pushed_into(): void
static::assertSame($plan, (new PushFilterIntoSource())->apply($plan, NodeMother::context()));
}
- public function test_a_source_without_partition_columns_is_never_pushed_into(): void
+ public function test_a_data_conjunct_is_pushed_into_a_source_without_partition_columns(): void
{
$extractor = new RecordingFileExtractor(
schema(int_schema('year')),
@@ -303,10 +323,37 @@ public function test_a_source_without_partition_columns_is_never_pushed_into():
df()
->read($extractor)
- ->filter(ref('year')->equals(lit(2023)))
+ ->filter($predicate = ref('year')->equals(lit(2023)))
+ ->fetch();
+
+ static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter);
+ static::assertSame([$predicate], $extractor->scans[0]->conjuncts);
+ }
+
+ public function test_a_non_deterministic_data_conjunct_is_not_pushed(): void
+ {
+ $extractor = PartitionedSourceMother::yearMonth();
+
+ df()
+ ->read($extractor)
+ ->filter(ref('value')->equals(random_string(2)))
+ ->fetch();
+
+ static::assertSame([], $extractor->scans[0]->conjuncts);
+ }
+
+ public function test_a_data_conjunct_under_a_reducing_node_is_not_pushed(): void
+ {
+ $extractor = PartitionedSourceMother::yearMonth();
+
+ df()
+ ->read($extractor)
+ ->limit(5)
+ ->filter(ref('value')->equals(lit('a')))
->fetch();
- static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]);
+ static::assertSame(5, $extractor->scans[0]->limit);
+ static::assertSame([], $extractor->scans[0]->conjuncts);
}
public function test_a_non_deterministic_predicate_is_not_pushed(): void
@@ -318,7 +365,7 @@ public function test_a_non_deterministic_predicate_is_not_pushed(): void
->filter(ref('month')->equals(random_string(2)))
->fetch();
- static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]);
+ static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter);
}
public function test_a_predicate_calling_a_user_callable_is_not_pushed(): void
@@ -330,16 +377,26 @@ public function test_a_predicate_calling_a_user_callable_is_not_pushed(): void
->filter(ref('year')->call(lit('is_string'), type_boolean()))
->fetch();
- static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]);
+ static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter);
}
- public function test_a_literal_only_predicate_is_not_pushed(): void
+ public function test_a_constant_conjunct_is_pushed_and_stops_nothing_above_it(): void
{
$extractor = PartitionedSourceMother::yearMonth();
- df()->read($extractor)->filter(lit(true))->fetch();
+ $rows = df()
+ ->read($extractor)
+ ->filter($constant = lit(true))
+ ->filter($year = ref('year')->equals(lit(2023)))
+ ->fetch();
+
+ static::assertSame([$constant, $year], $extractor->scans[0]->conjuncts);
+ static::assertSame([2023], $rows->reduceToArray('year'));
+ }
- static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]);
+ public function test_a_constant_false_conjunct_returns_no_row(): void
+ {
+ static::assertSame(0, df()->read(PartitionedSourceMother::yearMonth())->filter(lit(false))->fetch()->count());
}
public function test_a_filter_inside_a_joined_frame_is_pushed_with_that_frames_context(): void
@@ -351,7 +408,7 @@ public function test_a_filter_inside_a_joined_frame_is_pushed_with_that_frames_c
->join(df()->read($right)->filter(ref('year')->equals(lit(2023))), join_on(['id' => 'year'], 'r_'))
->fetch();
- $pathFilter = $right->pathFilters[0];
+ $pathFilter = $right->scans[0]->pathFilter;
static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07')));
static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07')));
}
@@ -379,7 +436,7 @@ public function test_the_extractor_instance_is_never_mutated(): void
$frame->explain()->toString(Stage::unoptimized),
);
static::assertStringEndsWith(
- "Extractor: RecordingFileExtractor\n Source: file:///dev/null\n Files: Filters",
+ "Extractor: RecordingFileExtractor\n Source: file:///dev/null\n Files: Filters\n Filter: Equals",
$frame->explain()->toString(),
);
}
@@ -394,7 +451,7 @@ public function test_a_filter_on_a_renamed_partition_column_is_pushed_as_the_ori
->filter(ref('y')->equals(lit(2023)))
->fetch();
- $pathFilter = $extractor->pathFilters[0];
+ $pathFilter = $extractor->scans[0]->pathFilter;
static::assertInstanceOf(Filters::class, $pathFilter);
static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07')));
static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07')));
@@ -411,23 +468,228 @@ public function test_a_filter_on_an_alias_of_a_partition_column_is_pushed_as_the
->filter(ref('y')->equals(lit(2023)))
->fetch();
- $pathFilter = $extractor->pathFilters[0];
+ $pathFilter = $extractor->scans[0]->pathFilter;
static::assertInstanceOf(Filters::class, $pathFilter);
static::assertTrue($pathFilter->accept(PartitionedSourceMother::file('year=2023/month=07')));
static::assertFalse($pathFilter->accept(PartitionedSourceMother::file('year=2024/month=07')));
static::assertSame([2023], $rows->reduceToArray('y'));
}
- public function test_a_filter_on_an_alias_of_a_body_column_is_not_pushed(): void
+ public function test_a_filter_on_an_alias_of_a_body_column_is_pushed_as_the_original_column_and_never_as_a_path_filter(): void
{
$extractor = PartitionedSourceMother::yearMonth();
df()
->read($extractor)
->withEntry('v', ref('value'))
- ->filter(ref('v')->equals(lit('a')))
+ ->filter(ref('v')->equals($literal = lit('a')))
+ ->fetch();
+
+ static::assertEquals(new OnlyFiles(), $extractor->scans[0]->pathFilter);
+ static::assertEquals([ref('value')->equals($literal)], $extractor->scans[0]->conjuncts);
+ }
+
+ public function test_a_filter_node_whose_only_conjunct_is_pushed_is_removed(): void
+ {
+ $plan = NodeMother::plan(
+ new Filter(new Read(PartitionedSourceMother::yearMonth()), $year = ref('year')->equals(lit(2023))),
+ );
+
+ $read = (new PushFilterIntoSource())->apply($plan, NodeMother::context())->root->children()[0];
+
+ static::assertInstanceOf(Read::class, $read);
+ static::assertSame([$year], $read->scan()->conjuncts);
+ }
+
+ public function test_a_pushed_conjunct_leaves_its_filter_node_and_the_rest_stays_in_it(): void
+ {
+ $plan = NodeMother::plan(
+ new Filter(
+ new Read(PartitionedSourceMother::yearMonth()),
+ ($year = ref('year')->equals(lit(2023)))->and($random = ref('value')->equals(random_string(2))),
+ ),
+ );
+
+ $filter = (new PushFilterIntoSource())->apply($plan, NodeMother::context())->root->children()[0];
+
+ static::assertInstanceOf(Filter::class, $filter);
+ static::assertSame($random, $filter->function);
+ $read = $filter->children()[0];
+ static::assertInstanceOf(Read::class, $read);
+ static::assertSame([$year], $read->scan()->conjuncts);
+ }
+
+ public function test_stacked_filter_nodes_are_pushed_leaf_first(): void
+ {
+ $plan = NodeMother::plan(
+ new Filter(
+ new Filter(new Read(PartitionedSourceMother::yearMonth()), $year = ref('year')->equals(lit(2023))),
+ $month = ref('month')->equals(lit('07')),
+ ),
+ );
+
+ $read = (new PushFilterIntoSource())->apply($plan, NodeMother::context())->root->children()[0];
+
+ static::assertInstanceOf(Read::class, $read);
+ static::assertEquals([$year, $month], $read->scan()->conjuncts);
+ }
+
+ public function test_a_filter_node_that_keeps_a_conjunct_stops_the_push_of_the_filters_above_it(): void
+ {
+ $plan = NodeMother::plan(
+ new Filter(
+ new Filter(
+ new Read(PartitionedSourceMother::yearMonth()),
+ ($year = ref('year')->equals(lit(2023)))->and($random = ref('value')->equals(random_string(2))),
+ ),
+ $month = ref('month')->equals(lit('07')),
+ ),
+ );
+
+ $top = (new PushFilterIntoSource())->apply($plan, NodeMother::context())->root->children()[0];
+
+ static::assertInstanceOf(Filter::class, $top);
+ static::assertSame($month, $top->function);
+ $bottom = $top->children()[0];
+ static::assertInstanceOf(Filter::class, $bottom);
+ static::assertSame($random, $bottom->function);
+ $read = $bottom->children()[0];
+ static::assertInstanceOf(Read::class, $read);
+ static::assertSame([$year], $read->scan()->conjuncts);
+ }
+
+ public function test_a_filter_node_keeping_two_conjuncts_keeps_them_as_one_all(): void
+ {
+ $plan = NodeMother::plan(
+ new Filter(
+ new Read(PartitionedSourceMother::yearMonth()),
+ ref('year')
+ ->equals(lit(2023))
+ ->and($first = ref('value')->equals(random_string(2)))
+ ->and($second = ref('value')->notEquals(random_string(3))),
+ ),
+ );
+
+ $filter = (new PushFilterIntoSource())->apply($plan, NodeMother::context())->root->children()[0];
+
+ static::assertInstanceOf(Filter::class, $filter);
+ static::assertEquals(new All($first, $second), $filter->function);
+ }
+
+ public function test_a_limit_above_a_fully_pushed_filter_is_pushed_beside_it(): void
+ {
+ $extractor = PartitionedSourceMother::yearMonth();
+
+ $rows = df()
+ ->read($extractor)
+ ->filter($year = ref('year')->equals(lit(2023)))
+ ->limit(1)
+ ->fetch();
+
+ static::assertSame(1, $extractor->scans[0]->limit);
+ static::assertSame([$year], $extractor->scans[0]->conjuncts);
+ static::assertSame([2023], $rows->reduceToArray('year'));
+ }
+
+ public function test_a_limit_above_a_filter_node_that_keeps_a_conjunct_is_not_pushed(): void
+ {
+ $extractor = PartitionedSourceMother::yearMonth();
+
+ df()
+ ->read($extractor)
+ ->filter(ref('year')->equals(lit(2023))->and(ref('value')->equals(random_string(2))))
+ ->limit(1)
+ ->fetch();
+
+ static::assertNull($extractor->scans[0]->limit);
+ }
+
+ public function test_a_kept_conjunct_keeps_every_conjunct_after_it_in_its_node(): void
+ {
+ $plan = NodeMother::plan(
+ new Filter(
+ new Read(PartitionedSourceMother::yearMonth()),
+ ($year = ref('year')->equals(lit(2023)))->and($random = ref('value')->equals(random_string(2)))->and(
+ $month = ref('month')->equals(lit('07')),
+ ),
+ ),
+ );
+
+ $filter = (new PushFilterIntoSource())->apply($plan, NodeMother::context())->root->children()[0];
+
+ static::assertInstanceOf(Filter::class, $filter);
+ static::assertEquals(new All($random, $month), $filter->function);
+ $read = $filter->children()[0];
+ static::assertInstanceOf(Read::class, $read);
+ static::assertSame([$year], $read->scan()->conjuncts);
+ }
+
+ /**
+ * @return iterable
+ */
+ public static function nodes_removing_year(): iterable
+ {
+ $read = new Read(PartitionedSourceMother::yearMonth());
+
+ yield 'rename' => [new Node\Rename($read, 'year', 'y')];
+ yield 'drop' => [new Node\Drop($read, ['year'])];
+ yield 'select' => [new Node\Select($read, ['month', 'value'])];
+ }
+
+ #[DataProvider('nodes_removing_year')]
+ public function test_a_filter_on_a_column_a_node_removes_is_not_pushed_past_it(Node $node): void
+ {
+ $plan = NodeMother::plan(new Filter($node, $year = ref('year')->equals(lit(2023))));
+
+ $filter = (new PushFilterIntoSource())->apply($plan, NodeMother::context())->root->children()[0];
+
+ static::assertInstanceOf(Filter::class, $filter);
+ static::assertSame($year, $filter->function);
+ }
+
+ public function test_a_filter_on_a_renamed_away_column_fails(): void
+ {
+ $this->expectException(SchemaDefinitionNotFoundException::class);
+
+ df()
+ ->read(PartitionedSourceMother::yearMonth())
+ ->rename('year', 'y')
+ ->filter(ref('year')->equals(lit(2023)))
+ ->fetch();
+ }
+
+ public function test_a_filter_on_a_dropped_column_fails(): void
+ {
+ $this->expectException(SchemaDefinitionNotFoundException::class);
+
+ df()
+ ->read(PartitionedSourceMother::yearMonth())
+ ->drop('year')
+ ->filter(ref('year')->equals(lit(2023)))
+ ->fetch();
+ }
+
+ public function test_a_filter_on_a_column_select_does_not_keep_fails(): void
+ {
+ $this->expectException(SchemaDefinitionNotFoundException::class);
+
+ df()
+ ->read(PartitionedSourceMother::yearMonth())
+ ->select('month')
+ ->filter(ref('year')->equals(lit(2023)))
+ ->fetch();
+ }
+
+ public function test_a_filter_on_a_column_select_keeps_is_pushed(): void
+ {
+ $extractor = PartitionedSourceMother::yearMonth();
+
+ df()
+ ->read($extractor)
+ ->select('year')
+ ->filter($year = ref('year')->equals(lit(2023)))
->fetch();
- static::assertEquals(new OnlyFiles(), $extractor->pathFilters[0]);
+ static::assertSame([$year], $extractor->scans[0]->conjuncts);
}
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushLimitIntoSourceTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushLimitIntoSourceTest.php
index e72ba207dc..579ddaa477 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushLimitIntoSourceTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushLimitIntoSourceTest.php
@@ -42,7 +42,7 @@ public function test_a_limit_directly_above_the_read_is_pushed(): void
NodeMother::context(),
);
- static::assertSame(10, $plan->source()->limit());
+ static::assertSame(10, $plan->source()->scan()->limit);
}
public function test_a_limit_passes_a_select(): void
@@ -52,7 +52,7 @@ public function test_a_limit_passes_a_select(): void
NodeMother::context(),
);
- static::assertSame(10, $plan->source()->limit());
+ static::assertSame(10, $plan->source()->scan()->limit);
}
#[DataProvider('expanding_expressions')]
@@ -63,7 +63,7 @@ public function test_a_limit_does_not_pass_an_expand_column(ScalarFunction $func
NodeMother::context(),
);
- static::assertNull($plan->source()->limit());
+ static::assertNull($plan->source()->scan()->limit);
}
public static function expanding_expressions(): Generator
@@ -82,7 +82,7 @@ public function test_a_limit_does_not_pass_a_filter(): void
NodeMother::context(),
);
- static::assertNull($plan->source()->limit());
+ static::assertNull($plan->source()->scan()->limit);
}
public function test_a_limit_does_not_pass_a_write(): void
@@ -92,7 +92,7 @@ public function test_a_limit_does_not_pass_a_write(): void
NodeMother::context(),
);
- static::assertNull($plan->source()->limit());
+ static::assertNull($plan->source()->scan()->limit);
}
public function test_a_blocker_discards_the_limit_collected_above_it_and_the_walk_continues(): void
@@ -105,7 +105,7 @@ public function test_a_blocker_discards_the_limit_collected_above_it_and_the_wal
NodeMother::context(),
);
- static::assertSame(5, $plan->source()->limit());
+ static::assertSame(5, $plan->source()->scan()->limit);
}
public function test_the_push_rewrites_the_leaf_and_keeps_the_spine(): void
@@ -120,10 +120,10 @@ public function test_the_push_rewrites_the_leaf_and_keeps_the_spine(): void
static::assertInstanceOf(Node\Limit::class, $limit);
static::assertSame(10, $limit->limit);
static::assertInstanceOf(Node\Select::class, $limit->children()[0]);
- static::assertSame(10, $pushed->source()->limit());
+ static::assertSame(10, $pushed->source()->scan()->limit);
static::assertSame($extractor, $pushed->source()->extractor());
- static::assertNull($plan->source()->limit());
- static::assertSame([], $extractor->limits);
+ static::assertNull($plan->source()->scan()->limit);
+ static::assertSame([], $extractor->scans);
}
public function test_the_minimum_of_two_limits_is_pushed(): void
@@ -133,7 +133,7 @@ public function test_the_minimum_of_two_limits_is_pushed(): void
NodeMother::context(),
);
- static::assertSame(3, $plan->source()->limit());
+ static::assertSame(3, $plan->source()->scan()->limit);
}
public function test_a_limit_narrows_an_already_pushed_limit(): void
@@ -143,7 +143,7 @@ public function test_a_limit_narrows_an_already_pushed_limit(): void
NodeMother::context(),
);
- static::assertSame(4, $plan->source()->limit());
+ static::assertSame(4, $plan->source()->scan()->limit);
}
public function test_a_plan_without_a_limit_pushes_nothing(): void
@@ -153,7 +153,7 @@ public function test_a_plan_without_a_limit_pushes_nothing(): void
NodeMother::context(),
);
- static::assertNull($plan->source()->limit());
+ static::assertNull($plan->source()->scan()->limit);
}
public function test_a_row_input_chain_that_does_not_end_in_a_read_throws(): void
@@ -185,7 +185,7 @@ public function test_a_transaction_contributes_one_walk_per_child(): void
NodeMother::context(),
);
- static::assertSame(3, $plan->source()->limit());
+ static::assertSame(3, $plan->source()->scan()->limit);
}
public function test_one_transaction_child_whose_walk_yields_no_limit_blocks_the_push(): void
@@ -207,7 +207,7 @@ public function test_one_transaction_child_whose_walk_yields_no_limit_blocks_the
NodeMother::context(),
);
- static::assertNull($plan->source()->limit());
+ static::assertNull($plan->source()->scan()->limit);
}
public function test_a_limit_below_a_sink_is_pushed(): void
@@ -219,7 +219,7 @@ public function test_a_limit_below_a_sink_is_pushed(): void
NodeMother::context(),
);
- static::assertSame(3, $plan->source()->limit());
+ static::assertSame(3, $plan->source()->scan()->limit);
}
public function test_a_limit_above_a_sink_does_not_narrow_the_sink(): void
@@ -233,7 +233,7 @@ public function test_a_limit_above_a_sink_does_not_narrow_the_sink(): void
NodeMother::context(),
);
- static::assertNull($plan->source()->limit());
+ static::assertNull($plan->source()->scan()->limit);
}
public function test_the_widest_limit_of_every_root_is_pushed(): void
@@ -247,7 +247,7 @@ public function test_the_widest_limit_of_every_root_is_pushed(): void
NodeMother::context(),
);
- static::assertSame(5, $plan->source()->limit());
+ static::assertSame(5, $plan->source()->scan()->limit);
}
public function test_a_sink_whose_walk_yields_no_limit_blocks_the_push(): void
@@ -267,7 +267,7 @@ public function test_a_sink_whose_walk_yields_no_limit_blocks_the_push(): void
NodeMother::context(),
);
- static::assertNull($plan->source()->limit());
+ static::assertNull($plan->source()->scan()->limit);
}
public function test_the_push_reaches_the_sinks_through_the_shared_leaf(): void
@@ -289,6 +289,6 @@ public function test_a_limit_above_an_offset_pushes_the_limit_plus_the_offset():
NodeMother::context(),
);
- static::assertSame(110, $plan->source()->limit());
+ static::assertSame(110, $plan->source()->scan()->limit);
}
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/TopNRewriteTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/TopNRewriteTest.php
index 053bafae5b..2985a2f83c 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/TopNRewriteTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/TopNRewriteTest.php
@@ -21,14 +21,14 @@ public function test_a_node_that_is_not_a_limit_is_returned_as_is(): void
{
$sort = NodeMother::sort(NodeMother::read());
- static::assertSame($sort, (new TopNRewrite())->of($sort));
+ static::assertSame($sort, (new TopNRewrite())->of($sort, $sort));
}
public function test_a_limit_over_anything_but_a_sort_is_returned_as_is(): void
{
$limit = NodeMother::limit(NodeMother::select(NodeMother::sort(NodeMother::read())), 5);
- static::assertSame($limit, (new TopNRewrite())->of($limit));
+ static::assertSame($limit, (new TopNRewrite())->of($limit, $limit));
}
public function test_a_limit_over_a_sort_becomes_a_top_n_over_the_sorts_input_refs_and_limit(): void
@@ -36,7 +36,8 @@ public function test_a_limit_over_a_sort_becomes_a_top_n_over_the_sorts_input_re
$read = NodeMother::read();
$refs = refs(ref('id'));
- $topN = (new TopNRewrite())->of(NodeMother::limit(new Sort($read, $refs), 5));
+ $limit = NodeMother::limit(new Sort($read, $refs), 5);
+ $topN = (new TopNRewrite())->of($limit, $limit);
static::assertInstanceOf(TopN::class, $topN);
static::assertSame([$read], $topN->children());
@@ -48,10 +49,8 @@ public function test_a_limit_over_any_sort_is_rewritten_and_keeps_the_pinned_alg
{
$algorithm = external_sort()->memoryLimit(Unit::fromMb(1));
- $topN = (new TopNRewrite())->of(NodeMother::limit(
- new Sort(NodeMother::read(), refs(ref('id')), $algorithm),
- 1_000_000,
- ));
+ $limit = NodeMother::limit(new Sort(NodeMother::read(), refs(ref('id')), $algorithm), 1_000_000);
+ $topN = (new TopNRewrite())->of($limit, $limit);
static::assertInstanceOf(TopN::class, $topN);
static::assertSame($algorithm, $topN->algorithm);
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/OptimizerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/OptimizerTest.php
index fa4487f34a..97075d094c 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/OptimizerTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/OptimizerTest.php
@@ -57,14 +57,14 @@ public function test_an_optimizer_without_rules_returns_the_plan_it_was_given():
static::assertSame($plan, (new Optimizer())->optimize($plan, NodeMother::context()));
}
- public function test_default_registers_combine_limits_combine_sort_and_limit_push_limit_push_filter_then_count_from_statistics(): void
+ public function test_default_registers_combine_limits_combine_sort_and_limit_push_filter_push_limit_then_count_from_statistics(): void
{
static::assertSame(
[
CombineLimits::class,
CombineSortAndLimit::class,
- PushLimitIntoSource::class,
PushFilterIntoSource::class,
+ PushLimitIntoSource::class,
CountFromStatistics::class,
PushProjectionIntoExpand::class,
],
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/BoxLayoutTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/BoxLayoutTest.php
index ce9a262b2c..db4411c12d 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/BoxLayoutTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/BoxLayoutTest.php
@@ -118,7 +118,7 @@ public function test_a_middle_child_branches_off_the_same_line(): void
┌─────────────┴─────────────┐┌─────────────┴─────────────┐┌─────────────┴─────────────┐
│ #1 Read ││ #1 Read ││ #4 Select │
│ ──────────────────── ││ (shared) ││ ──────────────────── │
- │ Extractor: ArrayExtractor ││ ││ Columns: id │
+ │ Extractor: ArrayExtractor ││ ││ Keeps only: id │
└───────────────────────────┘└───────────────────────────┘└─────────────┬─────────────┘
┌─────────────┴─────────────┐
│ #1 Read │
@@ -138,9 +138,9 @@ public function test_long_text_wraps_and_a_word_longer_than_the_box_is_split():
│ Rename: id → │
│ identifier_of_the_custome │
│ r_order │
- │ Defines columns: │
+ │ Changes columns: │
│ identifier_of_the_custome │
- │ r_order │
+ │ r_order, id │
└─────────────┬─────────────┘
┌─────────────┴─────────────┐
│ #1 Read │
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/DetailsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/DetailsTest.php
index 44e7139c9f..d1bf62490f 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/DetailsTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/DetailsTest.php
@@ -16,6 +16,7 @@
use Flow\ETL\Plan\Node\Filter;
use Flow\ETL\Plan\Node\Join as JoinNode;
use Flow\ETL\Plan\Node\Offset;
+use Flow\ETL\Plan\Node\Read;
use Flow\ETL\Plan\Node\Rename;
use Flow\ETL\Plan\Node\RenameEach;
use Flow\ETL\Plan\Node\Result;
@@ -26,6 +27,7 @@
use Flow\ETL\Plan\RequiredColumns;
use Flow\ETL\Tests\FlowTestCase;
use Flow\ETL\Tests\Mother\NodeMother;
+use Flow\ETL\Tests\Mother\PartitionedSourceMother;
use Flow\Filesystem\Path\Filter\KeepAll;
use function Flow\ETL\DSL\hash_join;
@@ -48,11 +50,23 @@ public function test_a_read_lists_its_extractor_only_when_nothing_was_pushed_int
static::assertSame(['Extractor: ArrayExtractor'], (new Details())->lines(NodeMother::read()));
}
- public function test_a_read_lists_a_pushed_limit_and_file_filter(): void
+ public function test_a_read_lists_a_pushed_limit_file_filter_and_each_conjunct(): void
{
static::assertSame(
- ['Extractor: ArrayExtractor', 'Limit: 3', 'Files: Filters'],
- (new Details())->lines(NodeMother::read()->withLimit(3)->withPathFilter(new KeepAll())),
+ [
+ 'Extractor: RecordingFileExtractor',
+ 'Source: file:///dev/null',
+ 'Limit: 3',
+ 'Files: Filters',
+ 'Filter: GreaterThan',
+ 'Filter: IsNotNull',
+ ],
+ (new Details())->lines(
+ (new Read(PartitionedSourceMother::yearMonth()))
+ ->withLimit(3)
+ ->withPathFilter(new KeepAll())
+ ->withFilter(ref('year')->greaterThan(lit(1))->and(ref('month')->isNotNull())),
+ ),
);
}
@@ -80,7 +94,7 @@ public function test_a_join_labels_its_type_and_condition(): void
'Left on: id',
'Right on: id',
'Buffers all rows before passing them on',
- 'Defines columns known only at run time',
+ 'Changes columns known only at run time',
],
(new Details())->lines(NodeMother::join(
NodeMother::read(),
@@ -171,12 +185,12 @@ public function test_redefined_columns_follow_the_labelled_details(): void
$details = new Details();
static::assertSame(
- ['Column: n = Literal', 'Defines columns: n'],
+ ['Column: n = Literal', 'Changes columns: n'],
$details->lines(new WithColumn($read, int_schema('n'), lit(1))),
);
- static::assertSame(['Rename: n → m', 'Defines columns: m'], $details->lines(new Rename($read, 'n', 'm')));
+ static::assertSame(['Rename: n → m', 'Changes columns: m, n'], $details->lines(new Rename($read, 'n', 'm')));
static::assertSame(
- ['Defines columns known only at run time'],
+ ['Changes columns known only at run time'],
$details->lines(new RenameEach($read, [rename_replace('_', '-')])),
);
}
@@ -188,7 +202,7 @@ public function test_declarations_are_the_row_count_transparency_materialization
static::assertSame('preserving · transparent · blocking', $details->declarations(new Collect($read)));
static::assertSame(
- 'preserving · transparent · streaming · redefines m',
+ 'preserving · transparent · streaming · redefines m, n',
$details->declarations(new Rename($read, 'n', 'm')),
);
static::assertSame('preserving · transparent · streaming · redefines unknown', $details->declarations(
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/FlowLayoutTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/FlowLayoutTest.php
index b6650dea73..d1ad96ed9c 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/FlowLayoutTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/FlowLayoutTest.php
@@ -72,7 +72,7 @@ public function test_every_reader_of_a_node_hangs_under_it(): void
├─ #3 Write
│ Loader: MemoryLoader
└─ #4 Select
- │ Columns: id
+ │ Keeps only: id
└─ #5 Write
Loader: MemoryLoader
PLAN, (new FlowLayout())->render((new Outline())->of($plan)));
@@ -89,7 +89,7 @@ public function test_every_source_starts_its_own_tree_and_a_node_reached_again_i
│ Extractor: ArrayExtractor
└─ #3 CrossJoin
│ Type: cross
- │ Defines columns known only at run time
+ │ Changes columns known only at run time
└─ #4 Result
Rows this plan hands out: to the trigger, or to the node reading it
#2 Read
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/PhysicalOutlineTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/PhysicalOutlineTest.php
index d8d612ddeb..77c2935cad 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/PhysicalOutlineTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/PhysicalOutlineTest.php
@@ -7,6 +7,7 @@
use Flow\ETL\Dataset\Memory\Unit;
use Flow\ETL\Plan\Explain\PhysicalOutline;
use Flow\ETL\Plan\Explain\TreeLayout;
+use Flow\ETL\Tests\Double\RecordingFileExtractor;
use Flow\ETL\Tests\Double\UndescribableRowLessExtractor;
use Flow\ETL\Tests\FlowTestCase;
use Flow\ETL\Tests\Mother\PhysicalPlanMother;
@@ -14,7 +15,11 @@
use function Flow\ETL\DSL\data_frame;
use function Flow\ETL\DSL\from_array;
use function Flow\ETL\DSL\hash_join;
+use function Flow\ETL\DSL\int_schema;
use function Flow\ETL\DSL\join_on;
+use function Flow\ETL\DSL\lit;
+use function Flow\ETL\DSL\ref;
+use function Flow\ETL\DSL\schema;
use function Flow\ETL\DSL\to_output;
final class PhysicalOutlineTest extends FlowTestCase
@@ -112,6 +117,23 @@ public function test_a_pushed_limit_is_listed_under_the_source_that_was_handed_i
);
}
+ public function test_a_pushed_filter_is_listed_under_the_file_source_that_was_offered_it(): void
+ {
+ $root = (new PhysicalOutline())->of(PhysicalPlanMother::of(
+ data_frame()
+ ->read(new RecordingFileExtractor(schema(int_schema('id'))))
+ ->filter(ref('id')->greaterThan(lit(1))),
+ ));
+
+ static::assertSame(
+ [
+ 'Extractor: RecordingFileExtractor',
+ ' Filter: GreaterThan',
+ ],
+ $root->children[0]->lines,
+ );
+ }
+
public function test_a_plan_that_could_not_derive_its_schema_says_why(): void
{
$root = (new PhysicalOutline())->of(PhysicalPlanMother::reading(new UndescribableRowLessExtractor()));
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/TreeLayoutTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/TreeLayoutTest.php
index 3f51fdfc36..8c5991cf56 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/TreeLayoutTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/TreeLayoutTest.php
@@ -91,7 +91,7 @@ public function test_every_child_but_the_last_keeps_the_rail_open(): void
└─ #5 Write
│ Loader: MemoryLoader
└─ #4 Select
- │ Columns: id
+ │ Keeps only: id
└─ #1 Read (shared)
PLAN, (new TreeLayout())->render((new Outline())->of($plan)));
}
@@ -105,8 +105,8 @@ public function test_with_declarations_a_single_chain_prints_one_line_per_node_w
│ Rows this plan hands out: to the trigger, or to the node reading it
└─ #3 Limit reducing · transparent · streaming
│ Limit: 5
- └─ #2 Select preserving · transparent · streaming
- │ Columns: id
+ └─ #2 Select preserving · transparent · streaming · redefines all but id
+ │ Keeps only: id
└─ #1 Read source · transparent · streaming
Extractor: ArrayExtractor
PLAN, (new TreeLayout())->render((new Outline(declarations: true))->of($plan->root)));
@@ -155,7 +155,7 @@ public function test_with_declarations_payloads_and_redefinitions(): void
│ Until: Literal
└─ #5 Offset reducing · transparent · streaming
│ Skip: 2
- └─ #4 Rename preserving · transparent · streaming · redefines m
+ └─ #4 Rename preserving · transparent · streaming · redefines m, n
│ Rename: n → m
└─ #3 WithColumn preserving · transparent · streaming · redefines n
│ Column: n = Literal
@@ -202,8 +202,8 @@ public function test_with_declarations_a_run_trigger_keeps_a_result_beside_the_w
Outputs preserving · opaque · streaming
├─ #3 Result preserving · transparent · streaming
│ │ Rows this plan hands out: to the trigger, or to the node reading it
- │ └─ #2 Select preserving · transparent · streaming
- │ │ Columns: id
+ │ └─ #2 Select preserving · transparent · streaming · redefines all but id
+ │ │ Keeps only: id
│ └─ #1 Read source · transparent · streaming
│ Extractor: ArrayExtractor
└─ #4 Write preserving · opaque · streaming
@@ -241,8 +241,8 @@ public function test_with_declarations_a_transaction_root_child_lists_its_writes
│ └─ #1 Read (shared)
└─ #5 Write preserving · opaque · streaming
│ Loader: MemoryLoader
- └─ #4 Select preserving · transparent · streaming
- │ Columns: id
+ └─ #4 Select preserving · transparent · streaming · redefines all but id
+ │ Keeps only: id
└─ #1 Read (shared)
PLAN, (new TreeLayout())->render((new Outline(declarations: true))->of($plan->root)));
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/LogicalPlanTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/LogicalPlanTest.php
index f3ba70577c..5c2f3c4cd7 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/LogicalPlanTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/LogicalPlanTest.php
@@ -13,7 +13,7 @@
use Flow\ETL\Plan\Node\Result;
use Flow\ETL\Plan\Node\Transaction;
use Flow\ETL\Plan\Node\Write;
-use Flow\ETL\Plan\ReplaceLeaf;
+use Flow\ETL\Plan\ReplaceNodes;
use Flow\ETL\Plan\Rewrite;
use Flow\ETL\Tests\Double\ChildlessNode;
use Flow\ETL\Tests\Double\RecordingTransaction;
@@ -107,12 +107,12 @@ public function test_consumer_inputs_include_the_inputs_of_sinks_below_the_root(
static::assertSame([$select, $limit], NodeMother::plan($select)->consumerInputs());
}
- public function test_transform_up_with_replace_leaf_rewrites_the_leaf_and_keeps_the_spine(): void
+ public function test_transform_up_with_replace_nodes_rewrites_the_leaf_and_keeps_the_spine(): void
{
$replacement = NodeMother::read(from_array([['id' => 2]]));
$plan = NodeMother::plan(NodeMother::select(NodeMother::read()));
- $rewritten = $plan->transformUp(new ReplaceLeaf($plan->source(), $replacement));
+ $rewritten = $plan->transformUp(ReplaceNodes::one($plan->source(), $replacement));
static::assertSame($replacement, $rewritten->source());
$select = $rewritten->root->children()[0];
@@ -148,9 +148,9 @@ public function test_transform_up_refuses_an_outputs_consumer_rewritten_to_a_non
);
$plan->transformUp(new class implements Rewrite {
- public function of(Node $node): Node
+ public function of(Node $original, Node $rebuilt): Node
{
- return $node instanceof Write ? $node->children()[0] : $node;
+ return $rebuilt instanceof Write ? $rebuilt->children()[0] : $rebuilt;
}
});
}
@@ -186,9 +186,9 @@ public function test_transform_up_rebuilds_bottom_up(): void
$plan = NodeMother::plan(NodeMother::limit(NodeMother::select($read), 5));
$rewritten = $plan->transformUp(new class implements Rewrite {
- public function of(Node $node): Node
+ public function of(Node $original, Node $rebuilt): Node
{
- return $node instanceof Limit ? new Limit($node->children()[0], 1) : $node;
+ return $rebuilt instanceof Limit ? new Limit($rebuilt->children()[0], 1) : $rebuilt;
}
});
@@ -205,9 +205,9 @@ public function test_transform_up_that_changes_nothing_returns_the_same_plan():
$plan = NodeMother::plan(NodeMother::limit(NodeMother::select(NodeMother::read()), 5));
$rewritten = $plan->transformUp(new class implements Rewrite {
- public function of(Node $node): Node
+ public function of(Node $original, Node $rebuilt): Node
{
- return $node;
+ return $rebuilt;
}
});
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/DropTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/DropTest.php
index 494bd90af7..df0a4cb119 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/DropTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/DropTest.php
@@ -44,6 +44,6 @@ public function test_declarations(): void
static::assertSame(RowCount::preserving, $node->rowCount());
static::assertSame(Transparency::transparent, $node->transparency());
static::assertSame(Materialization::streaming, $node->materialization());
- static::assertEquals(Redefined::none(), $node->redefines());
+ static::assertEquals(Redefined::names('id'), $node->redefines());
}
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/ReadTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/ReadTest.php
index 459547604d..f120798b2e 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/ReadTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/ReadTest.php
@@ -5,18 +5,21 @@
namespace Flow\ETL\Tests\Unit\Plan\Node;
use Flow\ETL\Exception\InvalidArgumentException;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Plan\Materialization;
use Flow\ETL\Plan\Node\Read;
use Flow\ETL\Plan\Redefined;
use Flow\ETL\Plan\RowCount;
use Flow\ETL\Plan\Transparency;
use Flow\ETL\Tests\FlowTestCase;
-use Flow\Filesystem\Path\Filter\Filters;
+use Flow\ETL\Tests\Mother\PartitionedSourceMother;
use Flow\Filesystem\Path\Filter\OnlyFiles;
use Flow\Filesystem\Tests\Double\RejectingFilter;
use function Flow\ETL\DSL\from_array;
use function Flow\ETL\DSL\int_schema;
+use function Flow\ETL\DSL\lit;
+use function Flow\ETL\DSL\ref;
use function Flow\ETL\DSL\schema;
final class ReadTest extends FlowTestCase
@@ -37,62 +40,74 @@ public function test_a_default_read_pushes_no_limit_and_lists_only_files(): void
{
$read = new Read(from_array([['id' => 1]]));
- static::assertNull($read->limit());
- static::assertInstanceOf(OnlyFiles::class, $read->pathFilter());
+ static::assertNull($read->scan()->limit);
+ static::assertInstanceOf(OnlyFiles::class, $read->scan()->pathFilter);
}
- public function test_extractor_limit_and_path_filter_are_the_values_it_was_built_with(): void
+ public function test_extractor_and_scan_are_the_values_it_was_built_with(): void
{
$extractor = from_array([['id' => 1]]);
- $filter = new RejectingFilter();
+ $scan = new FileScan(5, new RejectingFilter());
- $read = new Read($extractor, 5, $filter);
+ $read = new Read($extractor, $scan);
static::assertSame($extractor, $read->extractor());
- static::assertSame(5, $read->limit());
- static::assertSame($filter, $read->pathFilter());
+ static::assertSame($scan, $read->scan());
}
- public function test_with_path_filter_composes_with_the_filters_already_held(): void
+ public function test_with_path_filter_and_with_filter_push_into_the_scan_and_leave_the_original_alone(): void
{
- $extractor = from_array([['id' => 1]]);
- $read = new Read($extractor, 3);
- $first = new RejectingFilter();
- $second = new RejectingFilter();
+ $extractor = PartitionedSourceMother::yearMonth();
+ $read = new Read($extractor, new FileScan(3));
+ $pathFilter = new RejectingFilter();
+ $filter = ref('year')->greaterThan(lit(1));
- $filtered = $read->withPathFilter($first)->withPathFilter($second);
+ $pushed = $read->withPathFilter($pathFilter)->withFilter($filter);
- static::assertInstanceOf(OnlyFiles::class, $read->pathFilter());
- static::assertEquals(new Filters(new OnlyFiles(), $first, $second), $filtered->pathFilter());
- static::assertSame(3, $filtered->limit());
- static::assertSame($extractor, $filtered->extractor());
+ static::assertEquals(new FileScan(3), $read->scan());
+ static::assertEquals(
+ (new FileScan(3))
+ ->withPathFilter($pathFilter)
+ ->withFilter($filter),
+ $pushed->scan(),
+ );
+ static::assertSame($extractor, $pushed->extractor());
}
- public function test_with_limit_returns_a_new_read_and_leaves_the_original_alone(): void
+ public function test_with_limit_pushes_into_the_scan_of_any_extractor(): void
{
- $read = (new Read(from_array([['id' => 1]])))->withPathFilter($filter = new OnlyFiles());
+ $read = new Read(from_array([['id' => 1]]));
- $limited = $read->withLimit(5);
+ static::assertNull($read->scan()->limit);
+ static::assertSame(10, $read->withLimit(10)->withLimit(100)->scan()->limit);
+ }
+
+ public function test_with_limit_refuses_zero_or_less(): void
+ {
+ $this->expectException(InvalidArgumentException::class);
+ $this->expectExceptionMessage('Limit must be greater than 0');
- static::assertNull($read->limit());
- static::assertSame(5, $limited->limit());
- static::assertEquals(new Filters(new OnlyFiles(), $filter), $limited->pathFilter());
+ (new Read(from_array([['id' => 1]])))->withLimit(0);
}
- public function test_with_limit_narrows_and_never_widens(): void
+ public function test_a_filter_is_refused_for_an_extractor_that_is_no_file_extractor(): void
{
- $read = new Read(from_array([['id' => 1]]));
+ $this->expectException(InvalidArgumentException::class);
+ $this->expectExceptionMessage(
+ 'Only a FileExtractor reads a pushed filter, Flow\ETL\Extractor\ArrayExtractor does not',
+ );
- static::assertSame(10, $read->withLimit(10)->withLimit(100)->limit());
- static::assertSame(10, $read->withLimit(100)->withLimit(10)->limit());
+ (new Read(from_array([['id' => 1]])))->withFilter(ref('id')->greaterThan(lit(1)));
}
- public function test_with_limit_refuses_zero_or_less(): void
+ public function test_a_path_filter_is_refused_for_an_extractor_that_is_no_file_extractor(): void
{
$this->expectException(InvalidArgumentException::class);
- $this->expectExceptionMessage('Limit must be greater than 0');
+ $this->expectExceptionMessage(
+ 'Only a FileExtractor reads a pushed filter, Flow\ETL\Extractor\ArrayExtractor does not',
+ );
- (new Read(from_array([['id' => 1]])))->withLimit(0);
+ (new Read(from_array([['id' => 1]])))->withPathFilter(new RejectingFilter());
}
public function test_schema_delegates_to_the_extractor(): void
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/RenameTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/RenameTest.php
index 5776bf92ca..72205a2f26 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/RenameTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/RenameTest.php
@@ -45,6 +45,6 @@ public function test_declarations(): void
static::assertSame(RowCount::preserving, $node->rowCount());
static::assertSame(Transparency::transparent, $node->transparency());
static::assertSame(Materialization::streaming, $node->materialization());
- static::assertEquals(Redefined::alias('user_id', 'id'), $node->redefines());
+ static::assertEquals(Redefined::rename('id', 'user_id'), $node->redefines());
}
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/SelectTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/SelectTest.php
index f934c68ef9..845480c535 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/SelectTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Node/SelectTest.php
@@ -44,6 +44,6 @@ public function test_declarations(): void
static::assertSame(RowCount::preserving, $node->rowCount());
static::assertSame(Transparency::transparent, $node->transparency());
static::assertSame(Materialization::streaming, $node->materialization());
- static::assertEquals(Redefined::none(), $node->redefines());
+ static::assertEquals(Redefined::allBut('id'), $node->redefines());
}
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/RedefinedTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/RedefinedTest.php
index 21ddb51400..b5e0442dae 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/RedefinedTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/RedefinedTest.php
@@ -13,7 +13,8 @@ public function test_none_defines_nothing(): void
{
static::assertFalse(Redefined::none()->defines('year'));
static::assertSame([], Redefined::none()->names);
- static::assertFalse(Redefined::none()->unknown);
+ static::assertFalse(Redefined::none()->isUnknown());
+ static::assertFalse(Redefined::none()->isAllBut());
}
public function test_names_define_exactly_those_names_and_alias_none(): void
@@ -30,7 +31,8 @@ public function test_unknown_defines_every_name_and_aliases_none(): void
{
static::assertTrue(Redefined::unknown()->defines('anything'));
static::assertNull(Redefined::unknown()->aliasOf('anything'));
- static::assertTrue(Redefined::unknown()->unknown);
+ static::assertTrue(Redefined::unknown()->isUnknown());
+ static::assertFalse(Redefined::unknown()->isAllBut());
}
public function test_names_with_no_names_defines_nothing(): void
@@ -49,4 +51,27 @@ public function test_an_alias_defines_its_name_and_points_it_at_the_column_below
static::assertNull($redefined->aliasOf('year'));
static::assertSame(['y'], $redefined->names);
}
+
+ public function test_all_but_defines_every_name_except_the_kept_ones(): void
+ {
+ $redefined = Redefined::allBut('year');
+
+ static::assertFalse($redefined->defines('year'));
+ static::assertTrue($redefined->defines('month'));
+ static::assertNull($redefined->aliasOf('month'));
+ static::assertSame(['year'], $redefined->names);
+ static::assertTrue($redefined->isAllBut());
+ static::assertFalse($redefined->isUnknown());
+ }
+
+ public function test_a_rename_defines_both_names_and_points_the_new_one_at_the_old_one(): void
+ {
+ $redefined = Redefined::rename('year', 'y');
+
+ static::assertTrue($redefined->defines('y'));
+ static::assertTrue($redefined->defines('year'));
+ static::assertSame('year', $redefined->aliasOf('y'));
+ static::assertNull($redefined->aliasOf('year'));
+ static::assertSame(['y', 'year'], $redefined->names);
+ }
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/ReplaceLeafTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/ReplaceLeafTest.php
deleted file mode 100644
index 720dace269..0000000000
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/ReplaceLeafTest.php
+++ /dev/null
@@ -1,27 +0,0 @@
-of($target));
- }
-
- public function test_any_other_node_is_returned_unchanged(): void
- {
- $other = NodeMother::select(NodeMother::read());
-
- static::assertSame($other, (new ReplaceLeaf(NodeMother::read(), NodeMother::read()))->of($other));
- }
-}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/ReplaceNodesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/ReplaceNodesTest.php
new file mode 100644
index 0000000000..3853d01c2f
--- /dev/null
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/ReplaceNodesTest.php
@@ -0,0 +1,101 @@
+of($target, $target));
+ static::assertSame($other, $rewrite->of($other, $other));
+ }
+
+ public function test_a_replacement_takes_the_targets_rewritten_children(): void
+ {
+ $target = NodeMother::select(NodeMother::read());
+ $rebuilt = $target->withChildren([$read = NodeMother::read()]);
+
+ $replaced = ReplaceNodes::one($target, NodeMother::select($target->children()[0], 'name'))->of(
+ $target,
+ $rebuilt,
+ );
+
+ static::assertSame([$read], $replaced->children());
+ }
+
+ public function test_a_removed_node_gives_way_to_its_rewritten_child(): void
+ {
+ $target = new Filter(NodeMother::read(), lit(true));
+ $rebuilt = $target->withChildren([$read = NodeMother::read()]);
+
+ /** @var SplObjectStorage $replacements */
+ $replacements = new SplObjectStorage();
+
+ static::assertSame($read, (new ReplaceNodes($replacements, [$target]))->of($target, $rebuilt));
+ }
+
+ public function test_a_target_replaced_by_its_own_child_keeps_the_childs_rewritten_children(): void
+ {
+ $child = NodeMother::select(NodeMother::read());
+ $target = new Filter($child, lit(true));
+ $rebuilt = $target->withChildren([NodeMother::select(NodeMother::read())]);
+
+ static::assertSame($rebuilt->children(), ReplaceNodes::one($target, $child)->of($target, $rebuilt)->children());
+ }
+
+ public function test_a_node_of_other_than_one_child_is_not_removed(): void
+ {
+ $target = NodeMother::read();
+ /** @var SplObjectStorage $replacements */
+ $replacements = new SplObjectStorage();
+
+ $this->expectException(InvalidArgumentException::class);
+ $this->expectExceptionMessage('Only a node of one child gives way to it, ' . $target::class . ' has 0');
+
+ (new ReplaceNodes($replacements, [$target]))->of($target, $target);
+ }
+
+ public function test_a_node_both_replaced_and_removed_is_refused(): void
+ {
+ $target = new Filter(NodeMother::read(), lit(true));
+ /** @var SplObjectStorage $replacements */
+ $replacements = new SplObjectStorage();
+ $replacements[$target] = NodeMother::read();
+
+ $this->expectException(InvalidArgumentException::class);
+ $this->expectExceptionMessage('A node is either replaced or removed, ' . Filter::class . ' is both');
+
+ new ReplaceNodes($replacements, [$target]);
+ }
+
+ public function test_several_targets_are_replaced_in_one_rewrite(): void
+ {
+ $first = NodeMother::read();
+ $second = NodeMother::read();
+ /** @var SplObjectStorage $replacements */
+ $replacements = new SplObjectStorage();
+ $replacements[$first] = $firstReplacement = NodeMother::read();
+ $replacements[$second] = $secondReplacement = NodeMother::read();
+ $rewrite = new ReplaceNodes($replacements);
+
+ static::assertSame($firstReplacement, $rewrite->of($first, $first));
+ static::assertSame($secondReplacement, $rewrite->of($second, $second));
+ }
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/TransformUpTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/TransformUpTest.php
index 94628a6ba5..c74e53d68f 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/TransformUpTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/TransformUpTest.php
@@ -34,11 +34,11 @@ public function __construct(
private array &$visited,
) {}
- public function of(Node $node): Node
+ public function of(Node $original, Node $rebuilt): Node
{
- $this->visited[] = $node::class;
+ $this->visited[] = $rebuilt::class;
- return $node;
+ return $rebuilt;
}
});
@@ -50,9 +50,9 @@ public function test_a_rewrite_that_changes_nothing_returns_the_same_root_object
$root = NodeMother::limit(NodeMother::select(NodeMother::read()), 5);
static::assertSame($root, (new TransformUp())->of($root, new class implements Rewrite {
- public function of(Node $node): Node
+ public function of(Node $original, Node $rebuilt): Node
{
- return $node;
+ return $rebuilt;
}
}));
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/PipelineSplitTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/PipelineSplitTest.php
index 3ec9d45a48..62a19a2452 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/PipelineSplitTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/PipelineSplitTest.php
@@ -665,8 +665,8 @@ public function test_the_leaf_pipeline_carries_the_reads_limit_and_path_filter()
$plan = (new PipelineSplit())->of($logical, $planned, NodeMother::context());
- static::assertSame(5, $plan->root()->input()?->limit());
- static::assertNull($plan->root()->limit());
+ static::assertSame(5, $plan->root()->input()?->scan()->limit);
+ static::assertNull($plan->root()->scan()->limit);
}
public function test_only_the_pipeline_that_reads_the_source_counts_its_rows(): void
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/PlannerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/PlannerTest.php
index 191d1d00aa..e5feb8008f 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/PlannerTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/PlannerTest.php
@@ -90,7 +90,7 @@ public function test_the_default_rules_keep_every_sink_on_the_spine(): void
);
$steps = PipelineSteps::of($plan->root()->segments());
- static::assertSame(4, $plan->root()->limit());
+ static::assertSame(4, $plan->root()->scan()->limit);
static::assertSame(
[SinkFeed::class, LimitTransformer::class, MemoryLoader::class, SelectEntriesTransformer::class],
array_map(static fn($step) => $step::class, $steps),
@@ -141,8 +141,8 @@ public function test_a_linear_plan_is_one_pipeline_with_the_limit_pushed(): void
static::assertSame(['id', 'total'], $plan->schema->references()->names());
static::assertSame($csv, $plan->root()->segments()->extractor());
- static::assertSame(5, $plan->root()->limit());
- static::assertNull($logical->source()->limit());
+ static::assertSame(5, $plan->root()->scan()->limit);
+ static::assertNull($logical->source()->scan()->limit);
}
public function test_a_run_plan_over_one_sink_is_described_with_the_sinks_input_schema(): void
@@ -189,8 +189,8 @@ public function test_a_sort_splits_the_plan_into_two_pipelines(): void
static::assertSame(['id', 'total', 'seller_id'], $plan->schema->references()->names());
static::assertSame($csv, $upstream->segments()->extractor());
- static::assertNull($upstream->limit());
- static::assertNull($plan->root()->limit());
+ static::assertNull($upstream->scan()->limit);
+ static::assertNull($plan->root()->scan()->limit);
}
public function test_a_join_each_plan_refuses_its_schema_and_has_no_frame_edge(): void
@@ -234,7 +234,7 @@ public function test_a_planner_with_no_rules_plans_without_a_push(): void
$plan = (new Planner())->plan($logical, NodeMother::context());
static::assertSame($csv, $plan->root()->segments()->extractor());
- static::assertNull($plan->root()->limit());
+ static::assertNull($plan->root()->scan()->limit);
}
public function test_a_join_plans_its_right_side_as_a_physical_plan_of_its_own(): void
@@ -316,7 +316,7 @@ public function test_a_read_frame_is_a_source_of_this_pipeline(): void
PipelineSteps::classes($plan->root()->segments()),
);
static::assertSame($extractor, $plan->root()->segments()->extractor());
- static::assertSame(5, $plan->root()->limit());
+ static::assertSame(5, $plan->root()->scan()->limit);
}
public function test_a_read_frame_describes_the_rows_without_reading_them(): void
@@ -412,10 +412,7 @@ public function test_a_limit_over_a_read_frame_is_pushed_into_its_extractor(): v
static::assertSame(
1,
- (new Planner(Optimizer::default()))
- ->plan($logical, NodeMother::context())
- ->root()
- ->limit(),
+ (new Planner(Optimizer::default()))->plan($logical, NodeMother::context())->root()->scan()->limit,
);
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/StringDefinitionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/StringDefinitionTest.php
index 2fdc52f2c5..3f66734e82 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/StringDefinitionTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/StringDefinitionTest.php
@@ -13,9 +13,14 @@
use PHPUnit\Framework\Attributes\DataProvider;
use function Flow\ETL\DSL\bool_schema;
+use function Flow\ETL\DSL\definition_from_array;
+use function Flow\ETL\DSL\definition_from_type;
use function Flow\ETL\DSL\int_schema;
use function Flow\ETL\DSL\null_schema;
use function Flow\ETL\DSL\string_schema;
+use function Flow\Types\DSL\type_equals;
+use function Flow\Types\DSL\type_numeric_string;
+use function Flow\Types\DSL\type_string;
final class StringDefinitionTest extends FlowTestCase
{
@@ -67,6 +72,25 @@ public static function provideMergeCases(): Generator
];
}
+ public function test_a_numeric_string_definition_keeps_its_type_through_every_rebuild(): void
+ {
+ $def = definition_from_type('id', type_numeric_string());
+
+ static::assertInstanceOf(StringDefinition::class, $def);
+ static::assertTrue(type_equals(type_numeric_string(), $def->type()));
+ static::assertTrue(type_equals(type_numeric_string(), $def->makeNullable()->type()));
+ static::assertTrue(type_equals(type_numeric_string(), $def->rename('other')->type()));
+ static::assertTrue(type_equals(type_numeric_string(), $def->setMetadata(Metadata::with('k', 'v'))->type()));
+ static::assertTrue(type_equals(type_numeric_string(), $def->addMetadata('k', 'v')->type()));
+ static::assertTrue(type_equals(
+ type_numeric_string(),
+ $def->merge(definition_from_type('id', type_numeric_string()))->type(),
+ ));
+ static::assertTrue(type_equals(type_numeric_string(), $def->merge(null_schema('id'))->type()));
+ static::assertTrue(type_equals(type_string(), $def->merge(string_schema('id'))->type()));
+ static::assertTrue($def->isSame(definition_from_array($def->normalize())));
+ }
+
public function test_add_metadata(): void
{
$def = string_schema('name');
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Formatter/PHPSchemaFormatterTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Formatter/PHPSchemaFormatterTest.php
index 4a7d3f8ccb..9eda4c6c97 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Formatter/PHPSchemaFormatterTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Formatter/PHPSchemaFormatterTest.php
@@ -12,6 +12,7 @@
use function Flow\ETL\DSL\bool_schema;
use function Flow\ETL\DSL\date_schema;
use function Flow\ETL\DSL\datetime_schema;
+use function Flow\ETL\DSL\definition_from_type;
use function Flow\ETL\DSL\enum_schema;
use function Flow\ETL\DSL\float_schema;
use function Flow\ETL\DSL\html_element_schema;
@@ -31,6 +32,7 @@
use function Flow\Types\DSL\type_integer;
use function Flow\Types\DSL\type_list;
use function Flow\Types\DSL\type_map;
+use function Flow\Types\DSL\type_numeric_string;
use function Flow\Types\DSL\type_optional;
use function Flow\Types\DSL\type_string;
use function Flow\Types\DSL\type_structure;
@@ -88,6 +90,15 @@ public function test_formatting_map_schema(): void
))));
}
+ public function test_formatting_numeric_string_schema(): void
+ {
+ static::assertEquals(<<<'PHP'
+ \Flow\ETL\DSL\schema(
+ \Flow\ETL\DSL\definition_from_type("id", type: \Flow\Types\DSL\type_numeric_string(), nullable: true, metadata: \Flow\ETL\DSL\schema_metadata()),
+ );
+ PHP, (new PHPSchemaFormatter())->format(schema(definition_from_type('id', type_numeric_string(), true))));
+ }
+
public function test_formatting_simple_schema(): void
{
static::assertEquals(<<explain()->toString(format: Format::declarations));
}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionFilterTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionFilterTransformerTest.php
index 313fda2876..68ee8e404a 100644
--- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionFilterTransformerTest.php
+++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionFilterTransformerTest.php
@@ -231,4 +231,18 @@ public function test_bind_refuses_a_root_expand_over_booleans(): void
(new ScalarFunctionFilterTransformer(ref('flags')->expand()))->bind(ListColumnsMother::schema());
}
+
+ public function test_the_bound_transformer_keeps_the_rows_its_predicate_matches(): void
+ {
+ $input = schema(int_schema('a'));
+ $bound = (new ScalarFunctionFilterTransformer(ref('a')->greaterThan(lit(1))))->bind($input)->step;
+
+ static::assertInstanceOf(ScalarFunctionFilterTransformer::class, $bound);
+ static::assertSame(
+ [['a' => 2], ['a' => 3]],
+ $bound
+ ->transform(array_to_rows([['a' => 1], ['a' => 2], ['a' => 3]], $input), flow_context(config()))
+ ->toArray(),
+ );
+ }
}
diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeExtractorTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeExtractorTest.php
index 6dc5defddd..3510f1bdda 100644
--- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeExtractorTest.php
+++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeExtractorTest.php
@@ -6,6 +6,7 @@
use Flow\ETL\Exception\InferredSchemaException;
use Flow\ETL\Exception\InvalidArgumentException;
+use Flow\ETL\Extractor\File\FileScan;
use Flow\ETL\Extractor\Signal;
use Flow\ETL\Tests\Double\CountingFilesystem;
use Flow\ETL\Tests\Double\SpyBackend;
@@ -489,7 +490,10 @@ public function test_a_path_filter_narrows_the_read_but_not_the_schema(): void
$extractor = from_floe(path('memory://parts/*/*.floe'), filesystem: $counting);
$schema = $extractor->schema();
- $batches = iterator_to_array($extractor->extract(flow_context(config()), pathFilter: new RejectingFilter()));
+ $batches = iterator_to_array($extractor->scan(
+ flow_context(config()),
+ new FileScan(pathFilter: new RejectingFilter()),
+ ));
static::assertSame([], $batches);
static::assertSame(1, $counting->readFromCalls);
diff --git a/src/extension/arrow-ext/Cargo.lock b/src/extension/arrow-ext/Cargo.lock
index d69bb02fa8..1e7d3c2577 100644
--- a/src/extension/arrow-ext/Cargo.lock
+++ b/src/extension/arrow-ext/Cargo.lock
@@ -44,15 +44,15 @@ dependencies = [
[[package]]
name = "alloc-no-stdlib"
-version = "2.0.4"
+version = "3.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
+checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a"
[[package]]
name = "alloc-stdlib"
-version = "0.2.2"
+version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
+checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1"
dependencies = [
"alloc-no-stdlib",
]
@@ -87,9 +87,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "59.3.0"
+version = "60.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e5f6adeffdf587d7a31db5d2266189624b526730cd3627f9ff9fedae97ad584"
+checksum = "51dff6e4b9f158864a0aeb6a131ad53858ecbae7a2fc8989307b9b3c6b2f122e"
dependencies = [
"ahash",
"arrow-buffer",
@@ -106,9 +106,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "59.3.0"
+version = "60.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "097d193003ce7995d5d087089069ec2a6e0187faf5a6f8c9f38af2645d987182"
+checksum = "c7235b863533e9db3ab86905b4521251de11226275b436ff3cb2d0010558d1c5"
dependencies = [
"bytes",
"half",
@@ -116,11 +116,22 @@ dependencies = [
"num-traits",
]
+[[package]]
+name = "arrow-cmp"
+version = "60.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73a0e9adb28d95739fabcdae637261a2c14e618caf58ba1b88857120f9ee2687"
+dependencies = [
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-schema",
+]
+
[[package]]
name = "arrow-data"
-version = "59.3.0"
+version = "60.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ba2f832eaeca24b8f26143dba750e42ee4ab51cf7d65e701ca9607cfda9f358"
+checksum = "5aa06d5e22786c4966ffc4d2f171d0cb65d95f6bdea108a14e8c335e9faa437f"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -131,9 +142,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "59.3.0"
+version = "60.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcc41681ea80f521df14c36725b74d4c60702c47f0793af2be469c04527e2599"
+checksum = "86c523472d22e31f984df7a44fc36e6323d3c9b05ffe53d267ee4c3aa7c128d2"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -145,9 +156,9 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "59.3.0"
+version = "60.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd"
+checksum = "f85a9a58cd03560526b0066487574aacb9008e413504fa0ee23d61141e96c338"
dependencies = [
"bitflags",
"serde_core",
@@ -156,13 +167,14 @@ dependencies = [
[[package]]
name = "arrow-select"
-version = "59.3.0"
+version = "60.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc58569193c2525915f3cc6310edba3792f1200f65d6e9ed330aa33e691493b8"
+checksum = "ba3482b7d03bf4a3bf0b195f08a7f2b071595ccd54897e35b975ac63d3104977"
dependencies = [
"ahash",
"arrow-array",
"arrow-buffer",
+ "arrow-cmp",
"arrow-data",
"arrow-schema",
"num-traits",
@@ -207,7 +219,7 @@ dependencies = [
"regex",
"rustc-hash",
"shlex 2.0.1",
- "syn 2.0.117",
+ "syn 3.0.6",
]
[[package]]
@@ -227,9 +239,9 @@ dependencies = [
[[package]]
name = "brotli"
-version = "8.0.2"
+version = "9.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560"
+checksum = "f8b851b75c23ca7873623d612fe49bd1989aeb03d08fb9432187eb253d3d4c6b"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
@@ -238,9 +250,9 @@ dependencies = [
[[package]]
name = "brotli-decompressor"
-version = "5.0.0"
+version = "6.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03"
+checksum = "941cd9bd4ddab83cb46fa5a2d428f1c857b24ac78cb876cf7beb710840934bd7"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
@@ -1115,9 +1127,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "59.3.0"
+version = "60.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff322f54b1a0f9288e614ed1f2d329b380af5476420db19f46ffb865e1163d73"
+checksum = "8af83d2940bc0510f9aef86d865f56fdc6095f87ab115ac885a80b7c5226d3ba"
dependencies = [
"ahash",
"arrow-array",
@@ -1141,7 +1153,7 @@ dependencies = [
"simdutf8",
"snap",
"twox-hash",
- "zstd",
+ "zstd 0.14.2",
]
[[package]]
@@ -2021,7 +2033,7 @@ dependencies = [
"typed-path",
"zeroize",
"zopfli",
- "zstd",
+ "zstd 0.13.3",
]
[[package]]
@@ -2054,7 +2066,16 @@ version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
dependencies = [
- "zstd-safe",
+ "zstd-safe 7.2.4",
+]
+
+[[package]]
+name = "zstd"
+version = "0.14.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "057cfd910cfac363a0ada849592624b4c9ff2e10bef504c3433810d78ed96f93"
+dependencies = [
+ "zstd-safe 8.1.0",
]
[[package]]
@@ -2066,11 +2087,20 @@ dependencies = [
"zstd-sys",
]
+[[package]]
+name = "zstd-safe"
+version = "8.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bdd44c6a7284e91f3717755b24315a302edd9153a01f753c3cba3d765e8eafac"
+dependencies = [
+ "zstd-sys",
+]
+
[[package]]
name = "zstd-sys"
-version = "2.0.16+zstd.1.5.7"
+version = "2.1.1+zstd.1.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
+checksum = "aeec9eaf2dffbbd09201e23bd0ffcbaa33bb8e9266a10734fd7ed90a85eca078"
dependencies = [
"cc",
"pkg-config",
diff --git a/src/extension/arrow-ext/Cargo.toml b/src/extension/arrow-ext/Cargo.toml
index e528e0f770..0bbb474083 100644
--- a/src/extension/arrow-ext/Cargo.toml
+++ b/src/extension/arrow-ext/Cargo.toml
@@ -8,10 +8,10 @@ crate-type = ["cdylib"]
[dependencies]
ext-php-rs = "0.16"
-parquet = { version = "59", default-features = false, features = ["arrow", "arrow_canonical_extension_types", "snap", "flate2", "flate2-zlib-rs", "zstd", "lz4", "brotli", "simdutf8"] }
-arrow-array = { version = "59", features = ["ffi"] }
-arrow-buffer = "59"
-arrow-schema = { version = "59", features = ["canonical_extension_types"] }
+parquet = { version = "60", default-features = false, features = ["arrow", "arrow_canonical_extension_types", "snap", "flate2", "flate2-zlib-rs", "zstd", "lz4", "brotli", "simdutf8"] }
+arrow-array = { version = "60", features = ["ffi"] }
+arrow-buffer = "60"
+arrow-schema = { version = "60", features = ["canonical_extension_types"] }
bytes = "1"
[build-dependencies]
diff --git a/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustBatchReader.php b/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustBatchReader.php
index 6ea532f5c8..473d266161 100644
--- a/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustBatchReader.php
+++ b/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustBatchReader.php
@@ -7,6 +7,7 @@
use Flow\Arrow\RustArrowSchema;
use Flow\Arrow\RustParquetBatch;
use Flow\Parquet\Engine\RustParquetFileReader;
+use Flow\Parquet\Reader\ReadPlan;
use RuntimeException;
use function extension_loaded;
@@ -21,7 +22,7 @@ final class RustBatchReader
* @param list $columns root column names
* @param int<1, max> $batchSize
*/
- public function __construct(RustParquetFileReader $file, array $columns, int $batchSize, ?int $offset, ?int $limit)
+ public function __construct(RustParquetFileReader $file, array $columns, int $batchSize, ReadPlan $plan)
{
throw new RuntimeException('arrow extension is not loaded');
}
diff --git a/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustColumnsReader.php b/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustColumnsReader.php
index bf7ee0a131..c39b3400d5 100644
--- a/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustColumnsReader.php
+++ b/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustColumnsReader.php
@@ -5,6 +5,7 @@
namespace Flow\Arrow\Parquet;
use Flow\Parquet\Engine\RustParquetFileReader;
+use Flow\Parquet\Reader\ReadPlan;
use Iterator;
use RuntimeException;
@@ -25,7 +26,7 @@ final class RustColumnsReader implements Iterator
* @param list $columns root names or struct paths
* @param int<1, max> $batchSize
*/
- public function __construct(RustParquetFileReader $file, array $columns, int $batchSize, ?int $offset, ?int $limit)
+ public function __construct(RustParquetFileReader $file, array $columns, int $batchSize, ReadPlan $plan)
{
throw new RuntimeException('arrow extension is not loaded');
}
diff --git a/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFileReader.php b/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFileReader.php
index 7e041d8abc..38e400fded 100644
--- a/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFileReader.php
+++ b/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFileReader.php
@@ -6,10 +6,8 @@
use Flow\Arrow\Parquet\RustColumnsReader;
use Flow\Filesystem\SourceStream;
-use Flow\Parquet\ParquetFile\Metadata;
-use Flow\Parquet\ParquetFile\Schema;
use Flow\Parquet\ParquetFileReader;
-use Flow\Parquet\ThriftModel\FileMetaData;
+use Flow\Parquet\Reader\ReadPlan;
use RuntimeException;
use function extension_loaded;
@@ -30,49 +28,27 @@ public function __construct(SourceStream $stream, bool $int96AsDatetime = true)
}
/**
- * Closes the stream; any later call throws \Flow\Parquet\Exception\RuntimeException('Reader is not open').
+ * Closes the stream; any later call throws \Flow\Parquet\Exception\RuntimeException('Reader is not open'). A
+ * footer taken before stays readable.
*/
public function close(): void
{
throw new RuntimeException('arrow extension is not loaded');
}
- public function metadata(): Metadata
- {
- throw new RuntimeException('arrow extension is not loaded');
- }
-
- /**
- * @param list $columns resolved names, never empty
- * @param int<1, max> $batchSize upper bound of rows per chunk
- */
- public function readColumns(array $columns, int $batchSize, ?int $limit, ?int $offset): RustColumnsReader
- {
- throw new RuntimeException('arrow extension is not loaded');
- }
-
- public function rowsNumber(): int
- {
- throw new RuntimeException('arrow extension is not loaded');
- }
-
- public function schema(): Schema
- {
- throw new RuntimeException('arrow extension is not loaded');
- }
-
/**
- * The whole footer, decoded on the first call.
+ * Created on the first call.
*/
- public function thrift(): FileMetaData
+ public function footer(): RustParquetFooter
{
throw new RuntimeException('arrow extension is not loaded');
}
/**
- * Σ row group total_byte_size (uncompressed).
+ * @param list $columns resolved names, never empty
+ * @param int $batchSize upper bound of rows per chunk, refused below 1
*/
- public function totalByteSize(): int
+ public function readColumns(array $columns, int $batchSize, ReadPlan $plan): RustColumnsReader
{
throw new RuntimeException('arrow extension is not loaded');
}
diff --git a/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFooter.php b/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFooter.php
new file mode 100644
index 0000000000..6eaf75423a
--- /dev/null
+++ b/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFooter.php
@@ -0,0 +1,67 @@
+ $columns
+ */
+ public function rowGroupChunks(array $columns): RowGroupChunks
+ {
+ throw new RuntimeException('arrow extension is not loaded');
+ }
+
+ public function rowsNumber(): int
+ {
+ throw new RuntimeException('arrow extension is not loaded');
+ }
+
+ public function schema(): Schema
+ {
+ throw new RuntimeException('arrow extension is not loaded');
+ }
+
+ /**
+ * The whole footer, decoded on the first call.
+ */
+ public function thrift(): FileMetaData
+ {
+ throw new RuntimeException('arrow extension is not loaded');
+ }
+
+ /**
+ * Σ row group total_byte_size (uncompressed).
+ */
+ public function totalByteSize(): int
+ {
+ throw new RuntimeException('arrow extension is not loaded');
+ }
+}
diff --git a/src/extension/arrow-ext/src/interfaces.rs b/src/extension/arrow-ext/src/interfaces.rs
index cc8589a753..316de354b9 100644
--- a/src/extension/arrow-ext/src/interfaces.rs
+++ b/src/extension/arrow-ext/src/interfaces.rs
@@ -1,4 +1,4 @@
-//! `Flow\Parquet\{ParquetEngine, ParquetFileReader, ParquetFileWriter}`: registered at MINIT with the package files'
+//! `Flow\Parquet\{ParquetEngine, ParquetFileReader, ParquetFileWriter}` and `Flow\Parquet\ParquetFile\Footer`: registered at MINIT with the package files'
//! exact signatures, so the extension's classes implement them and the package files (autoloaded only without arrow)
//! stay unused.
@@ -14,14 +14,18 @@ const SOURCE_STREAM: DataType = DataType::object("Flow\\Filesystem\\SourceStream
const DESTINATION_STREAM: DataType = DataType::object("Flow\\Filesystem\\DestinationStream");
const SCHEMA: DataType = DataType::object("Flow\\Parquet\\ParquetFile\\Schema");
const METADATA: DataType = DataType::object("Flow\\Parquet\\ParquetFile\\Metadata");
+const READ_PLAN: DataType = DataType::object("Flow\\Parquet\\Reader\\ReadPlan");
+const ROW_GROUP_CHUNKS: DataType = DataType::object("Flow\\Parquet\\ParquetFile\\RowGroupChunks");
const COMPRESSIONS: DataType = DataType::object("Flow\\Parquet\\ParquetFile\\Compressions");
const OPTIONS: DataType = DataType::object("Flow\\Parquet\\Options");
+const FOOTER: DataType = DataType::object("Flow\\Parquet\\ParquetFile\\Footer");
const PARQUET_FILE_READER: DataType = DataType::object("Flow\\Parquet\\ParquetFileReader");
const PARQUET_FILE_WRITER: DataType = DataType::object("Flow\\Parquet\\ParquetFileWriter");
const ITERATOR: DataType = DataType::object("Iterator");
// ClassEntry::try_find reads EG(class_table), which MINIT has not set, so each CE is kept from its registration
static PARQUET_ENGINE_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut());
+static FOOTER_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut());
static PARQUET_FILE_READER_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut());
static PARQUET_FILE_WRITER_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut());
@@ -30,6 +34,11 @@ fn store_parquet_engine(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) {
PARQUET_ENGINE_CE.store(ce, Ordering::Release);
}
+fn store_footer(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) {
+ keep(arg_info);
+ FOOTER_CE.store(ce, Ordering::Release);
+}
+
fn store_parquet_file_reader(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) {
keep(arg_info);
PARQUET_FILE_READER_CE.store(ce, Ordering::Release);
@@ -53,6 +62,10 @@ pub fn parquet_engine_ce() -> &'static ClassEntry {
load(&PARQUET_ENGINE_CE, "Flow\\Parquet\\ParquetEngine")
}
+pub fn footer_ce() -> &'static ClassEntry {
+ load(&FOOTER_CE, "Flow\\Parquet\\ParquetFile\\Footer")
+}
+
pub fn parquet_file_reader_ce() -> &'static ClassEntry {
load(&PARQUET_FILE_READER_CE, "Flow\\Parquet\\ParquetFileReader")
}
@@ -82,25 +95,36 @@ fn interface(
}
pub fn register() -> Result<()> {
+ interface(
+ "Flow\\Parquet\\ParquetFile\\Footer",
+ store_footer,
+ vec![
+ method("metadata", vec![], METADATA),
+ method(
+ "rowGroupChunks",
+ vec![Arg::new("columns", DataType::Array)],
+ ROW_GROUP_CHUNKS,
+ ),
+ method("rowsNumber", vec![], DataType::Long),
+ method("schema", vec![], SCHEMA),
+ method("totalByteSize", vec![], DataType::Long),
+ ],
+ )?;
interface(
"Flow\\Parquet\\ParquetFileReader",
store_parquet_file_reader,
vec![
method("close", vec![], DataType::Void),
- method("metadata", vec![], METADATA),
+ method("footer", vec![], FOOTER),
method(
"readColumns",
vec![
Arg::new("columns", DataType::Array),
Arg::new("batchSize", DataType::Long),
- Arg::new("limit", DataType::Long).allow_null(),
- Arg::new("offset", DataType::Long).allow_null(),
+ Arg::new("plan", READ_PLAN),
],
ITERATOR,
),
- method("rowsNumber", vec![], DataType::Long),
- method("schema", vec![], SCHEMA),
- method("totalByteSize", vec![], DataType::Long),
],
)?;
interface(
diff --git a/src/extension/arrow-ext/src/lib.rs b/src/extension/arrow-ext/src/lib.rs
index 8804b3bff3..5d194b8aa6 100644
--- a/src/extension/arrow-ext/src/lib.rs
+++ b/src/extension/arrow-ext/src/lib.rs
@@ -29,8 +29,8 @@ pub extern "C" fn php_module_info(_module: *mut ModuleEntry) {
}
/// The contract version flow-php/parquet expects (`Flow\Parquet\Engine\ArrowExtension::ABI`); both change together
-/// whenever a registered class or interface changes.
-const FLOW_ARROW_ABI: i64 = 1;
+/// whenever a registered class or interface, or the schema, options or values crossing them, change.
+const FLOW_ARROW_ABI: i64 = 2;
/// # Safety
///
@@ -55,6 +55,7 @@ pub fn get_module(module: ModuleBuilder) -> ModuleBuilder {
.request_startup_function(php::request_startup)
.request_shutdown_function(php::request_shutdown)
.class::()
+ .class::()
.class::()
.class::()
.class::()
diff --git a/src/extension/arrow-ext/src/parquet/batch.rs b/src/extension/arrow-ext/src/parquet/batch.rs
index 60d5fde10f..af91b7ccdc 100644
--- a/src/extension/arrow-ext/src/parquet/batch.rs
+++ b/src/extension/arrow-ext/src/parquet/batch.rs
@@ -15,8 +15,8 @@ use ext_php_rs::types::{ZendObject, Zval};
use ext_php_rs::zend::ClassEntry;
use crate::parquet::error::Error;
-use crate::parquet::library::{batch_size, rows, RustParquetFileReader, INVALID_ARGUMENT};
-use crate::parquet::read::{ParquetReader, ReadPlan};
+use crate::parquet::library::{batch_size, read_plan, RustParquetFileReader, INVALID_ARGUMENT};
+use crate::parquet::read::ParquetReader;
use crate::parquet::source::PhpStream;
use crate::php::{call_handle_transparent, ce_method_ref};
use crate::render::{exception, parquet_exception, Side};
@@ -83,22 +83,14 @@ impl RustBatchReader {
file: &RustParquetFileReader,
columns: Vec,
batch_size: i64,
- offset: Option,
- limit: Option,
+ plan: &Zval,
) -> PhpResult {
let batch_size = self::batch_size(batch_size)?;
- let (offset, limit) = (rows(offset, "offset")?.unwrap_or(0), rows(limit, "limit")?);
- let (stream, size, meta) = file.read()?;
+ let (stream, meta) = file.read()?;
let stream = Arc::clone(stream);
- let reader = ParquetReader::open(
- Arc::clone(&stream),
- size,
- meta,
- &columns,
- ReadPlan::new(meta, offset, limit),
- batch_size,
- )
- .map_err(|error| parquet_exception(error, &stream, Side::Read))?;
+ let plan = read_plan(meta, plan, &stream)?;
+ let reader = ParquetReader::open(Arc::clone(&stream), meta, &columns, plan, batch_size)
+ .map_err(|error| parquet_exception(error, &stream, Side::Read))?;
let fields = columns
.iter()
.zip(reader.fields())
@@ -188,26 +180,13 @@ pub fn import(batch: &Zval) -> PhpResult {
)));
}
- let schema = unsafe { &*(address(object, ce, "arrowSchemaAddress")? as *const FFI_ArrowSchema) };
- let source = unsafe { &mut *(address(object, ce, "arrowArrayAddress")? as *mut FFI_ArrowArray) };
-
- if schema.format.is_null() || unsafe { CStr::from_ptr(schema.format) }.to_bytes() != b"+s" {
- return Err(refused("Arrow C Data batch must be a struct array".to_string()));
- }
-
- if source.release.is_none() {
- return Err(refused("Arrow C Data batch is already imported".to_string()));
- }
+ let schema = address(object, ce, "arrowSchemaAddress")? as *const FFI_ArrowSchema;
+ let source = address(object, ce, "arrowArrayAddress")? as *mut FFI_ArrowArray;
- if source.n_children != schema.n_children || source.length < 0 {
- return Err(refused(format!(
- "Arrow C Data batch of {} children does not match its schema of {}",
- source.n_children, schema.n_children
- )));
- }
+ unsafe { check(schema, source) }.map_err(refused)?;
- let moved = unsafe { std::ptr::read(source) };
- source.release = None;
+ let schema = unsafe { &*schema };
+ let moved = unsafe { FFI_ArrowArray::from_raw(source) };
let data = unsafe { from_ffi(moved, schema) }
.and_then(|data| data.validate().map(|()| data))
@@ -216,6 +195,59 @@ pub fn import(batch: &Zval) -> PhpResult {
Ok(StructArray::from(data))
}
+/// The leading fields of `struct ArrowSchema` and `struct ArrowArray` as the Arrow C Data Interface specification
+/// declares them (https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions): read before the
+/// arrow-rs accessors, which panic on a null format and cast a negative count to `usize`.
+#[repr(C)]
+struct SchemaHead {
+ format: *const c_char,
+ name: *const c_char,
+ metadata: *const c_char,
+ flags: i64,
+ n_children: i64,
+}
+
+#[repr(C)]
+struct ArrayHead {
+ length: i64,
+ null_count: i64,
+ offset: i64,
+ n_buffers: i64,
+ n_children: i64,
+}
+
+/// # Safety
+///
+/// `schema` and `array` point to live C Data Interface structs.
+unsafe fn check(schema: *const FFI_ArrowSchema, array: *const FFI_ArrowArray) -> Result<(), String> {
+ let schema_head = unsafe { &*(schema as *const SchemaHead) };
+ let array_head = unsafe { &*(array as *const ArrayHead) };
+
+ if schema_head.format.is_null() || unsafe { CStr::from_ptr(schema_head.format) }.to_bytes() != b"+s" {
+ return Err("Arrow C Data batch must be a struct array".to_string());
+ }
+
+ if unsafe { &*array }.is_released() {
+ return Err("Arrow C Data batch is already imported".to_string());
+ }
+
+ if array_head.length < 0 {
+ return Err(format!(
+ "Arrow C Data batch length must not be negative, {} given",
+ array_head.length
+ ));
+ }
+
+ if array_head.n_children < 0 || schema_head.n_children < 0 || array_head.n_children != schema_head.n_children {
+ return Err(format!(
+ "Arrow C Data batch of {} children does not match its schema of {}",
+ array_head.n_children, schema_head.n_children
+ ));
+ }
+
+ Ok(())
+}
+
#[cfg(test)]
mod tests {
use std::sync::Arc;
@@ -271,4 +303,53 @@ mod tests {
assert_eq!(allocated_bytes(), baseline);
}
+
+ fn refusal(mutate: impl FnOnce(*mut super::SchemaHead, *mut super::ArrayHead)) -> Result<(), String> {
+ let (mut array, mut schema) = to_ffi(&batch().to_data()).unwrap();
+ let schema_ptr = &mut schema as *mut arrow_array::ffi::FFI_ArrowSchema;
+ let array_ptr = &mut array as *mut arrow_array::ffi::FFI_ArrowArray;
+ let schema_head = schema_ptr as *mut super::SchemaHead;
+ let array_head = array_ptr as *mut super::ArrayHead;
+ let (format, length, children) =
+ unsafe { ((*schema_head).format, (*array_head).length, (*array_head).n_children) };
+
+ mutate(schema_head, array_head);
+ let result = unsafe { super::check(schema_ptr, array_ptr) };
+
+ unsafe {
+ (*schema_head).format = format;
+ (*array_head).length = length;
+ (*array_head).n_children = children;
+ }
+
+ result
+ }
+
+ #[test]
+ fn a_null_format_is_refused_before_the_accessors() {
+ assert_eq!(
+ refusal(|schema, _| unsafe { (*schema).format = std::ptr::null() }),
+ Err("Arrow C Data batch must be a struct array".to_string())
+ );
+ }
+
+ #[test]
+ fn a_negative_length_is_refused() {
+ assert_eq!(
+ refusal(|_, array| unsafe { (*array).length = -1 }),
+ Err("Arrow C Data batch length must not be negative, -1 given".to_string())
+ );
+ }
+
+ #[test]
+ fn a_negative_or_mismatched_child_count_is_refused() {
+ assert_eq!(
+ refusal(|_, array| unsafe { (*array).n_children = -1 }),
+ Err("Arrow C Data batch of -1 children does not match its schema of 2".to_string())
+ );
+ assert_eq!(
+ refusal(|_, array| unsafe { (*array).n_children = 1 }),
+ Err("Arrow C Data batch of 1 children does not match its schema of 2".to_string())
+ );
+ }
}
diff --git a/src/extension/arrow-ext/src/parquet/canonical.rs b/src/extension/arrow-ext/src/parquet/canonical.rs
index 80b5dfaca2..fc19dbae2a 100644
--- a/src/extension/arrow-ext/src/parquet/canonical.rs
+++ b/src/extension/arrow-ext/src/parquet/canonical.rs
@@ -2,8 +2,10 @@
//! storage (the layout flow_php's columns store), keyed on the arrow `DataType` alone. Temporal and decimal
//! casts are this module's own - `arrow_cast` rounds toward zero and nulls out overflows.
+use std::fmt::Write;
use std::sync::Arc;
+use arrow_array::builder::LargeBinaryBuilder;
use arrow_array::cast::AsArray;
use arrow_array::types::{
Date32Type, Date64Type, Decimal128Type, Decimal256Type, DecimalType, DurationMicrosecondType, Float32Type,
@@ -13,11 +15,12 @@ use arrow_array::types::{
};
use arrow_array::{
make_array, Array, ArrayRef, ArrowPrimitiveType, BinaryArray, FixedSizeBinaryArray, Float64Array, GenericByteArray,
- LargeBinaryArray, ListArray, MapArray, PrimitiveArray, StructArray,
+ LargeBinaryArray, ListArray, MapArray, PrimitiveArray, StructArray, UInt64Array,
};
use arrow_buffer::{OffsetBuffer, ScalarBuffer};
use arrow_schema::{DataType, Field, FieldRef, Fields, TimeUnit};
+use crate::parquet::cells::{unsigned, UNSIGNED_EXPECTED};
use crate::parquet::error::{parent_row, Error};
const UTC: &str = "UTC";
@@ -62,19 +65,16 @@ fn canonical_data_type(data_type: &DataType) -> Result {
| DataType::Float64
| DataType::Date32
| DataType::FixedSizeBinary(16) => data_type.clone(),
- DataType::Int8
- | DataType::Int16
- | DataType::Int32
- | DataType::UInt8
- | DataType::UInt16
- | DataType::UInt32
- | DataType::UInt64 => DataType::Int64,
+ DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::UInt8 | DataType::UInt16 | DataType::UInt32 => {
+ DataType::Int64
+ }
DataType::Float32 | DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => DataType::Float64,
DataType::Utf8
| DataType::Binary
| DataType::LargeUtf8
| DataType::LargeBinary
- | DataType::FixedSizeBinary(_) => DataType::LargeBinary,
+ | DataType::FixedSizeBinary(_)
+ | DataType::UInt64 => DataType::LargeBinary,
DataType::Timestamp(_, _) => timestamp(),
DataType::Time64(TimeUnit::Microsecond | TimeUnit::Nanosecond)
| DataType::Time32(TimeUnit::Millisecond | TimeUnit::Second) => DataType::Duration(TimeUnit::Microsecond),
@@ -146,6 +146,47 @@ fn checked(
Ok(PrimitiveArray::new(ScalarBuffer::from(values), input.nulls().cloned()))
}
+/// UINT_64 from its decimal text (a `numeric_string` column): a canonical digit string up to u64::MAX, refused at its
+/// row otherwise.
+fn unsigned_from_text(array: &ArrayRef) -> Result {
+ let input = array.as_binary::();
+ let mut values = Vec::with_capacity(input.len());
+
+ for row in 0..input.len() {
+ values.push(if input.is_null(row) {
+ 0
+ } else {
+ unsigned(input.value(row)).ok_or_else(|| Error::Value {
+ column: String::new(),
+ row,
+ expected: UNSIGNED_EXPECTED,
+ got: "string".to_string(),
+ })?
+ });
+ }
+
+ Ok(UInt64Array::new(ScalarBuffer::from(values), input.nulls().cloned()))
+}
+
+/// UINT_64 as its decimal text: one PHP type for the column, lossless above `PHP_INT_MAX`.
+fn decimal_text(array: &ArrayRef) -> ArrayRef {
+ let input = array.as_primitive::();
+ let mut builder = LargeBinaryBuilder::with_capacity(input.len(), input.len() * 20);
+ let mut text = String::with_capacity(20);
+
+ for (row, value) in input.values().iter().enumerate() {
+ if input.is_null(row) {
+ builder.append_null();
+ } else {
+ text.clear();
+ write!(text, "{value}").expect("writing to a String does not fail");
+ builder.append_value(&text);
+ }
+ }
+
+ Arc::new(builder.finish())
+}
+
fn retagged(array: &ArrayRef, to: DataType) -> ArrayRef {
make_array(unsafe { array.to_data().into_builder().data_type(to).build_unchecked() })
}
@@ -252,9 +293,7 @@ fn cast(array: &ArrayRef) -> Result {
DataType::UInt8 => widened::(array),
DataType::UInt16 => widened::(array),
DataType::UInt32 => widened::(array),
- DataType::UInt64 => Arc::new(checked::(array, |value| {
- i64::try_from(value).ok()
- })?),
+ DataType::UInt64 => decimal_text(array),
DataType::Float32 => Arc::new(array.as_primitive::().unary::<_, Float64Type>(f64::from)),
DataType::Utf8 | DataType::Binary => widened_binary(array),
DataType::LargeUtf8 | DataType::LargeBinary => retagged(array, DataType::LargeBinary),
@@ -369,6 +408,7 @@ pub fn to_parquet(array: &ArrayRef, target: &DataType) -> Result Arc::new(narrowed_int::(array)?),
(DataType::Int64, DataType::UInt32) => Arc::new(narrowed_int::(array)?),
(DataType::Int64, DataType::UInt64) => Arc::new(narrowed_int::(array)?),
+ (DataType::LargeBinary, DataType::UInt64) => Arc::new(unsigned_from_text(array)?),
(DataType::Float64, DataType::Float32) => Arc::new(
array
.as_primitive::()
@@ -701,28 +741,39 @@ mod tests {
int64(&read(Arc::new(UInt32Array::from(vec![Some(u32::MAX), None])))),
vec![Some(i64::from(u32::MAX)), None]
);
- assert_eq!(
- int64(&read(Arc::new(UInt64Array::from(vec![Some(i64::MAX as u64), None])))),
- vec![Some(i64::MAX), None]
- );
}
#[test]
- fn uint64_above_int64_is_refused_at_its_row() {
- let error = refusal(Arc::new(UInt64Array::from(vec![
- Some(1),
+ fn uint64_reads_as_its_decimal_text() {
+ let canonical = read(Arc::new(UInt64Array::from(vec![
+ Some(0),
None,
- Some(i64::MAX as u64 + 1),
+ Some(1 << 63),
+ Some(u64::MAX),
])));
- assert!(matches!(error, Error::Overflow { column, row: 2 } if column == "column"));
+ assert_eq!(
+ canonical.as_binary::().iter().collect::>(),
+ vec![
+ Some(&b"0"[..]),
+ None,
+ Some(&b"9223372036854775808"[..]),
+ Some(&b"18446744073709551615"[..]),
+ ]
+ );
}
#[test]
fn a_null_slot_never_overflows() {
- let values = UInt64Array::new(vec![1, u64::MAX].into(), Some(vec![true, false].into()));
+ let values = TimestampMillisecondArray::new(vec![1, i64::MAX].into(), Some(vec![true, false].into()));
- assert_eq!(int64(&read(Arc::new(values))), vec![Some(1), None]);
+ assert_eq!(
+ read(Arc::new(values))
+ .as_primitive::()
+ .iter()
+ .collect::>(),
+ vec![Some(1_000), None]
+ );
}
#[test]
@@ -952,9 +1003,13 @@ mod tests {
#[test]
fn a_nested_overflow_is_refused_at_its_parent_row() {
- let values = Arc::new(UInt64Array::from(vec![1, 2, u64::MAX]));
+ let values = Arc::new(TimestampMillisecondArray::from(vec![1, 2, i64::MAX]));
let list = ListArray::new(
- Arc::new(Field::new("element", DataType::UInt64, false)),
+ Arc::new(Field::new(
+ "element",
+ DataType::Timestamp(TimeUnit::Millisecond, None),
+ false,
+ )),
arrow_buffer::OffsetBuffer::new(vec![0, 2, 3].into()),
values,
None,
@@ -1025,15 +1080,16 @@ mod tests {
vec![Some(-128), None, Some(127)]
);
- let unsigned: ArrayRef = Arc::new(Int64Array::from(vec![Some(4_000_000_000), Some(i64::MAX)]));
-
assert_eq!(
- to_parquet(&unsigned, &DataType::UInt64)
- .unwrap()
- .as_primitive::()
- .values()
- .to_vec(),
- vec![4_000_000_000, i64::MAX as u64]
+ to_parquet(
+ &(Arc::new(Int64Array::from(vec![Some(4_000_000_000), None])) as ArrayRef),
+ &DataType::UInt64
+ )
+ .unwrap()
+ .as_primitive::()
+ .iter()
+ .collect::>(),
+ vec![Some(4_000_000_000), None]
);
for target in [
@@ -1381,4 +1437,35 @@ mod tests {
Err(Error::Unsupported { parquet, .. }) if parquet == "Float32"
));
}
+
+ #[test]
+ fn to_parquet_reads_uint64_from_digit_text() {
+ let text: ArrayRef = Arc::new(LargeBinaryArray::from(vec![
+ Some(&b"18446744073709551615"[..]),
+ None,
+ Some(&b"0"[..]),
+ ]));
+
+ assert_eq!(
+ to_parquet(&text, &DataType::UInt64)
+ .unwrap()
+ .as_primitive::()
+ .iter()
+ .collect::>(),
+ vec![Some(u64::MAX), None, Some(0)]
+ );
+ }
+
+ #[test]
+ fn to_parquet_refuses_uint64_text_that_is_not_a_canonical_digit_string() {
+ for text in [&b"18446744073709551616"[..], b"007", b"-1", b"abc", b""] {
+ let array: ArrayRef = Arc::new(LargeBinaryArray::from(vec![Some(&b"1"[..]), Some(text)]));
+
+ assert!(
+ matches!(to_parquet(&array, &DataType::UInt64), Err(Error::Value { row: 1, .. })),
+ "{:?}",
+ text
+ );
+ }
+ }
}
diff --git a/src/extension/arrow-ext/src/parquet/cells.rs b/src/extension/arrow-ext/src/parquet/cells.rs
index f5ce878e63..0c8ecd7ba0 100644
--- a/src/extension/arrow-ext/src/parquet/cells.rs
+++ b/src/extension/arrow-ext/src/parquet/cells.rs
@@ -7,6 +7,7 @@ use std::sync::Arc;
use arrow_array::builder::{
ArrayBuilder, BooleanBuilder, Date32Builder, DurationMicrosecondBuilder, FixedSizeBinaryBuilder, Float64Builder,
Int64Builder, LargeBinaryBuilder, ListBuilder, MapBuilder, MapFieldNames, TimestampMicrosecondBuilder,
+ UInt64Builder,
};
use arrow_array::cast::AsArray;
use arrow_array::types::{Date32Type, DurationMicrosecondType, Float64Type, Int64Type, TimestampMicrosecondType};
@@ -381,6 +382,7 @@ pub enum Scalar<'a> {
Null,
Bool(bool),
Int(i64),
+ UInt(u64),
Float(f64),
Bytes(&'a [u8]),
Uuid([u8; 16]),
@@ -396,6 +398,7 @@ impl Scalar<'_> {
Scalar::Null => Scalar::Null,
Scalar::Bool(value) => Scalar::Bool(value),
Scalar::Int(value) => Scalar::Int(value),
+ Scalar::UInt(value) => Scalar::UInt(value),
Scalar::Float(value) => Scalar::Float(value),
Scalar::Bytes(_) => Scalar::Bytes(bytes),
Scalar::Uuid(value) => Scalar::Uuid(value),
@@ -483,6 +486,7 @@ impl<'a> Checked<'a> for Validated<'a> {
pub enum ColumnBuilder {
Bool(BooleanBuilder),
Int(Int64Builder),
+ UInt(UInt64Builder),
Float(Float64Builder),
Bytes(LargeBinaryBuilder),
Uuid(FixedSizeBinaryBuilder),
@@ -532,6 +536,7 @@ impl ColumnBuilder {
match canonical {
DataType::Boolean => ColumnBuilder::Bool(BooleanBuilder::with_capacity(capacity)),
DataType::Int64 => ColumnBuilder::Int(Int64Builder::with_capacity(capacity)),
+ DataType::UInt64 => ColumnBuilder::UInt(UInt64Builder::with_capacity(capacity)),
DataType::Float64 => ColumnBuilder::Float(Float64Builder::with_capacity(capacity)),
DataType::LargeBinary => ColumnBuilder::Bytes(LargeBinaryBuilder::with_capacity(capacity, 1024)),
DataType::FixedSizeBinary(size) => {
@@ -584,6 +589,7 @@ impl ArrayBuilder for ColumnBuilder {
match self {
ColumnBuilder::Bool(builder) => builder.len(),
ColumnBuilder::Int(builder) => builder.len(),
+ ColumnBuilder::UInt(builder) => builder.len(),
ColumnBuilder::Float(builder) => builder.len(),
ColumnBuilder::Bytes(builder) => builder.len(),
ColumnBuilder::Uuid(builder) => builder.len(),
@@ -600,6 +606,7 @@ impl ArrayBuilder for ColumnBuilder {
match self {
ColumnBuilder::Bool(builder) => Arc::new(builder.finish()),
ColumnBuilder::Int(builder) => Arc::new(builder.finish()),
+ ColumnBuilder::UInt(builder) => Arc::new(builder.finish()),
ColumnBuilder::Float(builder) => Arc::new(builder.finish()),
ColumnBuilder::Bytes(builder) => Arc::new(builder.finish()),
ColumnBuilder::Uuid(builder) => Arc::new(builder.finish()),
@@ -616,6 +623,7 @@ impl ArrayBuilder for ColumnBuilder {
match self {
ColumnBuilder::Bool(builder) => Arc::new(builder.finish_cloned()),
ColumnBuilder::Int(builder) => Arc::new(builder.finish_cloned()),
+ ColumnBuilder::UInt(builder) => Arc::new(builder.finish_cloned()),
ColumnBuilder::Float(builder) => Arc::new(builder.finish_cloned()),
ColumnBuilder::Bytes(builder) => Arc::new(builder.finish_cloned()),
ColumnBuilder::Uuid(builder) => Arc::new(builder.finish_cloned()),
@@ -653,6 +661,8 @@ fn offsets_fit(current: usize, added: usize) -> bool {
pub enum WriteCell {
Bool,
Int,
+ /// UINT_64: an int >= 0, or a canonical digit string up to 18446744073709551615.
+ UInt64,
Float,
/// DECIMAL: a float or an int.
Decimal,
@@ -679,8 +689,8 @@ impl WriteCell {
| DataType::Int64
| DataType::UInt8
| DataType::UInt16
- | DataType::UInt32
- | DataType::UInt64 => (WriteCell::Int, DataType::Int64),
+ | DataType::UInt32 => (WriteCell::Int, DataType::Int64),
+ DataType::UInt64 => (WriteCell::UInt64, DataType::UInt64),
DataType::Float32 | DataType::Float64 => (WriteCell::Float, DataType::Float64),
DataType::Decimal128(_, _) => (WriteCell::Decimal, DataType::Float64),
DataType::Utf8 | DataType::Binary => (WriteCell::Bytes, DataType::LargeBinary),
@@ -754,6 +764,13 @@ impl WriteCell {
Ok(match self {
WriteCell::Bool => Scalar::Bool(value.bool().ok_or_else(|| refused("bool", value))?),
WriteCell::Int => Scalar::Int(value.long().ok_or_else(|| refused("int", value))?),
+ WriteCell::UInt64 => Scalar::UInt(
+ match value.long() {
+ Some(int) => u64::try_from(int).ok(),
+ None => value.zend_str().and_then(|text| unsigned(text.as_bytes())),
+ }
+ .ok_or_else(|| refused(UNSIGNED_EXPECTED, value))?,
+ ),
WriteCell::Float => Scalar::Float(value.double().ok_or_else(|| refused("float", value))?),
WriteCell::Decimal => Scalar::Float(match (value.double(), value.long()) {
(Some(float), _) => float,
@@ -830,7 +847,7 @@ impl WriteCell {
}
}
- Ok(())
+ Ok::<(), PhpException>(())
})?;
if let Some(refusal) = failure {
@@ -846,7 +863,7 @@ impl WriteCell {
ht_for_each(items, |name, index, item| {
if failure.is_some() {
- return Ok(());
+ return Ok::<(), PhpException>(());
}
let entry_key = match name {
@@ -986,6 +1003,7 @@ impl WriteCell {
(_, builder, Scalar::Null) => self.append_null(builder),
(WriteCell::Bool, ColumnBuilder::Bool(builder), Scalar::Bool(value)) => builder.append_value(*value),
(WriteCell::Int, ColumnBuilder::Int(builder), Scalar::Int(value)) => builder.append_value(*value),
+ (WriteCell::UInt64, ColumnBuilder::UInt(builder), Scalar::UInt(value)) => builder.append_value(*value),
(WriteCell::Float | WriteCell::Decimal, ColumnBuilder::Float(builder), Scalar::Float(value)) => {
builder.append_value(*value);
}
@@ -1022,6 +1040,7 @@ impl WriteCell {
}
(_, ColumnBuilder::Bool(builder)) => builder.append_null(),
(_, ColumnBuilder::Int(builder)) => builder.append_null(),
+ (_, ColumnBuilder::UInt(builder)) => builder.append_null(),
(_, ColumnBuilder::Float(builder)) => builder.append_null(),
(_, ColumnBuilder::Bytes(builder)) => builder.append_null(),
(_, ColumnBuilder::Uuid(builder)) => builder.append_null(),
@@ -1037,6 +1056,18 @@ impl WriteCell {
}
}
+/// What a UINT_64 cell takes, as its refusals name it.
+pub const UNSIGNED_EXPECTED: &str = "int >= 0 or digit string";
+
+/// A canonical digit string (no sign, no leading zero) within u64, as `UnsignedInt64Converter::toParquetType()` takes it.
+pub fn unsigned(text: &[u8]) -> Option {
+ if text.is_empty() || !text.iter().all(u8::is_ascii_digit) || (text.len() > 1 && text[0] == b'0') {
+ return None;
+ }
+
+ std::str::from_utf8(text).ok()?.parse().ok()
+}
+
/// `UuidConverter::toParquetType()`'s text of an object: `toString()`, else `__toString()`; `None` for neither.
fn uuid_text_of(object: &ZendObject) -> Result