From 68ab97ddb4f40802fccf2c758ef280a5c41d89e8 Mon Sep 17 00:00:00 2001 From: Norbert Orzechowicz Date: Tue, 20 Oct 2026 11:54:12 -0500 Subject: [PATCH] feat(flow-php/etl): explain() renders expressions instead of class names - filters, conditions and columns print as expressions - pushed path filters listed one per line - explain blocks in the docs regenerated --- documentation/components/adapters/parquet.md | 3 +- documentation/components/core/core.md | 55 +++--- documentation/components/core/partitioning.md | 15 ++ .../ETL/Filesystem/ScalarFunctionFilter.php | 7 +- .../etl/src/Flow/ETL/Plan/Explain/Details.php | 38 +++- .../src/Flow/ETL/Plan/Explain/Expression.php | 184 ++++++++++++++++++ .../Flow/ETL/Tests/Unit/DataFrameTest.php | 4 +- .../Filesystem/ScalarFunctionFilterTest.php | 10 + .../Rule/PushFilterIntoSourceTest.php | 2 +- .../Tests/Unit/Plan/Explain/BoxLayoutTest.php | 2 +- .../Tests/Unit/Plan/Explain/DetailsTest.php | 44 ++++- .../Unit/Plan/Explain/ExpressionTest.php | 139 +++++++++++++ .../Unit/Plan/Explain/FlowLayoutTest.php | 2 +- .../Unit/Plan/Explain/PhysicalOutlineTest.php | 2 +- .../Unit/Plan/Explain/TreeLayoutTest.php | 8 +- .../Flow/ETL/Tests/Unit/Sink/BranchedTest.php | 4 +- .../ETL/Tests/Unit/Sink/TransactionalTest.php | 2 +- .../ETL/Tests/Unit/Sink/TransformedTest.php | 2 +- .../Flow/Filesystem/Path/Filter/Filters.php | 9 + .../Tests/Unit/Path/Filter/FiltersTest.php | 21 ++ 20 files changed, 488 insertions(+), 65 deletions(-) create mode 100644 src/core/etl/src/Flow/ETL/Plan/Explain/Expression.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/ExpressionTest.php create mode 100644 src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/Path/Filter/FiltersTest.php diff --git a/documentation/components/adapters/parquet.md b/documentation/components/adapters/parquet.md index 06d5153c8b..0625dda533 100644 --- a/documentation/components/adapters/parquet.md +++ b/documentation/components/adapters/parquet.md @@ -140,5 +140,6 @@ Partition columns list fewer files instead ([partitioning](/documentation/compon Extractor: ParquetExtractor Source: file:///orders/*.parquet Limit: 10 - Filter: All + Filter: id > 1000 + Filter: status = 'paid' ``` diff --git a/documentation/components/core/core.md b/documentation/components/core/core.md index 01d6a5f377..751dc846be 100644 --- a/documentation/components/core/core.md +++ b/documentation/components/core/core.md @@ -64,18 +64,17 @@ echo $dataFrame->explain()->toString(); ```text Outputs -├─ #4 Result +├─ #3 Result │ │ Rows this plan hands out: to the trigger, or to the node reading it -│ └─ #3 Select +│ └─ #2 Select │ │ Keeps only: id, email -│ └─ #2 Filter -│ │ Condition: IsNotNull -│ └─ #1 Read -│ Extractor: CSVExtractor -│ Source: file:///data/orders.csv -└─ #5 Write +│ └─ #1 Read +│ Extractor: CSVExtractor +│ Source: file:///data/orders.csv +│ Filter: email IS NOT NULL +└─ #4 Write │ Loader: JsonLoader - └─ #2 Filter (shared) + └─ #1 Read (shared) ``` ```php @@ -87,13 +86,12 @@ echo data_frame() ``` ```text -#3 Write +#2 Write │ Loader: JsonLoader -└─ #2 Filter - │ Condition: IsNotNull - └─ #1 Read - Extractor: CSVExtractor - Source: file:///data/orders.csv +└─ #1 Read + Extractor: CSVExtractor + Source: file:///data/orders.csv + Filter: email IS NOT NULL ``` `run()` takes no rows, so its plan carries only the sinks - the `Write` is the root and there is no `Result`. A @@ -101,7 +99,7 @@ echo data_frame() it to pull the rows, and `run(analyze: ...)` still counts them. `Outputs` lists everything the frame produces: `Result` is what a rows-returning trigger reads, `Write` is the sink. -A node several consumers read is printed once, and every other consumer points back at it by number - `#2 Filter (shared)` is that same filter, not a second one. A node keeps its number in every format. +A node several consumers read is printed once, and every other consumer points back at it by number - `#1 Read (shared)` is that same read, not a second one. A node keeps its number in every format. A frame joined with `join()` / `crossJoin()` is part of the same tree: the join reads it the way it reads any other input, numbered with the rest of the plan, and it runs with this frame's configuration. A frame read with `from_data_frame()` is a `Read` of `DataFrameExtractor` and runs with its own. @@ -126,14 +124,13 @@ echo $dataFrame->explain()->toString(format: Format::flow); #1 Read │ Extractor: CSVExtractor │ Source: file:///data/orders.csv -└─ #2 Filter - │ Condition: IsNotNull - ├─ #3 Select - │ │ Keeps only: id, email - │ └─ #4 Result - │ Rows this plan hands out: to the trigger, or to the node reading it - └─ #5 Write - Loader: JsonLoader +│ Filter: email IS NOT NULL +├─ #2 Select +│ │ Keeps only: id, email +│ └─ #3 Result +│ Rows this plan hands out: to the trigger, or to the node reading it +└─ #4 Write + Loader: JsonLoader ``` ```php @@ -145,28 +142,24 @@ echo $dataFrame->explain()->toString(format: Format::boxes); │ Outputs ├──────────────┐ └─────────────┬─────────────┘ │ ┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -│ #4 Result ││ #5 Write │ +│ #3 Result ││ #4 Write │ │ ──────────────────── ││ ──────────────────── │ │ Rows this plan hands out: ││ Loader: JsonLoader │ │ to the trigger, or to the ││ │ │ node reading it ││ │ └─────────────┬─────────────┘└─────────────┬─────────────┘ ┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -│ #3 Select ││ #2 Filter │ +│ #2 Select ││ #1 Read │ │ ──────────────────── ││ (shared) │ │ Keeps only: id, email ││ │ └─────────────┬─────────────┘└───────────────────────────┘ ┌─────────────┴─────────────┐ -│ #2 Filter │ -│ ──────────────────── │ -│ Condition: IsNotNull │ -└─────────────┬─────────────┘ -┌─────────────┴─────────────┐ │ #1 Read │ │ ──────────────────── │ │ Extractor: CSVExtractor │ │ Source: │ │ file:///data/orders.csv │ +│ Filter: email IS NOT NULL │ └───────────────────────────┘ ``` diff --git a/documentation/components/core/partitioning.md b/documentation/components/core/partitioning.md index feb42eb533..28420df672 100644 --- a/documentation/components/core/partitioning.md +++ b/documentation/components/core/partitioning.md @@ -290,6 +290,21 @@ data_frame() ->run(); ``` +`explain()` lists what each file's path must pass under `Files:`, what each row must pass under `Filter:`: + +```text +#3 Write +│ Loader: StreamLoader +└─ #2 Limit + │ Limit: 10 + └─ #1 Read + Extractor: CSVExtractor + Source: file:///output/date=*/department=*/*.csv + Limit: 10 + Files: date >= '2024-01-01' + Filter: date >= '2024-01-01' +``` + ### Path Partitions `from_path_partitions()` reads partition metadata without opening the files. diff --git a/src/core/etl/src/Flow/ETL/Filesystem/ScalarFunctionFilter.php b/src/core/etl/src/Flow/ETL/Filesystem/ScalarFunctionFilter.php index 8f0f12f621..995f745366 100644 --- a/src/core/etl/src/Flow/ETL/Filesystem/ScalarFunctionFilter.php +++ b/src/core/etl/src/Flow/ETL/Filesystem/ScalarFunctionFilter.php @@ -23,7 +23,7 @@ private ScalarFunction $resolved; public function __construct( - ScalarFunction $function, + private ScalarFunction $function, private Schema $partitions, private FlowContext $context, ) { @@ -38,6 +38,11 @@ public function __construct( } } + public function function(): ScalarFunction + { + return $this->function; + } + public function accept(FileStatus $status): bool { $values = []; diff --git a/src/core/etl/src/Flow/ETL/Plan/Explain/Details.php b/src/core/etl/src/Flow/ETL/Plan/Explain/Details.php index a28edc17f3..6bceecb4d3 100644 --- a/src/core/etl/src/Flow/ETL/Plan/Explain/Details.php +++ b/src/core/etl/src/Flow/ETL/Plan/Explain/Details.php @@ -6,6 +6,7 @@ use Flow\ETL\Extractor\File\FileExtractor; use Flow\ETL\Extractor\File\FileScan; +use Flow\ETL\Filesystem\ScalarFunctionFilter; use Flow\ETL\Join\Comparison\Any; use Flow\ETL\Plan\Materialization; use Flow\ETL\Plan\Node; @@ -32,10 +33,13 @@ use Flow\ETL\Plan\Node\WithColumn; use Flow\ETL\Plan\Node\Write; use Flow\ETL\Row\Reference; +use Flow\Filesystem\Path\Filter as PathFilter; +use Flow\Filesystem\Path\Filter\Filters; use Flow\Filesystem\Path\Filter\OnlyFiles; use ReflectionClass; use function array_map; +use function array_merge; use function implode; use function sprintf; use function str_ends_with; @@ -45,6 +49,7 @@ { public function __construct( private Condition $condition = new Condition(), + private Expression $expression = new Expression(), ) {} public function name(object $object): string @@ -72,9 +77,11 @@ public function labelled(Node $node): array $node instanceof Join => $this->join($node), $node instanceof CrossJoin => $this->crossJoin($node), $node instanceof Result => ['Rows this plan hands out: to the trigger, or to the node reading it'], - $node instanceof Filter => ['Condition: ' . $this->name($node->function)], - $node instanceof Until => ['Until: ' . $this->name($node->function)], - $node instanceof WithColumn => [sprintf('Column: %s = %s', $node->name(), $this->name($node->function))], + $node instanceof Filter => ['Condition: ' . $this->expression->of($node->function)], + $node instanceof Until => ['Until: ' . $this->expression->of($node->function)], + $node instanceof WithColumn => [ + sprintf('Column: %s = %s', $node->name(), $this->expression->of($node->function)), + ], $node instanceof ExpandColumn => $this->expandColumn($node), $node instanceof Write => ['Loader: ' . $this->name($node->loader)], $node instanceof Limit => ['Limit: ' . $node->limit], @@ -168,7 +175,7 @@ public function columns(array $entries): string */ public function expandColumn(ExpandColumn $node): array { - $lines = [sprintf('Column: %s = %s', $node->name(), $this->name($node->function))]; + $lines = [sprintf('Column: %s = %s', $node->name(), $this->expression->of($node->function))]; return match (true) { $node->carries->isAll() => $lines, @@ -223,18 +230,33 @@ public function scan(FileScan $scan): array $lines[] = 'Limit: ' . $scan->limit; } - // every file source reads only files unless a filter was pushed into it - if (!$scan->pathFilter instanceof OnlyFiles) { - $lines[] = 'Files: ' . $this->name($scan->pathFilter); + foreach ($this->files($scan->pathFilter) as $filter) { + $lines[] = 'Files: ' . $filter; } foreach ($scan->conjuncts as $conjunct) { - $lines[] = 'Filter: ' . $this->name($conjunct); + $lines[] = 'Filter: ' . $this->expression->of($conjunct); } return $lines; } + /** + * Each filter a file must pass, one per line. + * + * @return list + */ + public function files(PathFilter $filter): array + { + return match (true) { + // every file source reads only files, saying so on every read would only add noise + $filter instanceof OnlyFiles => [], + $filter instanceof Filters => array_merge(...array_map($this->files(...), $filter->filters())), + $filter instanceof ScalarFunctionFilter => [$this->expression->of($filter->function())], + default => [$this->name($filter)], + }; + } + /** * The declarations below in plain words, for the ones that change how the rows flow. * diff --git a/src/core/etl/src/Flow/ETL/Plan/Explain/Expression.php b/src/core/etl/src/Flow/ETL/Plan/Explain/Expression.php new file mode 100644 index 0000000000..9ce09adcfb --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Plan/Explain/Expression.php @@ -0,0 +1,184 @@ + $function->name(), + $function instanceof Literal => $this->value($function->value()), + default => $this->operator($function) ?? $this->call($function), + }; + } + + /** + * An operator inside another operator is parenthesized; a reference, a value or a call reads unambiguously as is. + */ + public function operand(FunctionTree $function): string + { + $operator = $this->operator($function); + + return $operator === null ? $this->of($function) : '(' . $operator . ')'; + } + + /** + * Null for a function the explain has no wording for. + */ + public function operator(FunctionTree $function): ?string + { + $children = $function->children(); + + return match (true) { + $function instanceof All => $this->composite($children, ' AND '), + $function instanceof Any => $this->composite($children, ' OR '), + $function instanceof Not => 'NOT ' . $this->operand($children[0]), + $function instanceof Equals => $this->pair($children, '='), + $function instanceof NotEquals => $this->pair($children, '!='), + $function instanceof Same => $this->pair($children, '==='), + $function instanceof NotSame => $this->pair($children, '!=='), + $function instanceof GreaterThan => $this->pair($children, '>'), + $function instanceof GreaterThanEqual => $this->pair($children, '>='), + $function instanceof LessThan => $this->pair($children, '<'), + $function instanceof LessThanEqual => $this->pair($children, '<='), + $function instanceof IsIn => $this->operand($children[1]) . ' IN ' . $this->operand($children[0]), + $function instanceof StartsWith => $this->pair($children, 'STARTS WITH'), + $function instanceof IsNull => $this->operand($children[0]) . ' IS NULL', + $function instanceof IsNotNull => $this->operand($children[0]) . ' IS NOT NULL', + $function instanceof Between => $this->between($children), + default => null, + }; + } + + /** + * Comparisons bind tighter than AND/OR, so only a nested AND/OR is parenthesized. + * + * @param list $children + */ + public function composite(array $children, string $operator): string + { + return implode($operator, array_map(fn(FunctionTree $each): string => $each instanceof All + || $each instanceof Any + ? '(' . $this->of($each) . ')' + : $this->of($each), $children)); + } + + /** + * @param list $children + */ + public function pair(array $children, string $operator): string + { + return $this->operand($children[0]) . ' ' . $operator . ' ' . $this->operand($children[1]); + } + + /** + * @param list $children + */ + public function between(array $children): ?string + { + $boundary = $children[3]; + $type = type_instance_of(Boundary::class); + + if (!$boundary instanceof Literal || !$type->isValid($boundary->value())) { + return null; + } + + return $this->range( + $type->assert($boundary->value()), + $this->operand($children[0]), + $this->operand($children[1]), + $this->operand($children[2]), + ); + } + + public function range(Boundary $boundary, string $value, string $lower, string $upper): string + { + return match ($boundary) { + Boundary::INCLUSIVE => $value . ' BETWEEN ' . $lower . ' AND ' . $upper, + Boundary::LEFT_INCLUSIVE => $lower . ' <= ' . $value . ' < ' . $upper, + Boundary::RIGHT_INCLUSIVE => $lower . ' < ' . $value . ' <= ' . $upper, + Boundary::EXCLUSIVE => $lower . ' < ' . $value . ' < ' . $upper, + }; + } + + public function call(FunctionTree $function): string + { + return ( + (new ReflectionClass($function))->getShortName() + . '(' + . implode(', ', array_map($this->of(...), $function->children())) + . ')' + ); + } + + public function value(mixed $value): string + { + return match (true) { + $value === null => 'null', + is_bool($value) => $value ? 'true' : 'false', + is_string($value) => "'" . addcslashes(str_replace("'", "''", $value), "\0..\37") . "'", + is_float($value) => var_export($value, true), + is_array($value) => $this->list($value), + $value instanceof DateTimeInterface => $value->format(DateTimeInterface::ATOM), + $value instanceof UnitEnum => (new ReflectionClass($value))->getShortName() . '::' . $value->name, + is_object($value) => (new ReflectionClass($value))->getShortName(), + default => (string) $value, + }; + } + + /** + * Cut after a few values, so a long list never makes an unbounded line. + * + * @param array $values + */ + public function list(array $values): string + { + $shown = implode(', ', array_map($this->value(...), array_slice($values, 0, self::LIST_LIMIT))); + $rest = count($values) - self::LIST_LIMIT; + + return $rest > 0 ? '(' . $shown . ', … ' . $rest . ' more)' : '(' . $shown . ')'; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/DataFrameTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/DataFrameTest.php index 9a7205e89f..42729b4564 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/DataFrameTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/DataFrameTest.php @@ -1208,7 +1208,7 @@ public function test_explain_prints_the_logical_plan_without_reading_a_row(): vo └─ #1 Read source · transparent · streaming Extractor: RecordingFileExtractor Source: file:///dev/null - Filter: IsNotNull + Filter: id IS NOT NULL PLAN, $dataFrame->explain()->toString(format: Format::declarations)); static::assertSame([], $extractor->scans); } @@ -1371,7 +1371,7 @@ public function test_two_sinks_share_the_chain_and_do_not_alias(): void └─ #6 Write preserving · opaque · streaming │ Loader: MemoryLoader └─ #5 Filter reducing · transparent · streaming - │ Condition: Equals + │ Condition: id = 1 └─ #1 Read (shared) PLAN, $dataFrame->explain()->toString(format: Format::declarations)); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Filesystem/ScalarFunctionFilterTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Filesystem/ScalarFunctionFilterTest.php index d4b6a6d869..221d56ec00 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Filesystem/ScalarFunctionFilterTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Filesystem/ScalarFunctionFilterTest.php @@ -20,6 +20,16 @@ final class ScalarFunctionFilterTest extends FlowTestCase { + public function test_function_is_the_one_given_not_its_resolved_copy(): void + { + $function = ref('year')->equals(lit(2024)); + + static::assertSame( + $function, + (new ScalarFunctionFilter($function, schema(int_schema('year')), flow_context(config())))->function(), + ); + } + public function test_accept_evaluates_the_resolved_function(): void { static::assertTrue((new ScalarFunctionFilter( diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushFilterIntoSourceTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushFilterIntoSourceTest.php index 7849c68f54..792499ae7e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushFilterIntoSourceTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushFilterIntoSourceTest.php @@ -436,7 +436,7 @@ public function test_the_extractor_instance_is_never_mutated(): void $frame->explain()->toString(Stage::unoptimized), ); static::assertStringEndsWith( - "Extractor: RecordingFileExtractor\n Source: file:///dev/null\n Files: Filters\n Filter: Equals", + "Extractor: RecordingFileExtractor\n Source: file:///dev/null\n Files: year = 2023\n Filter: year = 2023", $frame->explain()->toString(), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/BoxLayoutTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/BoxLayoutTest.php index db4411c12d..e395922c08 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/BoxLayoutTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/BoxLayoutTest.php @@ -85,7 +85,7 @@ public function test_a_later_child_hangs_off_the_right_edge_and_a_shared_node_is ┌─────────────┴─────────────┐┌─────────────┴─────────────┐ │ #2 Filter ││ #2 Filter │ │ ──────────────────── ││ (shared) │ - │ Condition: IsNotNull ││ │ + │ Condition: id IS NOT NULL ││ │ └─────────────┬─────────────┘└───────────────────────────┘ ┌─────────────┴─────────────┐ │ #1 Read │ diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/DetailsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/DetailsTest.php index d1bf62490f..ca0fa441a7 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/DetailsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/DetailsTest.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Unit\Plan\Explain; +use Flow\ETL\Filesystem\ScalarFunctionFilter; use Flow\ETL\Join\Comparison\Equal; use Flow\ETL\Join\Expression; use Flow\ETL\Join\Join; @@ -28,14 +29,19 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\NodeMother; use Flow\ETL\Tests\Mother\PartitionedSourceMother; +use Flow\Filesystem\Path\Filter\Filters; use Flow\Filesystem\Path\Filter\KeepAll; +use Flow\Filesystem\Path\Filter\OnlyFiles; +use function Flow\ETL\DSL\config; +use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\hash_join; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\refs; use function Flow\ETL\DSL\rename_replace; +use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\to_memory; final class DetailsTest extends FlowTestCase @@ -57,9 +63,9 @@ public function test_a_read_lists_a_pushed_limit_file_filter_and_each_conjunct() 'Extractor: RecordingFileExtractor', 'Source: file:///dev/null', 'Limit: 3', - 'Files: Filters', - 'Filter: GreaterThan', - 'Filter: IsNotNull', + 'Files: KeepAll', + 'Filter: year > 1', + 'Filter: month IS NOT NULL', ], (new Details())->lines( (new Read(PartitionedSourceMother::yearMonth())) @@ -70,13 +76,31 @@ public function test_a_read_lists_a_pushed_limit_file_filter_and_each_conjunct() ); } + public function test_files_are_each_pushed_file_filter_but_the_default_one(): void + { + static::assertSame( + ["country IN ('PL', 'DE')", 'KeepAll', 'year > 2020'], + (new Details())->files( + new Filters( + new OnlyFiles(), + new ScalarFunctionFilter(ref('country')->isIn(lit(['PL', 'DE'])), schema(), flow_context(config())), + new Filters( + new KeepAll(), + new ScalarFunctionFilter(ref('year')->greaterThan(lit(2020)), schema(), flow_context(config())), + ), + ), + ), + ); + static::assertSame([], (new Details())->files(new OnlyFiles())); + } + public function test_nodes_with_arguments_label_them(): void { $read = NodeMother::read(); $details = new Details(); - static::assertSame(['Condition: IsNotNull'], $details->lines(new Filter($read, ref('id')->isNotNull()))); - static::assertSame(['Until: Literal'], $details->lines(new Until($read, lit(true)))); + static::assertSame(['Condition: id IS NOT NULL'], $details->lines(new Filter($read, ref('id')->isNotNull()))); + static::assertSame(['Until: true'], $details->lines(new Until($read, lit(true)))); static::assertSame(['Loader: MemoryLoader'], $details->lines(new Write($read, to_memory(new ArrayMemory())))); static::assertSame(['Limit: 5'], $details->lines(NodeMother::limit($read, 5))); static::assertSame(['Skip: 2'], $details->lines(new Offset($read, 2))); @@ -164,17 +188,17 @@ public function test_an_expand_column_shows_the_columns_it_keeps(): void $details = new Details(); $expand = new ExpandColumn($read, 'item', ref('items')->expand()); - static::assertSame(['Column: item = ArrayExpand'], $details->labelled($expand)); + static::assertSame(['Column: item = ArrayExpand(items)'], $details->labelled($expand)); static::assertSame( - ['Column: item = ArrayExpand', 'Keeps: id, item'], + ['Column: item = ArrayExpand(items)', 'Keeps: id, item'], $details->labelled($expand->withCarries(RequiredColumns::only('id', 'item'))), ); static::assertSame( - ['Column: item = ArrayExpand', 'Keeps: no input column'], + ['Column: item = ArrayExpand(items)', 'Keeps: no input column'], $details->labelled($expand->withCarries(RequiredColumns::only())), ); static::assertSame( - ['Column: item = ArrayExpand', 'Keeps: every column but items, body'], + ['Column: item = ArrayExpand(items)', 'Keeps: every column but items, body'], $details->labelled($expand->withCarries(RequiredColumns::allBut('items', 'body'))), ); } @@ -185,7 +209,7 @@ public function test_redefined_columns_follow_the_labelled_details(): void $details = new Details(); static::assertSame( - ['Column: n = Literal', 'Changes columns: n'], + ['Column: n = 1', 'Changes columns: n'], $details->lines(new WithColumn($read, int_schema('n'), lit(1))), ); static::assertSame(['Rename: n → m', 'Changes columns: m, n'], $details->lines(new Rename($read, 'n', 'm'))); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/ExpressionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/ExpressionTest.php new file mode 100644 index 0000000000..080e2271ca --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/ExpressionTest.php @@ -0,0 +1,139 @@ + [ref('a')->equals(lit(1)), 'a = 1']; + yield 'not equals' => [ref('a')->notEquals(lit(1)), 'a != 1']; + yield 'same' => [ref('a')->same(lit(1)), 'a === 1']; + yield 'not same' => [ref('a')->notSame(lit(1)), 'a !== 1']; + yield 'greater than' => [ref('a')->greaterThan(lit(1)), 'a > 1']; + yield 'greater than or equal' => [ref('a')->greaterThanEqual(lit(1)), 'a >= 1']; + yield 'less than' => [ref('a')->lessThan(lit(1)), 'a < 1']; + yield 'less than or equal' => [ref('a')->lessThanEqual(lit(1)), 'a <= 1']; + yield 'is in' => [ref('country')->isIn(lit(['PL', 'DE'])), "country IN ('PL', 'DE')"]; + yield 'is in a column' => [ref('country')->isIn(ref('countries')), 'country IN countries']; + yield 'starts with' => [ref('name')->startsWith('ab'), "name STARTS WITH 'ab'"]; + yield 'is null' => [ref('a')->isNull(), 'a IS NULL']; + yield 'is not null' => [ref('a')->isNotNull(), 'a IS NOT NULL']; + yield 'and' => [all(ref('a')->isNull(), ref('b')->isNull()), 'a IS NULL AND b IS NULL']; + yield 'or' => [any(ref('a')->isNull(), ref('b')->isNull()), 'a IS NULL OR b IS NULL']; + yield 'not a column' => [not(ref('active')), 'NOT active']; + yield 'not a comparison' => [not(ref('a')->equals(lit(1))), 'NOT (a = 1)']; + } + + public static function betweens(): Generator + { + yield 'inclusive' => [Boundary::INCLUSIVE, 'total BETWEEN 1 AND 10']; + yield 'left inclusive' => [Boundary::LEFT_INCLUSIVE, '1 <= total < 10']; + yield 'right inclusive' => [Boundary::RIGHT_INCLUSIVE, '1 < total <= 10']; + yield 'exclusive' => [Boundary::EXCLUSIVE, '1 < total < 10']; + } + + public static function values(): Generator + { + yield 'null' => [null, 'null']; + yield 'true' => [true, 'true']; + yield 'false' => [false, 'false']; + yield 'integer' => [12, '12']; + yield 'float keeps its fraction' => [1.0, '1.0']; + yield 'string is quoted, quotes doubled' => ["it's", "'it''s'"]; + yield 'control characters are escaped' => ["first\nsecond\t", "'first\\nsecond\\t'"]; + yield 'date is ISO 8601' => [ + new DateTimeImmutable('2026-10-20 00:00:00', new DateTimeZone('UTC')), + '2026-10-20T00:00:00+00:00', + ]; + yield 'enum is its case' => [Boundary::INCLUSIVE, 'Boundary::INCLUSIVE']; + yield 'any other object is its name' => [new stdClass(), 'stdClass']; + yield 'list in parentheses' => [[1, 2, 3, 4, 5], '(1, 2, 3, 4, 5)']; + yield 'nested list' => [[[1, 2], [3]], '((1, 2), (3))']; + yield 'long list is cut' => [ + ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l'], + "('a', 'b', 'c', 'd', 'e', … 7 more)", + ]; + } + + #[DataProvider('operators')] + public function test_an_operator_reads_as_its_operands_around_it(ScalarFunction $function, string $expected): void + { + static::assertSame($expected, (new Expression())->of($function)); + } + + #[DataProvider('betweens')] + public function test_between_reads_as_its_boundary(Boundary $boundary, string $expected): void + { + static::assertSame($expected, (new Expression())->of(ref('total')->between(lit(1), lit(10), $boundary))); + } + + #[DataProvider('values')] + public function test_a_literal_reads_as_its_value(mixed $value, string $expected): void + { + static::assertSame($expected, (new Expression())->of(lit($value))); + } + + public function test_a_nested_and_or_is_parenthesized(): void + { + static::assertSame( + 'a = 1 AND (b = 2 OR c = 3)', + (new Expression())->of(all( + ref('a')->equals(lit(1)), + any(ref('b')->equals(lit(2)), ref('c')->equals(lit(3))), + )), + ); + } + + public function test_a_not_or_comparison_inside_an_and_is_not_parenthesized(): void + { + static::assertSame( + 'NOT active AND a = 1', + (new Expression())->of(all(not(ref('active')), ref('a')->equals(lit(1)))), + ); + } + + public function test_an_operator_compared_is_parenthesized(): void + { + static::assertSame('(a = 1) = true', (new Expression())->of(ref('a')->equals(lit(1))->equals(lit(true)))); + } + + public function test_a_function_without_wording_reads_as_its_name_over_its_arguments(): void + { + static::assertSame("ToUpper(name) = 'X'", (new Expression())->of(ref('name')->upper()->equals(lit('X')))); + } + + public function test_between_with_a_literal_that_is_not_a_boundary_reads_as_its_name(): void + { + static::assertSame( + "Between(total, 1, 10, 'x')", + (new Expression())->of(ref('total')->between(lit(1), lit(10), lit('x'))), + ); + } + + public function test_between_with_a_boundary_known_only_at_run_time_reads_as_its_name(): void + { + static::assertSame( + 'Between(total, 1, 10, boundary)', + (new Expression())->of(ref('total')->between(lit(1), lit(10), ref('boundary'))), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/FlowLayoutTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/FlowLayoutTest.php index d1ad96ed9c..1fe6e6a168 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/FlowLayoutTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/FlowLayoutTest.php @@ -47,7 +47,7 @@ public function test_a_node_read_by_several_consumers_branches_and_outputs_is_le #1 Read │ Extractor: ArrayExtractor └─ #2 Filter - │ Condition: IsNotNull + │ Condition: id IS NOT NULL ├─ #3 Result │ Rows this plan hands out: to the trigger, or to the node reading it └─ #4 Write diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/PhysicalOutlineTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/PhysicalOutlineTest.php index 77c2935cad..395a200243 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/PhysicalOutlineTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/PhysicalOutlineTest.php @@ -128,7 +128,7 @@ public function test_a_pushed_filter_is_listed_under_the_file_source_that_was_of static::assertSame( [ 'Extractor: RecordingFileExtractor', - ' Filter: GreaterThan', + ' Filter: id > 1', ], $root->children[0]->lines, ); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/TreeLayoutTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/TreeLayoutTest.php index 8c5991cf56..671bf58b5b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/TreeLayoutTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/TreeLayoutTest.php @@ -61,7 +61,7 @@ public function test_a_shared_node_is_referenced_by_number_and_name(): void ├─ #3 Result │ │ Rows this plan hands out: to the trigger, or to the node reading it │ └─ #2 Filter - │ │ Condition: IsNotNull + │ │ Condition: id IS NOT NULL │ └─ #1 Read │ Extractor: ArrayExtractor └─ #4 Write @@ -122,7 +122,7 @@ public function test_with_declarations_a_subtree_two_consumers_share_is_printed_ ├─ #3 Result preserving · transparent · streaming │ │ Rows this plan hands out: to the trigger, or to the node reading it │ └─ #2 Filter reducing · transparent · streaming - │ │ Condition: IsNotNull + │ │ Condition: id IS NOT NULL │ └─ #1 Read source · transparent · streaming │ Extractor: ArrayExtractor └─ #4 Write preserving · opaque · streaming @@ -152,13 +152,13 @@ public function test_with_declarations_payloads_and_redefinitions(): void #7 Result preserving · transparent · streaming │ Rows this plan hands out: to the trigger, or to the node reading it └─ #6 Until reducing · transparent · streaming - │ Until: Literal + │ Until: true └─ #5 Offset reducing · transparent · streaming │ Skip: 2 └─ #4 Rename preserving · transparent · streaming · redefines m, n │ Rename: n → m └─ #3 WithColumn preserving · transparent · streaming · redefines n - │ Column: n = Literal + │ Column: n = 1 └─ #2 RenameEach preserving · transparent · streaming · redefines unknown └─ #1 Read source · transparent · streaming Extractor: ArrayExtractor diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sink/BranchedTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sink/BranchedTest.php index 36461c5d5b..ac4f7e6205 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sink/BranchedTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sink/BranchedTest.php @@ -35,7 +35,7 @@ public function test_the_condition_becomes_a_write_over_a_filter(): void └─ #4 Write preserving · opaque · streaming │ Loader: MemoryLoader └─ #3 Filter reducing · transparent · streaming - │ Condition: Equals + │ Condition: id = 1 └─ #1 Read (shared) PLAN, $dataFrame->explain()->toString(format: Format::declarations)); @@ -75,7 +75,7 @@ public function test_with_transformation_filters_before_it_transforms(): void └─ #4 Select preserving · transparent · streaming · redefines all but id │ Keeps only: id └─ #3 Filter reducing · transparent · streaming - │ Condition: Equals + │ Condition: id = 1 └─ #1 Read (shared) PLAN, $dataFrame->explain()->toString(format: Format::declarations)); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sink/TransactionalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sink/TransactionalTest.php index c83b7a9f4b..4f8f18a9b0 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sink/TransactionalTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sink/TransactionalTest.php @@ -57,7 +57,7 @@ public function test_every_child_becomes_a_write_of_one_transaction_sharing_the_ └─ #6 Write preserving · opaque · streaming │ Loader: MemoryLoader └─ #5 Filter reducing · transparent · streaming - │ Condition: Equals + │ Condition: id = 1 └─ #1 Read (shared) PLAN, $dataFrame->explain()->toString(format: Format::declarations)); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sink/TransformedTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sink/TransformedTest.php index 6aa9f5aa96..320fd82d1f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sink/TransformedTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sink/TransformedTest.php @@ -88,7 +88,7 @@ public function test_a_sink_child_extends_the_chain(): void └─ #5 Write preserving · opaque · streaming │ Loader: MemoryLoader └─ #4 Filter reducing · transparent · streaming - │ Condition: Equals + │ Condition: id = 1 └─ #3 Transform unknown · opaque · streaming · redefines unknown └─ #1 Read (shared) PLAN, $dataFrame->explain()->toString(format: Format::declarations)); diff --git a/src/lib/filesystem/src/Flow/Filesystem/Path/Filter/Filters.php b/src/lib/filesystem/src/Flow/Filesystem/Path/Filter/Filters.php index 1173f73f80..0172d4f1c5 100644 --- a/src/lib/filesystem/src/Flow/Filesystem/Path/Filter/Filters.php +++ b/src/lib/filesystem/src/Flow/Filesystem/Path/Filter/Filters.php @@ -8,6 +8,7 @@ use Flow\Filesystem\Path\Filter; use function array_merge; +use function array_values; final readonly class Filters implements Filter { @@ -36,4 +37,12 @@ public function add(Filter $filter): self { return new self(...array_merge($this->filters, [$filter])); } + + /** + * @return list + */ + public function filters(): array + { + return array_values($this->filters); + } } diff --git a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/Path/Filter/FiltersTest.php b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/Path/Filter/FiltersTest.php new file mode 100644 index 0000000000..09efed2fe6 --- /dev/null +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/Path/Filter/FiltersTest.php @@ -0,0 +1,21 @@ +add($keepAll)->filters()); + } +}