From 265ea5a34c6ec9a6a2c9627c5765bf800725e14b Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 28 Oct 2026 00:22:25 +0800 Subject: [PATCH 1/6] [datafusion] Upgrade to DataFusion 55.2.0 --- Cargo.toml | 28 +- benchmarks/tpcds/src/lib.rs | 9 +- bindings/c/DEPENDENCIES.rust.tsv | 32 +- bindings/go/DEPENDENCIES.rust.tsv | 32 +- bindings/python/Cargo.toml | 4 +- bindings/python/pyproject.toml | 2 +- bindings/python/src/context.rs | 4 +- .../integration_tests/DEPENDENCIES.rust.tsv | 32 +- .../datafusion/src/filter_pushdown.rs | 113 ++++- .../datafusion/src/lateral_vector_search.rs | 42 +- .../src/partition_count_pushdown.rs | 29 +- .../datafusion/src/physical_plan/audit_log.rs | 37 +- .../datafusion/src/physical_plan/scan.rs | 274 ++++++++++- .../src/physical_plan/search_score.rs | 31 +- .../datafusion/src/variant_pushdown.rs | 6 +- .../datafusion/src/vector_search.rs | 30 +- .../datafusion/tests/count_pushdown.rs | 2 +- .../tests/format_table_statistics.rs | 2 +- .../datafusion/tests/scan_pruning_stats.rs | 135 +++++- .../paimon-rest-server/DEPENDENCIES.rust.tsv | 32 +- crates/paimon/Cargo.toml | 9 +- crates/paimon/DEPENDENCIES.rust.tsv | 27 +- crates/paimon/src/arrow/format/avro.rs | 80 +++- crates/paimon/src/arrow/format/avro_write.rs | 91 +++- crates/paimon/src/arrow/format/mosaic.rs | 20 +- .../paimon/src/arrow/format/mosaic_write.rs | 5 +- crates/paimon/src/arrow/format/parquet.rs | 32 +- crates/paimon/src/arrow/format/row.rs | 167 +++++-- crates/paimon/src/arrow/format/vortex.rs | 21 +- crates/paimon/src/arrow/legacy.rs | 189 ++++++++ crates/paimon/src/arrow/mod.rs | 89 ++-- crates/paimon/src/arrow/multiset.rs | 446 ++++++++++++++++++ crates/paimon/src/arrow/nested_evolution.rs | 72 ++- crates/paimon/src/table/data_file_reader.rs | 12 +- crates/paimon/src/table/pk_full_text_read.rs | 6 +- .../src/table/pk_vector_indexed_split_read.rs | 12 +- .../src/table/pk_vector_orchestrator.rs | 6 +- .../src/table/pk_vector_position_read.rs | 12 +- .../residual_row_ranges_tests.rs | 6 +- .../src/table/postpone_bucket_planner.rs | 56 ++- .../paimon/src/table/write_batch_normalize.rs | 75 ++- docs/src/getting-started.md | 14 + docs/src/sql.md | 4 +- 43 files changed, 1973 insertions(+), 354 deletions(-) create mode 100644 crates/paimon/src/arrow/legacy.rs create mode 100644 crates/paimon/src/arrow/multiset.rs diff --git a/Cargo.toml b/Cargo.toml index 665c914a5..95f5d337a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,21 +36,21 @@ license = "Apache-2.0" rust-version = "1.95.0" [workspace.dependencies] -arrow = "58.0" -arrow-array = { version = "58.0", features = ["ffi"] } -arrow-arith = "58.0" -arrow-buffer = "58.0" -arrow-schema = "58.0" -arrow-cast = "58.0" -arrow-ord = "58.0" -arrow-row = "58.0" -arrow-select = "58.0" -arrow-string = "58.0" -datafusion = "54.0.0" -datafusion-ffi = "54.0.0" -datafusion-spark = { version = "54.0.0", features = ["core"] } +arrow = "59.2" +arrow-array = { version = "59.2", features = ["ffi"] } +arrow-arith = "59.2" +arrow-buffer = "59.2" +arrow-schema = "59.2" +arrow-cast = "59.2" +arrow-ord = "59.2" +arrow-row = "59.2" +arrow-select = "59.2" +arrow-string = "59.2" +datafusion = "55.2.0" +datafusion-ffi = "55.2.0" +datafusion-spark = { version = "55.0.0", features = ["core"] } paimon = { version = "0.6.0", path = "crates/paimon" } -parquet = "58.0" +parquet = "59.2" constant_time_eq = ">=0.4.0, <0.5.0" tokio = "1.39.2" tokio-util = "0.7" diff --git a/benchmarks/tpcds/src/lib.rs b/benchmarks/tpcds/src/lib.rs index b90f655dc..57ee72948 100644 --- a/benchmarks/tpcds/src/lib.rs +++ b/benchmarks/tpcds/src/lib.rs @@ -200,7 +200,12 @@ mod tests { 3 ); assert_eq!( - ctx.ctx().state().config_options().execution.batch_size, + ctx.ctx() + .state() + .config_options() + .execution + .batch_size + .get(), 4096 ); assert!( @@ -253,7 +258,7 @@ mod tests { assert_eq!(decoded.source, SourceKind::Paimon); assert!(decoded.disclosure.contains("non-TPC")); - assert_eq!(decoded.datafusion_version, "54.0.0"); + assert_eq!(decoded.datafusion_version, datafusion::DATAFUSION_VERSION); } #[test] diff --git a/bindings/c/DEPENDENCIES.rust.tsv b/bindings/c/DEPENDENCIES.rust.tsv index 8316c08d4..01cdc0d8c 100644 --- a/bindings/c/DEPENDENCIES.rust.tsv +++ b/bindings/c/DEPENDENCIES.rust.tsv @@ -12,20 +12,24 @@ anyhow@1.0.104 X X apache-avro@0.21.0 X approx@0.5.1 X arc-swap@1.9.2 X X -arrow@58.4.0 X -arrow-arith@58.4.0 X +arrow@59.3.2 X +arrow-arith@59.3.2 X arrow-array@58.4.0 X X +arrow-array@59.3.2 X X arrow-buffer@58.4.0 X -arrow-cast@58.4.0 X -arrow-csv@58.4.0 X +arrow-buffer@59.3.2 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X -arrow-ord@58.4.0 X -arrow-row@58.4.0 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X +arrow-ord@59.3.2 X +arrow-row@59.3.2 X arrow-schema@58.4.0 X -arrow-select@58.4.0 X -arrow-string@58.4.0 X +arrow-schema@59.3.2 X +arrow-select@59.3.2 X +arrow-string@59.3.2 X async-compression@0.4.42 X X async-stream@0.3.6 X async-stream-impl@0.3.6 X @@ -174,7 +178,6 @@ include-flate-codegen@0.3.4 X include-flate-compress@0.3.4 X indexmap@2.14.0 X X inout@0.1.4 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X itertools@0.14.0 X X @@ -235,6 +238,7 @@ no_std_io2@0.9.4 X X nom@7.1.3 X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-bigint-dig@0.8.6 X X num-complex@0.4.6 X X num-conv@0.2.2 X X @@ -259,8 +263,7 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X @@ -269,7 +272,7 @@ paimon-c@0.6.0 X paimon-ftindex-core@0.1.0 X paimon-mosaic-core@0.2.0 X paimon-vindex-core@0.5.0 X -parquet@58.4.0 X +parquet@59.3.2 X paste@1.0.15 X X pbkdf2@0.12.2 X X pem@4.0.0 X @@ -411,7 +414,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X diff --git a/bindings/go/DEPENDENCIES.rust.tsv b/bindings/go/DEPENDENCIES.rust.tsv index 8316c08d4..01cdc0d8c 100644 --- a/bindings/go/DEPENDENCIES.rust.tsv +++ b/bindings/go/DEPENDENCIES.rust.tsv @@ -12,20 +12,24 @@ anyhow@1.0.104 X X apache-avro@0.21.0 X approx@0.5.1 X arc-swap@1.9.2 X X -arrow@58.4.0 X -arrow-arith@58.4.0 X +arrow@59.3.2 X +arrow-arith@59.3.2 X arrow-array@58.4.0 X X +arrow-array@59.3.2 X X arrow-buffer@58.4.0 X -arrow-cast@58.4.0 X -arrow-csv@58.4.0 X +arrow-buffer@59.3.2 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X -arrow-ord@58.4.0 X -arrow-row@58.4.0 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X +arrow-ord@59.3.2 X +arrow-row@59.3.2 X arrow-schema@58.4.0 X -arrow-select@58.4.0 X -arrow-string@58.4.0 X +arrow-schema@59.3.2 X +arrow-select@59.3.2 X +arrow-string@59.3.2 X async-compression@0.4.42 X X async-stream@0.3.6 X async-stream-impl@0.3.6 X @@ -174,7 +178,6 @@ include-flate-codegen@0.3.4 X include-flate-compress@0.3.4 X indexmap@2.14.0 X X inout@0.1.4 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X itertools@0.14.0 X X @@ -235,6 +238,7 @@ no_std_io2@0.9.4 X X nom@7.1.3 X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-bigint-dig@0.8.6 X X num-complex@0.4.6 X X num-conv@0.2.2 X X @@ -259,8 +263,7 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X @@ -269,7 +272,7 @@ paimon-c@0.6.0 X paimon-ftindex-core@0.1.0 X paimon-mosaic-core@0.2.0 X paimon-vindex-core@0.5.0 X -parquet@58.4.0 X +parquet@59.3.2 X paste@1.0.15 X X pbkdf2@0.12.2 X X pem@4.0.0 X @@ -411,7 +414,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index bc459d552..2ae397cee 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -39,7 +39,7 @@ futures = "0.3" paimon = { path = "../../crates/paimon", features = ["storage-all", "internal-api"] } paimon-datafusion = { path = "../../crates/integrations/datafusion" } paimon-vindex-core = "0.5.0" -pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] } +pyo3 = { version = "0.29", features = ["abi3-py310", "chrono"] } roaring = "0.11" serde_json = "1.0" tokio = { workspace = true } @@ -47,4 +47,4 @@ tokio = { workspace = true } [dev-dependencies] # Enables `Python::attach` to auto-initialize an interpreter for Rust unit tests. # Only applied to the test build; the maturin/extension-module build does not pull dev-dependencies. -pyo3 = { version = "0.28", features = ["abi3-py310", "auto-initialize", "chrono"] } +pyo3 = { version = "0.29", features = ["abi3-py310", "auto-initialize", "chrono"] } diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 131404e9b..b94b89c3b 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -64,7 +64,7 @@ dev = [ "maturin>=1.9.4,<2.0", "pytest>=8.0", "pyarrow>=17.0,<24.0", - "datafusion==54.0.0", + "datafusion==55.0.0", "av>=17.0,<18.0", "pillow>=12.0,<13.0", ] diff --git a/bindings/python/src/context.rs b/bindings/python/src/context.rs index 10a952fd6..aa11040c6 100644 --- a/bindings/python/src/context.rs +++ b/bindings/python/src/context.rs @@ -357,11 +357,11 @@ impl PaimonCatalog { py: Python<'py>, session: Bound<'py, PyAny>, ) -> PyResult> { - let name = cr"datafusion_catalog_provider".into(); + let name = cr"datafusion_catalog_provider"; let provider = Arc::clone(&self.provider) as Arc; let codec = ffi_logical_codec_from_pycapsule(session)?; let provider = FFI_CatalogProvider::new_with_ffi_codec(provider, Some(runtime()), codec); - PyCapsule::new(py, provider, Some(name)) + PyCapsule::new_with_value(py, provider, name) } /// List all databases in this catalog. diff --git a/crates/integration_tests/DEPENDENCIES.rust.tsv b/crates/integration_tests/DEPENDENCIES.rust.tsv index a54283d15..ec13d5433 100644 --- a/crates/integration_tests/DEPENDENCIES.rust.tsv +++ b/crates/integration_tests/DEPENDENCIES.rust.tsv @@ -11,20 +11,24 @@ anyhow@1.0.104 X X apache-avro@0.21.0 X approx@0.5.1 X arc-swap@1.9.2 X X -arrow@58.4.0 X -arrow-arith@58.4.0 X +arrow@59.3.2 X +arrow-arith@59.3.2 X arrow-array@58.4.0 X X +arrow-array@59.3.2 X X arrow-buffer@58.4.0 X -arrow-cast@58.4.0 X -arrow-csv@58.4.0 X +arrow-buffer@59.3.2 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X -arrow-ord@58.4.0 X -arrow-row@58.4.0 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X +arrow-ord@59.3.2 X +arrow-row@59.3.2 X arrow-schema@58.4.0 X -arrow-select@58.4.0 X -arrow-string@58.4.0 X +arrow-schema@59.3.2 X +arrow-select@59.3.2 X +arrow-string@59.3.2 X async-compression@0.4.42 X X async-stream@0.3.6 X async-stream-impl@0.3.6 X @@ -165,7 +169,6 @@ include-flate@0.3.4 X include-flate-codegen@0.3.4 X include-flate-compress@0.3.4 X indexmap@2.14.0 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X itertools@0.14.0 X X @@ -226,6 +229,7 @@ no_std_io2@0.9.4 X X nom@7.1.3 X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-complex@0.4.6 X X num-conv@0.2.2 X X num-integer@0.1.46 X X @@ -243,8 +247,7 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X @@ -253,7 +256,7 @@ paimon-ftindex-core@0.1.0 X paimon-integration-tests@0.6.0 X paimon-mosaic-core@0.2.0 X paimon-vindex-core@0.5.0 X -parquet@58.4.0 X +parquet@59.3.2 X paste@1.0.15 X X percent-encoding@2.3.2 X X phf@0.12.1 X @@ -375,7 +378,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X diff --git a/crates/integrations/datafusion/src/filter_pushdown.rs b/crates/integrations/datafusion/src/filter_pushdown.rs index cdbe6460d..e13d797f8 100644 --- a/crates/integrations/datafusion/src/filter_pushdown.rs +++ b/crates/integrations/datafusion/src/filter_pushdown.rs @@ -249,6 +249,9 @@ impl<'a> FilterTranslator<'a> { let field = self.resolve_field(column_expr)?; let scalar = extract_scalar_literal(literal_expr)?; let datum = scalar_to_datum(scalar, field.data_type())?; + if is_float_zero(&datum) { + return None; + } match op { Operator::Eq => self.predicate_builder.equal(field.name(), datum).ok(), @@ -281,6 +284,9 @@ impl<'a> FilterTranslator<'a> { }) .collect(); let literals = literals?; + if literals.iter().any(is_float_zero) { + return None; + } self.exact(if in_list.negated { self.predicate_builder @@ -301,6 +307,9 @@ impl<'a> FilterTranslator<'a> { extract_scalar_literal(between.high.as_ref())?, field.data_type(), )?; + if is_float_zero(&low) || is_float_zero(&high) { + return None; + } // Native Between / NotBetween leaf: lets the planner / b-tree // recognize the range as a single op (see `btree::query::extract_between`). @@ -637,6 +646,19 @@ fn reverse_comparison_operator(op: Operator) -> Option { } } +/// Scalar comparisons in DataFusion 55 normalize signed zeroes, while Paimon's +/// partition and row predicates distinguish them. Reject zero boundaries instead +/// of dropping the other zero before DataFusion's residual can evaluate it. +/// Array membership still follows Arrow's total ordering, so it must retain its +/// own literal conversion without this restriction. +pub(crate) fn is_float_zero(datum: &Datum) -> bool { + match datum { + Datum::Float(value) => *value == 0.0, + Datum::Double(value) => *value == 0.0, + _ => false, + } +} + pub(crate) fn scalar_to_datum(scalar: &ScalarValue, data_type: &DataType) -> Option { match data_type { DataType::Boolean(_) => match scalar { @@ -773,8 +795,8 @@ mod tests { use paimon::catalog::Identifier; use paimon::io::FileIOBuilder; use paimon::spec::{ - ArrayType, BigIntType, FloatType, IntType, LocalZonedTimestampType, PredicateOperator, - Schema, SmallIntType, TableSchema, TimeType, TimestampType, VarCharType, + ArrayType, BigIntType, DoubleType, FloatType, IntType, LocalZonedTimestampType, + PredicateOperator, Schema, SmallIntType, TableSchema, TimeType, TimestampType, VarCharType, }; use paimon::table::Table; @@ -835,6 +857,93 @@ mod tests { } } + #[test] + fn test_float_zero_filters_fall_open_without_disabling_array_membership() { + use datafusion::functions_nested::expr_fn::array_has; + + for (data_type, zeroes) in [ + ( + DataType::Float(FloatType::new()), + [lit(0.0_f32), lit(-0.0_f32)], + ), + ( + DataType::Double(DoubleType::new()), + [lit(0.0_f64), lit(-0.0_f64)], + ), + ] { + let fields = vec![ + DataField::new(0, "id".into(), DataType::Int(IntType::new())), + DataField::new(1, "value".into(), data_type.clone()), + DataField::new( + 2, + "items".into(), + DataType::Array(ArrayType::new(data_type)), + ), + ]; + let column = Expr::Column(Column::from_name("value")); + let id_filter = Expr::Column(Column::from_name("id")).eq(lit(1)); + for zero in zeroes { + let mut filters = [ + Operator::Eq, + Operator::NotEq, + Operator::Lt, + Operator::LtEq, + Operator::Gt, + Operator::GtEq, + ] + .map(|op| { + Expr::BinaryExpr(BinaryExpr::new( + Box::new(column.clone()), + op, + Box::new(zero.clone()), + )) + }) + .to_vec(); + for negated in [false, true] { + filters.push(Expr::InList(InList::new( + Box::new(column.clone()), + vec![zero.clone()], + negated, + ))); + filters.push(Expr::Between(Between::new( + Box::new(column.clone()), + negated, + Box::new(zero.clone()), + Box::new(zero.clone()), + ))); + } + for filter in filters { + for filter in [filter.clone(), Expr::Not(Box::new(filter))] { + assert_eq!( + classify_filter_pushdown(&filter, &fields, true, |_| true), + TableProviderFilterPushDown::Unsupported, + "zero comparison must retain DataFusion semantics: {filter}" + ); + let conjunction = analyze_filters( + &[filter.clone().and(id_filter.clone())], + &fields, + true, + ); + assert!(conjunction.requires_residual); + assert_eq!( + conjunction.pushed_predicate, + build_pushed_predicate(std::slice::from_ref(&id_filter), &fields) + ); + assert!( + build_pushed_predicate(&[filter.or(id_filter.clone())], &fields) + .is_none() + ); + } + } + assert!(build_pushed_predicate( + &[array_has(Expr::Column(Column::from_name("items")), zero)], + &fields, + ) + .is_some()); + } + } + } + #[test] fn test_translate_datafusion_array_membership_functions() { use datafusion::functions_nested::expr_fn::{ diff --git a/crates/integrations/datafusion/src/lateral_vector_search.rs b/crates/integrations/datafusion/src/lateral_vector_search.rs index 2f5bea5dd..1ebb71836 100644 --- a/crates/integrations/datafusion/src/lateral_vector_search.rs +++ b/crates/integrations/datafusion/src/lateral_vector_search.rs @@ -34,14 +34,16 @@ use datafusion::arrow::array::{ use datafusion::arrow::compute::cast; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::catalog::default_table_source::source_as_provider; +use datafusion::catalog::Session; use datafusion::common::stats::Precision; -use datafusion::common::tree_node::Transformed; +use datafusion::common::tree_node::{Transformed, TreeNodeRecursion}; use datafusion::common::{ internal_err, DFSchemaRef, DataFusionError, JoinType, Result as DFResult, Statistics, }; use datafusion::datasource::TableProvider; -use datafusion::execution::context::{QueryPlanner, SessionState}; +use datafusion::execution::context::QueryPlanner; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; +use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext; use datafusion::logical_expr::utils::{conjunction, split_conjunction}; use datafusion::logical_expr::{ Expr, Extension, Filter, LogicalPlan, Projection, TableScan, UserDefinedLogicalNode, @@ -51,8 +53,8 @@ use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning, - PlanProperties, RecordBatchStream, + ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, + Partitioning, PlanProperties, RecordBatchStream, ReplaceChildrenOptions, StatisticsArgs, }; use datafusion::physical_planner::{DefaultPhysicalPlanner, ExtensionPlanner, PhysicalPlanner}; use datafusion::prelude::SessionConfig; @@ -80,7 +82,7 @@ impl QueryPlanner for PaimonQueryPlanner { async fn create_physical_plan( &self, logical_plan: &LogicalPlan, - session_state: &SessionState, + session_state: &dyn Session, ) -> DFResult> { let planner = DefaultPhysicalPlanner::with_extension_planners(vec![ Arc::new(LateralVectorSearchExtensionPlanner), @@ -418,7 +420,8 @@ impl ExtensionPlanner for LateralVectorSearchExtensionPlanner { node: &dyn UserDefinedLogicalNode, logical_inputs: &[&LogicalPlan], physical_inputs: &[Arc], - session_state: &SessionState, + session_state: &dyn Session, + planning_ctx: &PhysicalPlanningContext, ) -> DFResult>> { let Some(node) = node.as_any().downcast_ref::() else { return Ok(None); @@ -431,6 +434,7 @@ impl ExtensionPlanner for LateralVectorSearchExtensionPlanner { node.query_vector_expr(), logical_inputs[0].schema(), session_state, + planning_ctx, )?; let mut exec = LateralVectorSearchExec::new( Arc::clone(&physical_inputs[0]), @@ -475,7 +479,7 @@ struct ExecutionPreparedFilterEntry { struct ExecutionPreparedFilterCache { // DataFusion passes the same TaskContext Arc to every partition of one // execution. Keep the prepared filter alive for that TaskContext so - // sequential partitions resolve the same target snapshot. DataFusion 54 + // sequential partitions resolve the same target snapshot. DataFusion // exposes no TaskContext drop hook, so one cache-level reaper observes the // weak contexts and removes subset executions after their context dies. entries: Mutex>, @@ -857,9 +861,27 @@ impl ExecutionPlan for LateralVectorSearchExec { vec![&self.input] } + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + datafusion::physical_plan::apply_expression_roots([&self.query_vector_expr], f) + } + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DFResult> { + self.replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + } + + fn replace_children( self: Arc, mut children: Vec>, + _options: ReplaceChildrenOptions, ) -> DFResult> { if children.len() != 1 { return internal_err!("LateralVectorSearchExec expects one child"); @@ -920,7 +942,11 @@ impl ExecutionPlan for LateralVectorSearchExec { }) } - fn partition_statistics(&self, _partition: Option) -> DFResult> { + fn statistics_from_inputs( + &self, + _input_stats: &[Arc], + _args: &StatisticsArgs, + ) -> DFResult> { Ok(Arc::new(Statistics { num_rows: Precision::Absent, total_byte_size: Precision::Absent, diff --git a/crates/integrations/datafusion/src/partition_count_pushdown.rs b/crates/integrations/datafusion/src/partition_count_pushdown.rs index f7fc0cbc0..3a1f90f6f 100644 --- a/crates/integrations/datafusion/src/partition_count_pushdown.rs +++ b/crates/integrations/datafusion/src/partition_count_pushdown.rs @@ -55,6 +55,7 @@ use datafusion::catalog::Session; use datafusion::common::tree_node::Transformed; use datafusion::common::{ internal_err, project_schema, Column, DataFusionError, Result as DFResult, ScalarValue, + TableReference, }; use datafusion::datasource::{provider_as_source, source_as_provider, TableProvider, TableType}; use datafusion::execution::{SendableRecordBatchStream, SessionState, TaskContext}; @@ -63,13 +64,12 @@ use datafusion::functions_aggregate::count::count_udaf; use datafusion::functions_aggregate::expr_fn::{count, sum}; use datafusion::logical_expr::expr::AggregateFunction; use datafusion::logical_expr::{ - col, lit, Aggregate, Expr, LogicalPlan, LogicalPlanBuilder, TableScan, TableSource, + col, lit, Aggregate, Expr, LogicalPlan, LogicalPlanBuilder, TableScanBuilder, TableSource, }; use datafusion::optimizer::{ApplyOrder, OptimizerConfig, OptimizerRule}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::streaming::{PartitionStream, StreamingTableExec}; use datafusion::physical_plan::ExecutionPlan; -use datafusion::sql::TableReference; use futures::{stream, TryStreamExt}; use paimon::spec::{CoreOptions, DataField, Datum, Predicate}; use paimon::table::Table; @@ -191,13 +191,9 @@ fn rewrite_aggregate(aggregate: &Aggregate) -> DFResult> { }; // Only the synthetic scan changes qualifier; fallback filters still refer // to the original scan's table name. - let counts_scan = LogicalPlan::TableScan(TableScan::try_new( - qualifier.clone(), - provider_as_source(Arc::new(counts)), - None, - vec![], - None, - )?); + let counts_scan = LogicalPlan::TableScan( + TableScanBuilder::new(qualifier.clone(), provider_as_source(Arc::new(counts))).build()?, + ); let row_count = Expr::Column(Column::new(Some(qualifier.clone()), ROW_COUNT_COLUMN)); let summed = LogicalPlan::Aggregate(Aggregate::try_new( @@ -510,13 +506,12 @@ impl PartitionRowCountStream { .iter() .map(|field| source_schema.index_of(field.name())) .collect::, _>>()?; - let scan = LogicalPlan::TableScan(TableScan::try_new( - self.table_name.clone(), - Arc::clone(source), - Some(partition_indices), - self.filters.clone(), - None, - )?); + let scan = LogicalPlan::TableScan( + TableScanBuilder::new(self.table_name.clone(), Arc::clone(source)) + .with_projection(Some(partition_indices)) + .with_filters(self.filters.clone()) + .build()?, + ); let group_by: Vec = self .partition_fields @@ -538,7 +533,7 @@ impl PartitionRowCountStream { // rewrite its own fallback. self.state .query_planner() - .create_physical_plan(&counted, &self.state) + .create_physical_plan(&counted, self.state.as_ref()) .await } } diff --git a/crates/integrations/datafusion/src/physical_plan/audit_log.rs b/crates/integrations/datafusion/src/physical_plan/audit_log.rs index 50df0b1e6..7b48d58d6 100644 --- a/crates/integrations/datafusion/src/physical_plan/audit_log.rs +++ b/crates/integrations/datafusion/src/physical_plan/audit_log.rs @@ -19,10 +19,15 @@ use std::sync::Arc; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{stats::Precision, Statistics}; use datafusion::error::Result as DFResult; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_plan::{DisplayAs, ExecutionPlan, PlanProperties}; +use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_plan::{ + ChildrenPropertiesMode, DisplayAs, ExecutionPlan, PlanProperties, ReplaceChildrenOptions, + StatisticsArgs, +}; use paimon::table::AuditLogRead; use super::PaimonTableScan; @@ -52,9 +57,27 @@ impl ExecutionPlan for PaimonAuditLogScan { vec![] } + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + self.inner.apply_expressions(f) + } + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DFResult> { + self.replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + } + + fn replace_children( self: Arc, _children: Vec>, + _options: ReplaceChildrenOptions, ) -> DFResult> { Ok(self) } @@ -69,8 +92,12 @@ impl ExecutionPlan for PaimonAuditLogScan { }) } - fn partition_statistics(&self, partition: Option) -> DFResult> { - let mut statistics = self.inner.partition_statistics(partition)?; + fn statistics_from_inputs( + &self, + _input_stats: &[Arc], + args: &StatisticsArgs, + ) -> DFResult> { + let mut statistics = self.inner.statistics_from_inputs(&[], args)?; Arc::make_mut(&mut statistics).num_rows = Precision::Absent; Ok(statistics) } @@ -202,7 +229,9 @@ mod tests { )); assert!(result.updated_node.is_none()); assert_eq!( - scan.partition_statistics(None).unwrap().num_rows, + scan.statistics_from_inputs(&[], &StatisticsArgs::new()) + .unwrap() + .num_rows, Precision::Absent ); } diff --git a/crates/integrations/datafusion/src/physical_plan/scan.rs b/crates/integrations/datafusion/src/physical_plan/scan.rs index e7edacbaa..9ee660d75 100644 --- a/crates/integrations/datafusion/src/physical_plan/scan.rs +++ b/crates/integrations/datafusion/src/physical_plan/scan.rs @@ -24,6 +24,7 @@ use datafusion::arrow::datatypes::{ }; use datafusion::arrow::record_batch::{RecordBatch, RecordBatchOptions}; use datafusion::common::stats::Precision; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{ColumnStatistics, ScalarValue, Statistics}; use datafusion::config::ConfigOptions; use datafusion::datasource::physical_plan::parquet::can_expr_be_pushed_down_with_schemas; @@ -47,7 +48,10 @@ use datafusion::physical_plan::filter_pushdown::{ ChildPushdownResult, FilterPushdownPhase, FilterPushdownPropagation, PushedDown, }; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{DisplayAs, ExecutionPlan, Partitioning, PlanProperties}; +use datafusion::physical_plan::{ + ChildrenPropertiesMode, DisplayAs, ExecutionPlan, Partitioning, PlanProperties, + ReplaceChildrenOptions, StatisticsArgs, +}; use futures::{FutureExt, StreamExt, TryStreamExt}; use paimon::arrow::ReadBudget; use paimon::spec::{ @@ -57,7 +61,7 @@ use paimon::table::{ArrowRecordBatchStream, ScanStats, Table, TableRead}; use paimon::DataSplit; use crate::error::to_datafusion_error; -use crate::filter_pushdown::scalar_to_datum; +use crate::filter_pushdown::{is_float_zero, scalar_to_datum}; fn to_datafusion_batch(batch: RecordBatch, schema: &ArrowSchemaRef) -> DFResult { if batch.num_columns() != schema.fields().len() { @@ -359,6 +363,9 @@ fn translate_physical_predicate( scalar_to_datum(literal.value(), field.data_type()) }) .collect::>>()?; + if literals.iter().any(is_float_zero) { + return None; + } return if in_list.negated() { predicate_builder.is_not_in(field.name(), literals).ok() } else { @@ -513,6 +520,12 @@ fn translate_physical_comparison( }) })?; + // DataFusion equates signed zeros, while the native reader orders them separately. + // Keep zero boundaries in DataFusion so native pruning cannot discard matches. + if is_float_zero(&datum) { + return None; + } + if matches!( field.data_type(), paimon::spec::DataType::Binary(_) | paimon::spec::DataType::VarBinary(_) @@ -1109,9 +1122,30 @@ impl ExecutionPlan for PaimonTableScan { vec![] } + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + datafusion::physical_plan::apply_expression_roots( + self.runtime_filters.iter().chain(&self.decoder_filters), + f, + ) + } + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DFResult> { + self.replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + } + + fn replace_children( self: Arc, _children: Vec>, + _options: ReplaceChildrenOptions, ) -> DFResult> { Ok(self) } @@ -1192,12 +1226,16 @@ impl ExecutionPlan for PaimonTableScan { self.execute_with(partition, context, |read, splits| read.to_arrow(splits)) } - fn partition_statistics(&self, partition: Option) -> DFResult> { + fn statistics_from_inputs( + &self, + _input_stats: &[Arc], + args: &StatisticsArgs, + ) -> DFResult> { // Statistics include hidden rows and unmasked values. if self.has_query_auth_rules() { return Ok(Arc::new(Statistics::new_unknown(&self.schema()))); } - let partitions: &[Arc<[DataSplit]>] = match partition { + let partitions: &[Arc<[DataSplit]>] = match args.partition() { Some(idx) => std::slice::from_ref(&self.planned_partitions[idx]), None => &self.planned_partitions, }; @@ -1465,24 +1503,38 @@ mod tests { #[test] fn test_partition_statistics_include_manifest_column_bounds() { - let split = paimon::DataSplitBuilder::new() - .with_snapshot(1) - .with_partition(BinaryRow::new(0)) - .with_bucket(0) - .with_bucket_path("file:/tmp/test-table/bucket-0".to_string()) - .with_total_buckets(1) - .with_data_files(vec![ - data_file_with_int_stats("a.parquet", 4, 2, 8, 1), - data_file_with_int_stats("b.parquet", 6, 1, 10, 2), - ]) - .build() - .unwrap(); + let split = |file| { + paimon::DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path("file:/tmp/test-table/bucket-0".to_string()) + .with_total_buckets(1) + .with_data_files(vec![file]) + .build() + .unwrap() + }; let scan = PaimonTableScan::new( test_schema(), dummy_table(), test_read_type(), None, - vec![Arc::from(vec![split])], + vec![ + Arc::from(vec![split(data_file_with_int_stats( + "a.parquet", + 4, + 2, + 8, + 1, + ))]), + Arc::from(vec![split(data_file_with_int_stats( + "b.parquet", + 6, + 1, + 10, + 2, + ))]), + ], None, true, None, @@ -1490,7 +1542,8 @@ mod tests { true, ); - let statistics = scan.partition_statistics(None).unwrap(); + let context = datafusion::physical_plan::StatisticsContext::new(); + let statistics = context.compute(&scan, &StatisticsArgs::new()).unwrap(); let id = &statistics.column_statistics[0]; assert_eq!( @@ -1502,6 +1555,24 @@ mod tests { Precision::Inexact(ScalarValue::Int32(Some(10))) ); assert_eq!(id.null_count, Precision::Exact(3)); + for (partition, min, max, nulls) in [(0, 2, 8, 1), (1, 1, 10, 2)] { + let statistics = context + .compute( + &scan, + &StatisticsArgs::new().with_partition(Some(partition)), + ) + .unwrap(); + let id = &statistics.column_statistics[0]; + assert_eq!( + id.min_value, + Precision::Inexact(ScalarValue::Int32(Some(min))) + ); + assert_eq!( + id.max_value, + Precision::Inexact(ScalarValue::Int32(Some(max))) + ); + assert_eq!(id.null_count, Precision::Exact(nulls)); + } } #[test] @@ -1528,7 +1599,9 @@ mod tests { true, ); - let statistics = scan.partition_statistics(None).unwrap(); + let statistics = scan + .statistics_from_inputs(&[], &StatisticsArgs::new()) + .unwrap(); assert_eq!(statistics.num_rows, Precision::Exact(4)); assert_eq!(statistics.total_byte_size, Precision::Absent); @@ -1572,7 +1645,10 @@ mod tests { true, ); assert_eq!( - scan.partition_statistics(None).unwrap().column_statistics[0].null_count, + scan.statistics_from_inputs(&[], &StatisticsArgs::new()) + .unwrap() + .column_statistics[0] + .null_count, Precision::Absent ); } @@ -1628,7 +1704,10 @@ mod tests { true, ); - let value_stats = &scan.partition_statistics(None).unwrap().column_statistics[0]; + let value_stats = &scan + .statistics_from_inputs(&[], &StatisticsArgs::new()) + .unwrap() + .column_statistics[0]; assert_eq!(value_stats.min_value, Precision::Absent); assert_eq!(value_stats.max_value, Precision::Absent); assert_eq!(value_stats.null_count, Precision::Absent); @@ -1667,7 +1746,17 @@ mod tests { let updated = result .updated_node .expect("the scan must retain physical filters so dynamic join filters remain active"); - let statistics = updated.partition_statistics(None).unwrap(); + let mut found_dynamic_filter = false; + updated + .apply_expressions(&mut |expr| { + found_dynamic_filter |= expr.downcast_ref::().is_some(); + Ok(TreeNodeRecursion::Continue) + }) + .unwrap(); + assert!(found_dynamic_filter); + let statistics = updated + .statistics_from_inputs(&[], &StatisticsArgs::new()) + .unwrap(); assert_eq!(statistics.num_rows, Precision::Inexact(0)); assert_eq!( statistics.column_statistics[0].null_count, @@ -2095,7 +2184,14 @@ mod tests { datafusion::logical_expr::col("name"), datafusion::logical_expr::lit("ab"), ); - let physical = create_physical_expr(&logical, &df_schema, &ExecutionProps::new()).unwrap(); + let physical = create_physical_expr( + &logical, + &df_schema, + &ExecutionProps::new(), + &datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext::default( + ), + ) + .unwrap(); assert!(paimon_predicate_covers_filter( Some(&pushed), @@ -2228,6 +2324,138 @@ mod tests { )); } + #[test] + fn test_completed_dynamic_float_zero_filter_keeps_both_signed_zeros() { + use datafusion::arrow::array::{ArrayRef, Float32Array, Float64Array}; + use paimon::spec::{DoubleType, FloatType}; + + type FilterCase = (Arc, Vec>); + + for zero in [ + ScalarValue::Float32(Some(0.0)), + ScalarValue::Float32(Some(-0.0)), + ScalarValue::Float64(Some(0.0)), + ScalarValue::Float64(Some(-0.0)), + ] { + let (data_type, values): (_, ArrayRef) = match zero { + ScalarValue::Float32(_) => ( + paimon::spec::DataType::Float(FloatType::new()), + Arc::new(Float32Array::from(vec![ + Some(-1.0), + Some(-0.0), + Some(0.0), + Some(1.0), + None, + ])), + ), + _ => ( + paimon::spec::DataType::Double(DoubleType::new()), + Arc::new(Float64Array::from(vec![ + Some(-1.0), + Some(-0.0), + Some(0.0), + Some(1.0), + None, + ])), + ), + }; + let fields = vec![DataField::new(0, "value".to_string(), data_type)]; + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "value", + values.data_type().clone(), + true, + )])); + let batch = RecordBatch::try_new(Arc::clone(&schema), vec![values]).unwrap(); + let column: Arc = Arc::new(Column::new("value", 0)); + let is_negative_zero = match &zero { + ScalarValue::Float32(Some(value)) => value.is_sign_negative(), + ScalarValue::Float64(Some(value)) => value.is_sign_negative(), + _ => unreachable!(), + }; + let literal = lit(zero); + let mut cases: Vec = Vec::new(); + for (op, expected) in [ + (Operator::Eq, [false, true, true, false]), + (Operator::NotEq, [true, false, false, true]), + (Operator::Lt, [true, false, false, false]), + (Operator::LtEq, [true, true, true, false]), + (Operator::Gt, [false, false, false, true]), + (Operator::GtEq, [false, true, true, true]), + ] { + let expected = expected.into_iter().map(Some).chain([None]).collect(); + cases.push(( + Arc::new(BinaryExpr::new( + Arc::clone(&column), + op, + Arc::clone(&literal), + )), + expected, + )); + } + cases.push(( + Arc::new(BinaryExpr::new( + Arc::clone(&literal), + Operator::LtEq, + Arc::clone(&column), + )), + vec![Some(false), Some(true), Some(true), Some(true), None], + )); + // Physical IN retains total-order equality even though BinaryExpr + // comparisons normalize signed zeros. Preserve each expression's semantics. + for negated in [false, true] { + cases.push(( + Arc::new( + InListExpr::try_new( + Arc::clone(&column), + vec![Arc::clone(&literal)], + negated, + schema.as_ref(), + ) + .unwrap(), + ), + [false, is_negative_zero, !is_negative_zero, false] + .into_iter() + .map(|matched| Some(matched != negated)) + .chain([None]) + .collect(), + )); + } + cases.push(( + Arc::new(NotExpr::new(Arc::clone(&cases[0].0))), + vec![Some(true), Some(false), Some(false), Some(true), None], + )); + + for (snapshot, expected) in cases { + let dynamic = Arc::new(DynamicFilterPhysicalExpr::new( + vec![Arc::clone(&column)], + lit(true), + )); + dynamic.update(Arc::clone(&snapshot)).unwrap(); + dynamic.mark_complete(); + let dynamic: Arc = dynamic; + let plan = partition_runtime_decoder_filters(&[dynamic], &fields, true); + assert!( + plan.paimon_predicates.is_empty(), + "native total-order comparisons cannot implement {snapshot}" + ); + assert_eq!(plan.datafusion_filters.len(), 1); + let factory = DataFusionRowFilterFactory::new( + conjunction(plan.datafusion_filters), + Arc::clone(&schema), + ); + let mut filters = paimon::arrow::RowFilterFactory::create( + &factory, + paimon::arrow::RowFilterContext { + file_schema: &schema, + }, + ) + .unwrap(); + let actual = filters[0].evaluate(batch.clone()).unwrap(); + assert_eq!(actual.iter().collect::>(), expected, "{snapshot}"); + } + } + } + #[test] fn test_incomplete_dynamic_filter_stays_live_in_datafusion() { let fields = test_read_type(); diff --git a/crates/integrations/datafusion/src/physical_plan/search_score.rs b/crates/integrations/datafusion/src/physical_plan/search_score.rs index 39d01cedc..8bcbe1d2b 100644 --- a/crates/integrations/datafusion/src/physical_plan/search_score.rs +++ b/crates/integrations/datafusion/src/physical_plan/search_score.rs @@ -24,15 +24,16 @@ use datafusion::arrow::array::{ }; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::common::stats::Precision; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{internal_err, DataFusionError, Statistics}; use datafusion::error::Result as DFResult; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::EquivalenceProperties; +use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning, - PlanProperties, + ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, + Partitioning, PlanProperties, ReplaceChildrenOptions, StatisticsArgs, }; use futures::StreamExt; @@ -155,9 +156,27 @@ impl ExecutionPlan for SearchScoreExec { vec![&self.input] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + Ok(TreeNodeRecursion::Continue) + } + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DFResult> { + self.replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + } + + fn replace_children( self: Arc, mut children: Vec>, + _options: ReplaceChildrenOptions, ) -> DFResult> { if children.len() != 1 { return internal_err!("SearchScoreExec expects one child"); @@ -185,7 +204,11 @@ impl ExecutionPlan for SearchScoreExec { ))) } - fn partition_statistics(&self, _partition: Option) -> DFResult> { + fn statistics_from_inputs( + &self, + _input_stats: &[Arc], + _args: &StatisticsArgs, + ) -> DFResult> { Ok(Arc::new(Statistics { num_rows: Precision::Absent, total_byte_size: Precision::Absent, diff --git a/crates/integrations/datafusion/src/variant_pushdown.rs b/crates/integrations/datafusion/src/variant_pushdown.rs index 15ecfb403..d4c197a95 100644 --- a/crates/integrations/datafusion/src/variant_pushdown.rs +++ b/crates/integrations/datafusion/src/variant_pushdown.rs @@ -25,13 +25,14 @@ use std::sync::Arc; use async_trait::async_trait; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::catalog::default_table_source::source_as_provider; +use datafusion::catalog::Session; use datafusion::common::tree_node::{Transformed, TreeNode}; use datafusion::common::{ internal_err, plan_err, Column, DFSchema, DFSchemaRef, Result as DFResult, ScalarValue, }; -use datafusion::execution::context::SessionState; use datafusion::functions::core::expr_fn::get_field; use datafusion::logical_expr::expr::{InList, ScalarFunction}; +use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext; use datafusion::logical_expr::{ Between, BinaryExpr, Case, Cast, Expr, Extension, Filter, Like, LogicalPlan, Projection, TableScan, TryCast, UserDefinedLogicalNode, @@ -199,7 +200,8 @@ impl ExtensionPlanner for VariantExtractionExtensionPlanner { node: &dyn UserDefinedLogicalNode, logical_inputs: &[&LogicalPlan], physical_inputs: &[Arc], - session_state: &SessionState, + session_state: &dyn Session, + _planning_ctx: &PhysicalPlanningContext, ) -> DFResult>> { let Some(node) = node.as_any().downcast_ref::() else { return Ok(None); diff --git a/crates/integrations/datafusion/src/vector_search.rs b/crates/integrations/datafusion/src/vector_search.rs index 715ce37ee..c483d0843 100644 --- a/crates/integrations/datafusion/src/vector_search.rs +++ b/crates/integrations/datafusion/src/vector_search.rs @@ -28,17 +28,19 @@ use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::catalog::Session; use datafusion::catalog::TableFunctionImpl; use datafusion::common::stats::Precision; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{internal_err, project_schema, Statistics}; use datafusion::datasource::{TableProvider, TableType}; use datafusion::error::{DataFusionError, Result as DFResult}; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::logical_expr::{Expr, TableProviderFilterPushDown}; -use datafusion::physical_expr::EquivalenceProperties; +use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; use datafusion::physical_plan::empty::EmptyExec; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, + ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, + PlanProperties, ReplaceChildrenOptions, StatisticsArgs, }; use datafusion::prelude::SessionContext; use futures::{stream, TryStreamExt}; @@ -475,9 +477,27 @@ impl ExecutionPlan for VectorSearchExec { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + Ok(TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, children: Vec>, + ) -> DFResult> { + self.replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + } + + fn replace_children( + self: Arc, + children: Vec>, + _options: ReplaceChildrenOptions, ) -> DFResult> { if !children.is_empty() { return internal_err!("VectorSearchExec is a leaf and takes no children"); @@ -503,7 +523,11 @@ impl ExecutionPlan for VectorSearchExec { ))) } - fn partition_statistics(&self, _partition: Option) -> DFResult> { + fn statistics_from_inputs( + &self, + _input_stats: &[Arc], + _args: &StatisticsArgs, + ) -> DFResult> { Ok(Arc::new(Statistics { num_rows: Precision::Absent, total_byte_size: Precision::Absent, diff --git a/crates/integrations/datafusion/tests/count_pushdown.rs b/crates/integrations/datafusion/tests/count_pushdown.rs index fd800618c..f76420eaf 100644 --- a/crates/integrations/datafusion/tests/count_pushdown.rs +++ b/crates/integrations/datafusion/tests/count_pushdown.rs @@ -19,7 +19,7 @@ //! //! DataFusion's `aggregate_statistics` optimizer rule can replace a //! scan+aggregate subtree with a literal projection when the source -//! reports `Precision::Exact(num_rows)` from `partition_statistics()`. +//! reports `Precision::Exact(num_rows)` from `statistics_from_inputs()`. //! //! These tests verify that: //! - COUNT(*) pushdown succeeds when statistics are exact (no PaimonTableScan in plan) diff --git a/crates/integrations/datafusion/tests/format_table_statistics.rs b/crates/integrations/datafusion/tests/format_table_statistics.rs index b29ae0f1e..667fb7832 100644 --- a/crates/integrations/datafusion/tests/format_table_statistics.rs +++ b/crates/integrations/datafusion/tests/format_table_statistics.rs @@ -151,7 +151,7 @@ async fn test_empty_format_table_counts_zero() { ); } -/// The partition-pruned plan reaches `partition_statistics()` too, so a pruned +/// The partition-pruned plan reaches `statistics_from_inputs()` too, so a pruned /// `COUNT(*)` must also come from the data and not from the placeholder. /// /// Only predicated queries are asserted here. An unfiltered scan over a diff --git a/crates/integrations/datafusion/tests/scan_pruning_stats.rs b/crates/integrations/datafusion/tests/scan_pruning_stats.rs index 27929709a..dd721945d 100644 --- a/crates/integrations/datafusion/tests/scan_pruning_stats.rs +++ b/crates/integrations/datafusion/tests/scan_pruning_stats.rs @@ -355,9 +355,8 @@ async fn test_physical_plan_displays_scan_stats_summary() { /// files here are exactly the shapes the writer produces for the two ways the /// rule breaks: `[1.0, NaN]` reports `min == max == 1.0` over two rows because /// NaN goes into neither min/max nor the null count, and a file holding only -/// `-0.0` reports `min = -0.0, max = 0.0`, which `Datum` sees as equal to `0.0` -/// while the row-level filter does not. Pruning either file drops a row the -/// filter would have returned. +/// `-0.0` reports `min = -0.0, max = 0.0`. Keep pruning conservative for both +/// layouts and let DataFusion apply its own NaN and signed-zero semantics. #[tokio::test] async fn test_not_equal_keeps_float_files_with_nan_or_signed_zero() { let (_tmp, catalog) = common::create_test_env(); @@ -395,11 +394,11 @@ async fn test_not_equal_keeps_float_files_with_nan_or_signed_zero() { "SELECT id FROM paimon.test_db.stats_float WHERE d <> 1.0 ORDER BY id", vec![2, 3], ), - // `-0.0` is not `0.0` to the filter, so row 3 survives; rows 1 and 2 are - // kept as well because `1.0 <> 0.0` and NaN is not `0.0` either. + // DataFusion 55 treats signed zeroes as equal, matching its native + // in-memory reader; NaN and 1.0 still differ from zero. ( "SELECT id FROM paimon.test_db.stats_float WHERE d <> 0.0 ORDER BY id", - vec![1, 2, 3], + vec![1, 2], ), ( "SELECT id FROM paimon.test_db.stats_float WHERE d NOT IN (1.0, 2.0) ORDER BY id", @@ -423,3 +422,127 @@ async fn test_not_equal_keeps_float_files_with_nan_or_signed_zero() { assert_eq!(ids, expected, "wrong rows for `{sql}`"); } } + +/// DataFusion 55 compares both zero signs as equal. Paimon's own predicates +/// retain Arrow/Java total-order semantics, including when pruning partitions. +/// Compare against an unfiltered in-memory copy so an inexact pushdown cannot +/// silently remove rows before DataFusion applies its residual. +#[tokio::test] +async fn test_float_zero_pushdown_matches_native_datafusion() { + use std::sync::Arc; + + use datafusion::arrow::array::Int32Array; + use datafusion::arrow::record_batch::RecordBatch; + use datafusion::datasource::MemTable; + use datafusion::prelude::SessionContext; + + fn ids(batches: &[RecordBatch]) -> Vec { + batches + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect() + } + + let (_tmp, catalog) = common::create_test_env(); + let sql_context = common::create_sql_context(catalog).await; + common::exec(&sql_context, "CREATE SCHEMA paimon.test_db").await; + for (name, partitioning) in [ + ("zero_data", ""), + ("zero_partitions", "PARTITIONED BY (f, d)"), + ] { + common::exec( + &sql_context, + &format!( + "CREATE TABLE paimon.test_db.{name} \ + (id INT, f FLOAT, d DOUBLE, items ARRAY) {partitioning}" + ), + ) + .await; + common::exec( + &sql_context, + &format!( + "INSERT INTO paimon.test_db.{name} VALUES \ + (1, CAST(-0.0 AS FLOAT), CAST(-0.0 AS DOUBLE), [CAST(-0.0 AS DOUBLE)]), \ + (2, CAST(0.0 AS FLOAT), CAST(0.0 AS DOUBLE), [CAST(0.0 AS DOUBLE)]), \ + (3, -1.0, -1.0, [-1.0]), (4, 1.0, 1.0, [1.0]), \ + (5, CAST('NaN' AS FLOAT), CAST('NaN' AS DOUBLE), [CAST('NaN' AS DOUBLE)]), \ + (6, NULL, NULL, NULL)" + ), + ) + .await; + let batches = sql_context + .sql(&format!("SELECT * FROM paimon.test_db.{name}")) + .await + .unwrap() + .collect() + .await + .unwrap(); + let native = SessionContext::new(); + native + .register_table( + "native", + Arc::new(MemTable::try_new(batches[0].schema(), vec![batches]).unwrap()), + ) + .unwrap(); + + let mut predicates = Vec::new(); + for column in ["f", "d"] { + for zero in ["0.0", "-0.0"] { + for operator in ["=", "<>", "<", "<=", ">", ">="] { + predicates.push(format!("{column} {operator} {zero}")); + } + predicates.extend([ + format!("{zero} = {column}"), + format!("{column} IN ({zero}, 1.0)"), + format!("{column} NOT IN ({zero}, 1.0)"), + format!("{column} BETWEEN {zero} AND 1.0"), + format!("{column} BETWEEN -1.0 AND {zero}"), + format!("{column} NOT BETWEEN {zero} AND 1.0"), + format!("{column} NOT BETWEEN -1.0 AND {zero}"), + format!("NOT ({column} = {zero})"), + format!("NOT ({column} <= {zero})"), + format!("{column} = {zero} AND id > 0"), + format!("{column} = {zero} OR id = 4"), + ]); + } + } + // Array membership continues to distinguish zero signs in DataFusion 55. + for function in ["array_has", "array_has_any", "array_has_all"] { + let needle = if function == "array_has" { + "0.0" + } else { + "[0.0]" + }; + predicates.push(format!("{function}(items, {needle})")); + } + for predicate in predicates { + let actual = sql_context + .sql(&format!( + "SELECT id FROM paimon.test_db.{name} WHERE {predicate} ORDER BY id" + )) + .await + .unwrap() + .collect() + .await + .unwrap(); + let expected = native + .sql(&format!( + "SELECT id FROM native WHERE {predicate} ORDER BY id" + )) + .await + .unwrap() + .collect() + .await + .unwrap(); + assert_eq!(ids(&actual), ids(&expected), "{name}: {predicate}"); + } + } +} diff --git a/crates/paimon-rest-server/DEPENDENCIES.rust.tsv b/crates/paimon-rest-server/DEPENDENCIES.rust.tsv index 35d5429d5..ecdb37da4 100644 --- a/crates/paimon-rest-server/DEPENDENCIES.rust.tsv +++ b/crates/paimon-rest-server/DEPENDENCIES.rust.tsv @@ -11,20 +11,24 @@ anyhow@1.0.104 X X apache-avro@0.21.0 X approx@0.5.1 X arc-swap@1.9.2 X X -arrow@58.4.0 X -arrow-arith@58.4.0 X +arrow@59.3.2 X +arrow-arith@59.3.2 X arrow-array@58.4.0 X X +arrow-array@59.3.2 X X arrow-buffer@58.4.0 X -arrow-cast@58.4.0 X -arrow-csv@58.4.0 X +arrow-buffer@59.3.2 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X -arrow-ord@58.4.0 X -arrow-row@58.4.0 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X +arrow-ord@59.3.2 X +arrow-row@59.3.2 X arrow-schema@58.4.0 X -arrow-select@58.4.0 X -arrow-string@58.4.0 X +arrow-schema@59.3.2 X +arrow-select@59.3.2 X +arrow-string@59.3.2 X async-compression@0.4.42 X X async-stream@0.3.6 X async-stream-impl@0.3.6 X @@ -168,7 +172,6 @@ include-flate@0.3.4 X include-flate-codegen@0.3.4 X include-flate-compress@0.3.4 X indexmap@2.14.0 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X itertools@0.14.0 X X @@ -230,6 +233,7 @@ no_std_io2@0.9.4 X X nom@7.1.3 X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-complex@0.4.6 X X num-conv@0.2.2 X X num-integer@0.1.46 X X @@ -247,8 +251,7 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X @@ -257,7 +260,7 @@ paimon-ftindex-core@0.1.0 X paimon-mosaic-core@0.2.0 X paimon-rest-server@0.6.0 X paimon-vindex-core@0.5.0 X -parquet@58.4.0 X +parquet@59.3.2 X paste@1.0.15 X X percent-encoding@2.3.2 X X phf@0.12.1 X @@ -381,7 +384,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X diff --git a/crates/paimon/Cargo.toml b/crates/paimon/Cargo.toml index 3bc4385cf..3bc041655 100644 --- a/crates/paimon/Cargo.toml +++ b/crates/paimon/Cargo.toml @@ -127,11 +127,14 @@ bzip2 = "0.6" liblzma = { version = "0.4", default-features = false, features = ["static"] } lzokay-native = { version = "0.1", default-features = false, features = ["decompress"] } arrow-array = { workspace = true } +# Mosaic and Vortex still expose Arrow 58; bridge their arrays through the C Data Interface. +arrow-array-58 = { package = "arrow-array", version = "58.4", features = ["ffi"] } +arrow-schema-58 = { package = "arrow-schema", version = "58.4" } arrow-arith = { workspace = true } arrow-buffer = { workspace = true } arrow-cast = { workspace = true } -arrow-json = "58.0" -arrow-ipc = "58.0" +arrow-json = "59.2" +arrow-ipc = "59.2" arrow-ord = { workspace = true } arrow-row = { workspace = true } arrow-schema = { workspace = true } @@ -141,7 +144,7 @@ futures = "0.3" crossbeam-channel = "0.5" tokio-util = { workspace = true, features = ["compat", "io-util"] } parquet = { workspace = true, features = ["async", "zstd", "lz4", "snap"] } -orc-rust = "0.8.0" +orc-rust = "0.9.0" async-stream = "0.3.6" reqwest = { version = "0.12", features = ["json", "gzip", "deflate"] } # Already in the tree via `reqwest`; direct dep for the object storage User-Agent transport. diff --git a/crates/paimon/DEPENDENCIES.rust.tsv b/crates/paimon/DEPENDENCIES.rust.tsv index 4080653ae..0630619e3 100644 --- a/crates/paimon/DEPENDENCIES.rust.tsv +++ b/crates/paimon/DEPENDENCIES.rust.tsv @@ -14,20 +14,29 @@ apache-avro@0.21.0 X approx@0.5.1 X arc-swap@1.9.2 X X arcref@0.2.0 X -arrow@58.4.0 X +arrow@59.3.2 X arrow-arith@58.4.0 X +arrow-arith@59.3.2 X arrow-array@58.4.0 X X +arrow-array@59.3.2 X X arrow-buffer@58.4.0 X +arrow-buffer@59.3.2 X arrow-cast@58.4.0 X -arrow-csv@58.4.0 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X arrow-ord@58.4.0 X -arrow-row@58.4.0 X +arrow-ord@59.3.2 X +arrow-row@59.3.2 X arrow-schema@58.4.0 X +arrow-schema@59.3.2 X arrow-select@58.4.0 X +arrow-select@59.3.2 X arrow-string@58.4.0 X +arrow-string@59.3.2 X async-channel@2.5.0 X X async-compression@0.4.42 X X async-executor@1.14.0 X X @@ -227,7 +236,6 @@ include-flate-compress@0.3.4 X indexmap@2.14.0 X X inout@0.1.4 X X inout@0.2.2 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X itertools@0.13.0 X X @@ -302,6 +310,7 @@ nougat@0.2.4 X X X nougat-proc_macros@0.2.4 X X X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-bigint-dig@0.8.6 X X num-complex@0.4.6 X X num-conv@0.2.2 X X @@ -332,8 +341,7 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X @@ -344,7 +352,7 @@ paimon-vindex-core@0.5.0 X parking@2.2.1 X X parking_lot@0.12.5 X X parking_lot_core@0.9.12 X X -parquet@58.4.0 X +parquet@59.3.2 X paste@1.0.15 X X pbkdf2@0.12.2 X X pco@1.0.2 X @@ -508,7 +516,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X diff --git a/crates/paimon/src/arrow/format/avro.rs b/crates/paimon/src/arrow/format/avro.rs index 2b593559c..6ada1f59d 100644 --- a/crates/paimon/src/arrow/format/avro.rs +++ b/crates/paimon/src/arrow/format/avro.rs @@ -410,15 +410,13 @@ fn build_column( DataType::Map(map_type) => build_map_column(records, name, map_type, num_rows)?, // Java encodes MULTISET as a map from the element to an INT count, // sharing the MAP path (`AvroSchemaConverter#extractValueTypeToAvroMap` - // returns IntType). Unlike MAP, `paimon_type_to_arrow` lets the key here - // follow the element's nullability and pins the count non-nullable. + // returns IntType). A list preserves NULL elements; counts are non-nullable. DataType::Multiset(multiset_type) => build_map_like_column( records, name, multiset_type.element_type(), &DataType::Int(IntType::new()), - multiset_type.element_type().is_nullable(), - false, + true, num_rows, )?, DataType::Row(row_type) => build_row_column(records, name, row_type, num_rows)?, @@ -589,36 +587,32 @@ fn build_map_column( name, map_type.key_type(), map_type.value_type(), - // `paimon_type_to_arrow` declares a MAP key non-nullable and lets the - // value follow its own type. Mirror that exactly or `RecordBatch::try_new` - // rejects the column. false, - map_type.value_type().is_nullable(), num_rows, ) } -/// Build a Map array from either Avro map encoding Paimon can produce. +/// Build MAP or MULTISET values from either Java Avro map encoding. /// /// Avro maps natively support string keys only, so Java writes a map with any /// other key type as an array of `{key, value}` records instead /// (`AvroSchemaConverter#isArrayMap`). Both shapes decode into the same Arrow -/// `Map` array here. +/// key/value entries here. /// /// The key and value types are passed separately rather than taken from a /// `MapType` so `MULTISET` can reuse this with an implicit `INT` count as the -/// value. The two nullability flags must match what `paimon_type_to_arrow` -/// declares for the same type -- MAP and MULTISET differ here -- or -/// `RecordBatch::try_new` rejects the column. +/// value. MULTISET uses a list because its elements can be NULL; ordinary MAP +/// keys remain non-nullable. fn build_map_like_column( records: &[Value], name: &str, key_type: &DataType, value_type: &DataType, - key_nullable: bool, - value_nullable: bool, + multiset: bool, num_rows: usize, ) -> crate::Result> { + let key_nullable = multiset && key_type.is_nullable(); + let value_nullable = !multiset && value_type.is_nullable(); let arrow_key_type = crate::arrow::paimon_type_to_arrow(key_type)?; let arrow_value_type = crate::arrow::paimon_type_to_arrow(value_type)?; @@ -702,6 +696,18 @@ fn build_map_like_column( .collect::>(), )); + if multiset { + let field = crate::arrow::multiset::entries_field( + crate::arrow::paimon_type_to_arrow(key_type)?, + key_nullable, + ); + let list = ListArray::try_new(field, offsets_buf, Arc::new(struct_arr), Some(nulls)) + .map_err(|e| Error::UnexpectedError { + message: format!("Failed to build MULTISET: {e}"), + source: Some(Box::new(e)), + })?; + return Ok(Arc::new(list)); + } let map_arr = MapArray::try_new( Arc::new(entries_field), offsets_buf, @@ -1119,6 +1125,7 @@ mod tests { ); let col = build_map_column(&records, "m", &map_type, 2).unwrap(); let arr = col.as_any().downcast_ref::().unwrap(); + assert!(!arr.entries().fields()[0].is_nullable()); assert_eq!(arr.len(), 2); // First row has 2 entries, second row is null. assert!(!arr.is_null(0)); @@ -1243,9 +1250,15 @@ mod tests { VarCharType::new(10).unwrap(), ))); let col = build_column(&records, "ms", &multiset, 1).unwrap(); - let arr = col.as_any().downcast_ref::().unwrap(); + let arr = col.as_any().downcast_ref::().unwrap(); + let entries = arr.values().as_any().downcast_ref::().unwrap(); + assert!(entries.fields()[0].is_nullable()); assert_eq!(arr.value(0).len(), 1); - let values = arr.values().as_any().downcast_ref::().unwrap(); + let values = entries + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); assert_eq!(values.value(0), 2); } @@ -1255,19 +1268,36 @@ mod tests { fn test_build_column_multiset_int_element() { let records = vec![Value::Record(vec![( "ms".to_string(), - Value::Array(vec![Value::Record(vec![ - ("key".to_string(), Value::Int(5)), - ("value".to_string(), Value::Int(3)), - ])]), + Value::Array(vec![ + Value::Record(vec![ + ("key".to_string(), Value::Int(5)), + ("value".to_string(), Value::Int(3)), + ]), + Value::Record(vec![ + ("key".to_string(), av_null()), + ("value".to_string(), Value::Int(2)), + ]), + ]), )])]; let multiset = DataType::Multiset(MultisetType::new(DataType::Int(IntType::new()))); let col = build_column(&records, "ms", &multiset, 1).unwrap(); - let arr = col.as_any().downcast_ref::().unwrap(); - assert_eq!(arr.value(0).len(), 1); - let keys = arr.keys().as_any().downcast_ref::().unwrap(); + let arr = col.as_any().downcast_ref::().unwrap(); + let entries = arr.values().as_any().downcast_ref::().unwrap(); + assert_eq!(arr.value(0).len(), 2); + let keys = entries + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); assert_eq!(keys.value(0), 5); - let values = arr.values().as_any().downcast_ref::().unwrap(); + assert!(keys.is_null(1)); + let values = entries + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); assert_eq!(values.value(0), 3); + assert_eq!(values.value(1), 2); } /// Going through `build_target_arrow_schema` is what catches a field-level diff --git a/crates/paimon/src/arrow/format/avro_write.rs b/crates/paimon/src/arrow/format/avro_write.rs index d52ef6006..ba46810f5 100644 --- a/crates/paimon/src/arrow/format/avro_write.rs +++ b/crates/paimon/src/arrow/format/avro_write.rs @@ -26,7 +26,7 @@ use apache_avro::{Codec, Schema, Writer}; use arrow_array::{ Array, BinaryArray, BooleanArray, Date32Array, Decimal128Array, FixedSizeListArray, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, LargeBinaryArray, - ListArray, MapArray, RecordBatch, StringArray, StructArray, Time32MillisecondArray, + ListArray, RecordBatch, StringArray, StructArray, Time32MillisecondArray, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampSecondArray, }; use arrow_schema::SchemaRef; @@ -447,16 +447,23 @@ fn nonnull_value_at(array: &dyn Array, row: usize, typ: &DataType) -> Result Result { - let map = downcast::(array)?; - let start = map.value_offsets()[row] as usize; - let end = map.value_offsets()[row + 1] as usize; + let structure = crate::arrow::multiset::entries(array, row)?; + let start = 0; + let end = structure.len(); if matches!(key, DataType::Char(_) | DataType::VarChar(_)) { let mut entries = HashMap::with_capacity(end - start); for index in start..end { - let map_key = downcast::(map.keys())?.value(index); + if structure.column(0).is_null(index) { + return Err(Error::DataInvalid { + message: "Avro string maps cannot contain NULL keys or multiset elements" + .into(), + source: None, + }); + } + let map_key = downcast::(structure.column(0))?.value(index); entries.insert( map_key.to_string(), - value_at(map.values().as_ref(), index, value)?, + value_at(structure.column(1).as_ref(), index, value)?, ); } Ok(Value::Map(entries)) @@ -464,10 +471,13 @@ fn map_value(array: &dyn Array, row: usize, key: &DataType, value: &DataType) -> let mut entries = Vec::with_capacity(end - start); for index in start..end { entries.push(Value::Record(vec![ - ("key".into(), value_at(map.keys(), index, key)?), + ( + "key".into(), + value_at(structure.column(0).as_ref(), index, key)?, + ), ( "value".into(), - value_at(map.values().as_ref(), index, value)?, + value_at(structure.column(1).as_ref(), index, value)?, ), ])); } @@ -530,6 +540,71 @@ mod tests { VectorType, }; + #[tokio::test] + async fn multiset_preserves_null_elements_and_counts_in_avro() { + let fields = vec![DataField::new( + 0, + "bag".into(), + DataType::Multiset(crate::spec::MultisetType::new( + DataType::Int(IntType::new()), + )), + )]; + let schema = build_target_arrow_schema(&fields).unwrap(); + let arrow_schema::DataType::List(child) = schema.field(0).data_type() else { + unreachable!() + }; + let arrow_schema::DataType::Struct(entry_fields) = child.data_type() else { + unreachable!() + }; + let entries = StructArray::try_new( + entry_fields.clone(), + vec![ + Arc::new(Int32Array::from(vec![None, Some(5)])), + Arc::new(Int32Array::from(vec![3, 2])), + ], + None, + ) + .unwrap(); + let bag = ListArray::try_new( + child.clone(), + arrow_buffer::OffsetBuffer::new(vec![0, 2, 2, 2].into()), + Arc::new(entries), + Some(NullBuffer::from(vec![true, false, true])), + ) + .unwrap(); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(bag)]).unwrap(); + let io = FileIOBuilder::new("memory").build().unwrap(); + let path = "memory:/multiset-null.avro"; + let mut writer = AvroFormatWriter::new( + &io.new_output(path).unwrap(), + schema, + fields.clone(), + "none", + 1, + None, + ) + .await + .unwrap(); + writer.write(&batch).await.unwrap(); + Box::new(writer).close().await.unwrap(); + let bytes = io.new_input(path).unwrap().read().await.unwrap(); + let decoded = AvroFormatReader + .read_batch_stream( + Box::new(BytesFileRead(bytes.clone())), + bytes.len() as u64, + &fields, + None, + None, + None, + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(decoded, vec![batch]); + } + fn fields() -> Vec { vec![ DataField::new(0, "id".into(), DataType::BigInt(BigIntType::new())), diff --git a/crates/paimon/src/arrow/format/mosaic.rs b/crates/paimon/src/arrow/format/mosaic.rs index f84ee8730..136c27d85 100644 --- a/crates/paimon/src/arrow/format/mosaic.rs +++ b/crates/paimon/src/arrow/format/mosaic.rs @@ -511,6 +511,7 @@ fn read_planned_row_group( .row_group_reader_by_names(plan.index, &names) .map_err(mosaic_read_error)?; let batch = row_group_reader.read_columns().map_err(mosaic_read_error)?; + let batch = crate::arrow::legacy::from_legacy_batch(&batch)?; take_row_slices(batch, plan.selected_slices.as_deref(), read_schema) } @@ -818,7 +819,12 @@ pub(super) fn validate_mosaic_schema(schema: &SchemaRef) -> crate::Result<()> { } fn validate_mosaic_arrow_type(data_type: &ArrowDataType) -> Result<(), String> { - paimon_mosaic_core::types::validate_data_type(data_type) + if crate::arrow::multiset::contains_multiset(data_type) { + return Err("MULTISET is not supported by Mosaic".into()); + } + let data_type = + crate::arrow::legacy::to_legacy_data_type(data_type).map_err(|error| error.to_string())?; + paimon_mosaic_core::types::validate_data_type(&data_type) } fn selected_slices_for_row_group( @@ -1219,7 +1225,7 @@ mod tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - batches[0].schema().as_ref(), + &crate::arrow::legacy::to_legacy_schema(batches[0].schema().as_ref()).unwrap(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -1230,7 +1236,9 @@ mod tests { ) .unwrap(); for batch in batches { - writer.write_batch(batch).unwrap(); + writer + .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) + .unwrap(); } writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) @@ -1434,7 +1442,7 @@ mod tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - batches[0].schema().as_ref(), + &crate::arrow::legacy::to_legacy_schema(batches[0].schema().as_ref()).unwrap(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 1, @@ -1445,7 +1453,9 @@ mod tests { ) .unwrap(); for batch in batches { - writer.write_batch(batch).unwrap(); + writer + .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) + .unwrap(); } writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) diff --git a/crates/paimon/src/arrow/format/mosaic_write.rs b/crates/paimon/src/arrow/format/mosaic_write.rs index 23b6267e7..9a026da08 100644 --- a/crates/paimon/src/arrow/format/mosaic_write.rs +++ b/crates/paimon/src/arrow/format/mosaic_write.rs @@ -166,7 +166,8 @@ impl MosaicFormatWriter { }) }) .collect::>>()?; - let encoder = MosaicWriter::new(PendingOutput::default(), &physical_schema, options) + let legacy_schema = crate::arrow::legacy::to_legacy_schema(&physical_schema)?; + let encoder = MosaicWriter::new(PendingOutput::default(), &legacy_schema, options) .map_err(mosaic_write_error)?; Ok(Self { output: output.writer().await?, @@ -330,7 +331,7 @@ impl FormatFileWriter for MosaicFormatWriter { }; let groups_before = self.encoder.num_row_groups(); self.encoder - .write_batch(&batch) + .write_batch(&crate::arrow::legacy::to_legacy_batch(&batch)?) .map_err(mosaic_write_error)?; self.pending_rows = if self.encoder.num_row_groups() > groups_before { 0 diff --git a/crates/paimon/src/arrow/format/parquet.rs b/crates/paimon/src/arrow/format/parquet.rs index 0ade1fbf6..988e75637 100644 --- a/crates/paimon/src/arrow/format/parquet.rs +++ b/crates/paimon/src/arrow/format/parquet.rs @@ -35,7 +35,6 @@ use crate::spec::{ use crate::table::{ArrowRecordBatchStream, RowRange}; use crate::Error; use arrow_array::{BooleanArray, RecordBatch}; -use arrow_cast::cast; use async_trait::async_trait; use bytes::Bytes; use futures::future::BoxFuture; @@ -374,7 +373,8 @@ impl ParquetFormatWriter { let max_row_group_bytes = props.max_row_group_bytes(); let async_write = output.async_writer().await?; let input_schema = schema; - let schema = timestamp_millis_schema(&input_schema); + let schema = + crate::arrow::multiset::parquet_schema(×tamp_millis_schema(&input_schema)); let inner = create_parquet_arrow_writer(async_write, schema.clone(), props)?; let stats_modes = write_fields .map(|fields| core_options.metadata_stats_modes(fields.iter().map(DataField::name))) @@ -792,15 +792,17 @@ fn cast_record_batch_to_schema( if column.data_type() == field.data_type() { Ok(column.clone()) } else { - cast(column, field.data_type()).map_err(|error| crate::Error::DataInvalid { - message: format!( - "Failed to convert Parquet column '{}' from {:?} to {:?}: {error}", - field.name(), - column.data_type(), - field.data_type() - ), - source: Some(Box::new(error)), - }) + crate::arrow::multiset::to_parquet_array(column, field.data_type()).map_err( + |error| crate::Error::DataInvalid { + message: format!( + "Failed to convert Parquet column '{}' from {:?} to {:?}: {error}", + field.name(), + column.data_type(), + field.data_type() + ), + source: Some(Box::new(error)), + }, + ) } }) .collect::>>()?; @@ -5088,10 +5090,10 @@ mod tests { assert_eq!(column.physical_type(), parquet::basic::Type::INT64); assert_eq!( column.logical_type_ref(), - Some(&parquet::basic::LogicalType::Timestamp { - is_adjusted_to_u_t_c: adjusted_to_utc, - unit: parquet::basic::TimeUnit::MILLIS, - }) + Some(&parquet::basic::LogicalType::timestamp( + adjusted_to_utc, + parquet::basic::TimeUnit::MILLIS, + )) ); let ParquetStatistics::Int64(stats) = metadata.row_group(0).column(index).statistics().unwrap() diff --git a/crates/paimon/src/arrow/format/row.rs b/crates/paimon/src/arrow/format/row.rs index 3294fdf68..1d00ef08c 100644 --- a/crates/paimon/src/arrow/format/row.rs +++ b/crates/paimon/src/arrow/format/row.rs @@ -550,7 +550,8 @@ fn validate_arrow_type_for_row_field( )?; true } - (ArrowDataType::Map(entries, _), DataType::Multiset(m)) => { + (ArrowDataType::List(entries), DataType::Multiset(m)) + | (ArrowDataType::Map(entries, _), DataType::Multiset(m)) => { validate_arrow_map_entries( field_name, entries.data_type(), @@ -828,9 +829,15 @@ fn write_field_value( write_map_value(out, map, row_idx, m.key_type(), m.value_type())?; } DataType::Multiset(m) => { - let map = downcast::(array, data_type)?; - let count_type = DataType::Int(IntType::new()); - write_map_value(out, map, row_idx, m.element_type(), &count_type)?; + let entries = crate::arrow::multiset::entries(array, row_idx)?; + write_array_slice(out, entries.column(0), 0, entries.len(), m.element_type())?; + write_array_slice( + out, + entries.column(1), + 0, + entries.len(), + &DataType::Int(IntType::new()), + )?; } DataType::Row(r) => { let row = downcast::(array, data_type)?; @@ -1322,6 +1329,7 @@ enum ColumnBuilder { values: Box, }, Map { + multiset: bool, entries_field: Arc, offsets: Vec, validities: Vec, @@ -1391,6 +1399,7 @@ impl ColumnBuilder { values: Box::new(ColumnBuilder::new(a.element_type(), capacity)?), }, DataType::Map(m) => Self::Map { + multiset: false, entries_field: map_entries_field( m.key_type(), m.value_type(), @@ -1403,22 +1412,21 @@ impl ColumnBuilder { values: Box::new(ColumnBuilder::new(m.value_type(), capacity)?), sorted: false, }, - DataType::Multiset(m) => { - let count_type = DataType::Int(IntType::new()); - Self::Map { - entries_field: map_entries_field( - m.element_type(), - &count_type, - m.element_type().is_nullable(), - false, - )?, - offsets: vec![0], - validities: Vec::with_capacity(capacity), - keys: Box::new(ColumnBuilder::new(m.element_type(), capacity)?), - values: Box::new(ColumnBuilder::new(&count_type, capacity)?), - sorted: false, - } - } + DataType::Multiset(m) => Self::Map { + multiset: true, + entries_field: crate::arrow::multiset::entries_field( + paimon_type_to_arrow(m.element_type())?, + m.element_type().is_nullable(), + ), + offsets: vec![0], + validities: Vec::with_capacity(capacity), + keys: Box::new(ColumnBuilder::new(m.element_type(), capacity)?), + values: Box::new(ColumnBuilder::new( + &DataType::Int(IntType::new()), + capacity, + )?), + sorted: false, + }, DataType::Row(r) => Self::Row { fields: arrow_fields_for_row(r.fields())?, columns: r @@ -1720,6 +1728,7 @@ impl ColumnBuilder { })?, ), Self::Map { + multiset, entries_field, offsets, validities, @@ -1747,19 +1756,34 @@ impl ColumnBuilder { message: format!("Failed to build .row Map entries: {e}"), source: Some(Box::new(e)), })?; - Arc::new( - MapArray::try_new( - entries_field, - OffsetBuffer::new(ScalarBuffer::from(offsets)), - entries, - Some(null_buffer(validities)), - sorted, + if multiset { + Arc::new( + ListArray::try_new( + entries_field, + OffsetBuffer::new(ScalarBuffer::from(offsets)), + Arc::new(entries), + Some(null_buffer(validities)), + ) + .map_err(|e| Error::UnexpectedError { + message: format!("Failed to build .row MULTISET: {e}"), + source: Some(Box::new(e)), + })?, ) - .map_err(|e| Error::UnexpectedError { - message: format!("Failed to build .row MapArray: {e}"), - source: Some(Box::new(e)), - })?, - ) + } else { + Arc::new( + MapArray::try_new( + entries_field, + OffsetBuffer::new(ScalarBuffer::from(offsets)), + entries, + Some(null_buffer(validities)), + sorted, + ) + .map_err(|e| Error::UnexpectedError { + message: format!("Failed to build .row MapArray: {e}"), + source: Some(Box::new(e)), + })?, + ) + } } Self::Row { fields, @@ -3452,11 +3476,30 @@ mod tests { let bag = test_map_array( vec![0, 2, 2, 3], vec![true, false, true], - true, + false, false, vec![Some("x"), Some("y"), Some("z")], vec![Some(2), Some(1), Some(4)], ); + let ArrowDataType::List(bag_field) = schema.field(2).data_type() else { + unreachable!() + }; + let bag_entries = StructArray::try_new( + match bag_field.data_type() { + ArrowDataType::Struct(fields) => fields.clone(), + _ => unreachable!(), + }, + bag.entries().columns().to_vec(), + None, + ) + .unwrap(); + let bag = ListArray::try_new( + bag_field.clone(), + bag.offsets().clone(), + Arc::new(bag_entries), + bag.nulls().cloned(), + ) + .unwrap(); let nested = StructArray::try_new( vec![ @@ -3542,11 +3585,24 @@ mod tests { assert!(label_values.is_null(1)); assert_eq!(label_values.value(2), 30); - let bag = batch.column(2).as_any().downcast_ref::().unwrap(); + let bag = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); assert_eq!(bag.value_offsets(), &[0, 2, 2, 3]); assert!(bag.is_null(1)); - let bag_keys = bag.keys().as_any().downcast_ref::().unwrap(); - let bag_counts = bag.values().as_any().downcast_ref::().unwrap(); + let entries = bag.values().as_any().downcast_ref::().unwrap(); + let bag_keys = entries + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let bag_counts = entries + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); assert_eq!(bag_keys.value(0), "x"); assert_eq!(bag_keys.value(2), "z"); assert_eq!(bag_counts.value(0), 2); @@ -3576,6 +3632,45 @@ mod tests { assert_eq!(names.value(2), "nine"); } + #[test] + fn decode_java_multiset_with_null_element() { + let fields = vec![DataField::new( + 0, + "bag".into(), + DataType::Multiset(MultisetType::new(DataType::Int(IntType::new()))), + )]; + // Java RowBlockWriter: one non-null bag, key array [NULL], counts [3], + // row offset 0, and row count 1. No Arrow writer participates here. + let block = [0, 1, 1, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0]; + let batch = decode_row_block( + &block, + &fields, + build_target_arrow_schema(&fields).unwrap(), + &[0], + ) + .unwrap(); + let mut writer = RowBlockWriter::new(); + writer.write_row(&batch, 0, &fields).unwrap(); + assert_eq!(writer.finish(), block); + let bag = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let entries = bag.value(0); + let entries = entries.as_any().downcast_ref::().unwrap(); + assert!(entries.column(0).is_null(0)); + assert_eq!( + entries + .column(1) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 3 + ); + } + #[test] fn blob_data_block_matches_java_materialized_descriptor() { use crate::spec::BlobType; diff --git a/crates/paimon/src/arrow/format/vortex.rs b/crates/paimon/src/arrow/format/vortex.rs index 92fb7bf95..9ad65e852 100644 --- a/crates/paimon/src/arrow/format/vortex.rs +++ b/crates/paimon/src/arrow/format/vortex.rs @@ -24,7 +24,7 @@ use crate::spec::{DataField, Predicate}; use crate::table::{ArrowRecordBatchStream, RowRange}; use crate::Error; use arrow_array::RecordBatch; -use arrow_schema::{DataType as ArrowDataType, Field, SchemaRef}; +use arrow_schema::{DataType as ArrowDataType, SchemaRef}; use async_trait::async_trait; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, OnceLock}; @@ -424,9 +424,9 @@ fn vortex_array_to_record_batch( vortex_array: ArrayRef, schema: &SchemaRef, ) -> crate::Result { - let target_field = Field::new( + let target_field = arrow_schema_58::Field::new( "", - ArrowDataType::Struct(schema.fields().clone()), + crate::arrow::legacy::to_legacy_data_type(&ArrowDataType::Struct(schema.fields().clone()))?, vortex_array.dtype().is_nullable(), ); let mut ctx = session.create_execution_ctx(); @@ -438,6 +438,7 @@ fn vortex_array_to_record_batch( source: None, })?; + let arrow_array = crate::arrow::legacy::from_legacy_array(&arrow_array)?; let struct_array = arrow_array .as_any() .downcast_ref::() @@ -490,7 +491,7 @@ pub(crate) struct VortexFormatWriter { impl VortexFormatWriter { pub(crate) async fn new(output: &OutputFile, schema: SchemaRef) -> crate::Result { validate_vortex_schema(&schema)?; - let dtype = DType::from_arrow(schema); + let dtype = DType::from_arrow(Arc::new(crate::arrow::legacy::to_legacy_schema(&schema)?)); let bytes_written = Arc::new(AtomicU64::new(0)); Ok(Self { @@ -504,9 +505,8 @@ impl VortexFormatWriter { } /// `DType::from_arrow` ends in `unimplemented!` for a type it has no arm for, so screen -/// the schema first. `Map` is the only such type Paimon builds — it is what -/// `paimon_type_to_arrow` emits for both `MAP` and `MULTISET` — and it can sit at any -/// depth, so the walk descends the same containers `DType::from_arrow` does. +/// the schema first. MAP and MULTISET remain unsupported at any depth, so the +/// walk also checks the metadata marking the Arrow list used for MULTISET. fn validate_vortex_schema(schema: &SchemaRef) -> crate::Result<()> { for field in schema.fields() { if contains_unconvertible_type(field.data_type()) { @@ -525,6 +525,7 @@ fn validate_vortex_schema(schema: &SchemaRef) -> crate::Result<()> { fn contains_unconvertible_type(data_type: &ArrowDataType) -> bool { match data_type { ArrowDataType::Map(..) => true, + ArrowDataType::List(field) if crate::arrow::multiset::is_multiset(field) => true, ArrowDataType::List(field) | ArrowDataType::LargeList(field) | ArrowDataType::ListView(field) @@ -543,8 +544,8 @@ fn contains_unconvertible_type(data_type: &ArrowDataType) -> bool { impl FormatFileWriter for VortexFormatWriter { async fn write(&mut self, batch: &RecordBatch) -> crate::Result<()> { let staged_bytes = batch.get_array_memory_size(); - let vortex_arr = - ArrayRef::from_arrow(batch.clone(), false).map_err(|e| Error::DataInvalid { + let vortex_arr = ArrayRef::from_arrow(crate::arrow::legacy::to_legacy_batch(batch)?, false) + .map_err(|e| Error::DataInvalid { message: format!("Failed to convert RecordBatch to Vortex: {e}"), source: None, })?; @@ -1648,7 +1649,7 @@ mod tests { panic!("expected Unsupported, got {err:?}"); }; assert!(message.contains("'nested'"), "message: {message}"); - assert!(message.contains("Map"), "message: {message}"); + assert!(message.contains("List"), "message: {message}"); } /// Positive control: a nested ROW of primitives must still be accepted, so the walk diff --git a/crates/paimon/src/arrow/legacy.rs b/crates/paimon/src/arrow/legacy.rs new file mode 100644 index 000000000..6ff2d18e9 --- /dev/null +++ b/crates/paimon/src/arrow/legacy.rs @@ -0,0 +1,189 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Zero-copy C Data Interface bridge for Mosaic and Vortex's Arrow 58 APIs. +//! Release callbacks keep ownership of buffers in the exporting Arrow version. + +use std::sync::Arc; + +use arrow_array::{Array, ArrayRef, RecordBatch, RecordBatchOptions}; +use arrow_schema::{DataType, Schema}; + +use crate::{Error, Result}; + +fn bridge_error(source: impl std::error::Error + Send + Sync + 'static) -> Error { + Error::DataInvalid { + message: "Cannot bridge Arrow 58 and Arrow 59 data".into(), + source: Some(Box::new(source)), + } +} + +pub(crate) fn to_legacy_schema(schema: &Schema) -> Result { + let schema = arrow_schema::ffi::FFI_ArrowSchema::try_from(schema).map_err(bridge_error)?; + // SAFETY: Both versions implement the same repr(C) ArrowSchema ABI. The + // exporter remains alive while the importer copies the schema and metadata. + let schema = + unsafe { &*std::ptr::from_ref(&schema).cast::() }; + arrow_schema_58::Schema::try_from(schema).map_err(bridge_error) +} + +fn from_legacy_schema(schema: &arrow_schema_58::Schema) -> Result { + let schema = arrow_schema_58::ffi::FFI_ArrowSchema::try_from(schema).map_err(bridge_error)?; + // SAFETY: ArrowSchema has the same C ABI in both versions, and the exported + // schema outlives this import, which copies its fields and metadata. + let schema = + unsafe { &*std::ptr::from_ref(&schema).cast::() }; + Schema::try_from(schema).map_err(bridge_error) +} + +pub(crate) fn to_legacy_data_type(data_type: &DataType) -> Result { + let schema = Schema::new(vec![arrow_schema::Field::new("", data_type.clone(), true)]); + Ok(to_legacy_schema(&schema)?.field(0).data_type().clone()) +} + +fn to_legacy_array(array: &ArrayRef) -> Result { + let (mut array, schema) = arrow_array::ffi::to_ffi(&array.to_data()).map_err(bridge_error)?; + // SAFETY: The C Data Interface structs share a stable ABI. from_raw moves + // ownership and marks the source array released, preventing a double drop. + // from_ffi retains the exporter's release callback until all buffers die. + let data = unsafe { + let array = + arrow_array_58::ffi::FFI_ArrowArray::from_raw(std::ptr::from_mut(&mut array).cast()); + let schema = &*std::ptr::from_ref(&schema).cast(); + arrow_array_58::ffi::from_ffi(array, schema) + } + .map_err(bridge_error)?; + Ok(arrow_array_58::make_array(data)) +} + +pub(crate) fn from_legacy_array(array: &arrow_array_58::ArrayRef) -> Result { + let (mut array, schema) = + arrow_array_58::ffi::to_ffi(&array.to_data()).map_err(bridge_error)?; + // SAFETY: This is the inverse of to_legacy_array: the shared C ABI moves + // ownership via from_raw, and imported buffers retain the release callback. + let data = unsafe { + let array = + arrow_array::ffi::FFI_ArrowArray::from_raw(std::ptr::from_mut(&mut array).cast()); + let schema = &*std::ptr::from_ref(&schema).cast(); + arrow_array::ffi::from_ffi(array, schema) + } + .map_err(bridge_error)?; + Ok(arrow_array::make_array(data)) +} + +pub(crate) fn to_legacy_batch(batch: &RecordBatch) -> Result { + let schema = Arc::new(to_legacy_schema(batch.schema().as_ref())?); + let columns = batch + .columns() + .iter() + .map(to_legacy_array) + .collect::>()?; + arrow_array_58::RecordBatch::try_new_with_options( + schema, + columns, + &arrow_array_58::RecordBatchOptions::new().with_row_count(Some(batch.num_rows())), + ) + .map_err(bridge_error) +} + +pub(crate) fn from_legacy_batch(batch: &arrow_array_58::RecordBatch) -> Result { + let schema = Arc::new(from_legacy_schema(batch.schema().as_ref())?); + let columns = batch + .columns() + .iter() + .map(from_legacy_array) + .collect::>()?; + RecordBatch::try_new_with_options( + schema, + columns, + &RecordBatchOptions::new().with_row_count(Some(batch.num_rows())), + ) + .map_err(bridge_error) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_array::{types::Int32Type, Int32Array, ListArray, StringArray}; + use arrow_schema::Field; + + #[test] + fn round_trip_preserves_slices_nulls_nested_values_metadata_and_buffer_ownership() { + let ids = Arc::new(Int32Array::from(vec![Some(0), Some(1), None, Some(3)])); + let names = Arc::new(StringArray::from(vec![ + Some("skip"), + Some("one"), + None, + Some("三"), + ])); + let lists = Arc::new(ListArray::from_iter_primitive::(vec![ + Some(vec![Some(0)]), + Some(vec![Some(1), None]), + None, + Some(vec![]), + ])); + let schema = Arc::new(Schema::new_with_metadata( + vec![ + Field::new("id", DataType::Int32, true) + .with_metadata([("field_id".into(), "42".into())].into()), + Field::new("name", DataType::Utf8, true), + Field::new("values", lists.data_type().clone(), true), + ], + [("table".into(), "bridge".into())].into(), + )); + let original = RecordBatch::try_new(schema, vec![ids, names, lists]) + .unwrap() + .slice(1, 3); + let expected = original.clone(); + let legacy = to_legacy_batch(&original).unwrap(); + drop(original); + let imported = from_legacy_batch(&legacy).unwrap(); + drop(legacy); + assert_eq!(imported, expected); + let ids = imported + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let original_ids = expected + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(ids.values().as_ptr(), original_ids.values().as_ptr()); + drop(expected); + assert_eq!(ids.value(0), 1); + assert!(ids.is_null(1)); + assert_eq!(ids.value(2), 3); + } + + #[test] + fn round_trip_preserves_row_counts_without_columns() { + for rows in [0, 7] { + let batch = RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(rows)), + ) + .unwrap(); + assert_eq!( + from_legacy_batch(&to_legacy_batch(&batch).unwrap()).unwrap(), + batch + ); + } + } +} diff --git a/crates/paimon/src/arrow/mod.rs b/crates/paimon/src/arrow/mod.rs index fddf9f193..fc353c6c4 100644 --- a/crates/paimon/src/arrow/mod.rs +++ b/crates/paimon/src/arrow/mod.rs @@ -16,6 +16,8 @@ // under the License. mod date_extract; +pub(crate) mod legacy; +pub(crate) mod multiset; pub(crate) use date_extract::evaluate_date_extract; mod cast_width; pub(crate) use cast_width::apply_declared_width; @@ -118,23 +120,10 @@ pub fn paimon_type_to_arrow(dt: &PaimonDataType) -> crate::Result false, ) } - PaimonDataType::Multiset(m) => { - let element_type = paimon_type_to_arrow(m.element_type())?; - ArrowDataType::Map( - Arc::new(ArrowField::new( - "entries", - ArrowDataType::Struct( - vec![ - ArrowField::new("key", element_type, m.element_type().is_nullable()), - ArrowField::new("value", ArrowDataType::Int32, false), - ] - .into(), - ), - false, - )), - false, - ) - } + PaimonDataType::Multiset(m) => ArrowDataType::List(multiset::entries_field( + paimon_type_to_arrow(m.element_type())?, + m.element_type().is_nullable(), + )), PaimonDataType::Row(r) => { let fields: Vec = r .fields() @@ -240,6 +229,22 @@ pub fn arrow_to_paimon_type( ArrowDataType::Decimal128(p, s) => Ok(PaimonDataType::Decimal(DecimalType::with_nullable( nullable, *p as u32, *s as u32, )?)), + ArrowDataType::List(field) if multiset::is_multiset(field) => { + let ArrowDataType::Struct(entries) = field.data_type() else { + return Err(crate::Error::Unsupported { + message: "Invalid MULTISET entries".into(), + }); + }; + if entries.len() != 2 || entries[1].data_type() != &ArrowDataType::Int32 { + return Err(crate::Error::Unsupported { + message: "Invalid MULTISET element/count fields".into(), + }); + } + let element = arrow_to_paimon_type(entries[0].data_type(), entries[0].is_nullable())?; + Ok(PaimonDataType::Multiset( + crate::spec::MultisetType::with_nullable(nullable, element), + )) + } ArrowDataType::List(field) | ArrowDataType::LargeList(field) => { let element = arrow_to_paimon_type(field.data_type(), field.is_nullable())?; Ok(PaimonDataType::Array(ArrayType::with_nullable( @@ -427,24 +432,10 @@ pub(crate) fn paimon_type_to_arrow_for_read(dt: &PaimonDataType) -> crate::Resul )), false, ), - PaimonDataType::Multiset(multiset) => ArrowDataType::Map( - Arc::new(ArrowField::new( - "entries", - ArrowDataType::Struct( - vec![ - ArrowField::new( - "key", - paimon_type_to_arrow_for_read(multiset.element_type())?, - multiset.element_type().is_nullable(), - ), - ArrowField::new("value", ArrowDataType::Int32, false), - ] - .into(), - ), - false, - )), - false, - ), + PaimonDataType::Multiset(multiset) => ArrowDataType::List(self::multiset::entries_field( + paimon_type_to_arrow_for_read(multiset.element_type())?, + true, + )), _ => paimon_type_to_arrow(dt)?, }) } @@ -556,6 +547,34 @@ mod tests { } } + #[test] + fn multiset_schemas_preserve_nullable_elements() { + let fields = vec![DataField::new( + 0, + "bag".into(), + PaimonDataType::Multiset(crate::spec::MultisetType::new(PaimonDataType::VarChar( + VarCharType::string_type(), + ))), + )]; + for schema in [ + build_target_arrow_schema(&fields).unwrap(), + build_read_arrow_schema(&fields).unwrap(), + ] { + let ArrowDataType::List(entries) = schema.field(0).data_type() else { + panic!("multiset must use Arrow list storage"); + }; + let ArrowDataType::Struct(fields) = entries.data_type() else { + panic!("map entries must be a struct"); + }; + assert!(fields[0].is_nullable()); + assert!(!fields[1].is_nullable()); + assert!(matches!( + arrow_to_paimon_type(schema.field(0).data_type(), true).unwrap(), + PaimonDataType::Multiset(_) + )); + } + } + #[test] fn test_dictionary_resolves_to_value_type() { // A dictionary-encoded column carries the logical type of its values; diff --git a/crates/paimon/src/arrow/multiset.rs b/crates/paimon/src/arrow/multiset.rs new file mode 100644 index 000000000..c9f810955 --- /dev/null +++ b/crates/paimon/src/arrow/multiset.rs @@ -0,0 +1,446 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A MULTISET permits NULL elements, whereas Arrow Map keys cannot be NULL. +//! Use a list of element/count entries and mark it to distinguish ARRAY. + +use std::sync::Arc; + +use arrow_array::{ + Array, ArrayRef, FixedSizeListArray, LargeListArray, ListArray, MapArray, StructArray, +}; +use arrow_schema::{DataType, Field, FieldRef, Schema, SchemaRef}; + +use crate::{Error, Result}; + +const MULTISET_KEY: &str = "paimon.multiset"; + +pub(crate) fn entries_field(element: DataType, nullable: bool) -> FieldRef { + Arc::new( + Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("key", element, nullable), + Field::new("value", DataType::Int32, false), + ] + .into(), + ), + false, + ) + .with_metadata([(MULTISET_KEY.into(), "true".into())].into()), + ) +} + +pub(crate) fn is_multiset(field: &Field) -> bool { + field + .metadata() + .get(MULTISET_KEY) + .is_some_and(|value| value == "true") +} + +pub(crate) fn contains_multiset(data_type: &DataType) -> bool { + match data_type { + DataType::List(field) if is_multiset(field) => true, + DataType::List(field) + | DataType::LargeList(field) + | DataType::ListView(field) + | DataType::LargeListView(field) + | DataType::FixedSizeList(field, _) + | DataType::Map(field, _) + | DataType::RunEndEncoded(_, field) => contains_multiset(field.data_type()), + DataType::Struct(fields) => fields + .iter() + .any(|field| contains_multiset(field.data_type())), + DataType::Union(fields, _) => fields + .iter() + .any(|(_, field)| contains_multiset(field.data_type())), + DataType::Dictionary(_, values) => contains_multiset(values), + _ => false, + } +} + +pub(crate) fn entries(array: &dyn Array, row: usize) -> Result { + let entries = if let Some(map) = array.as_any().downcast_ref::() { + Some(map.value(row)) + } else if let Some(list) = array.as_any().downcast_ref::() { + list.value(row) + .as_any() + .downcast_ref::() + .cloned() + } else { + None + }; + entries + .filter(|entries| entries.num_columns() == 2 && entries.null_count() == 0) + .ok_or_else(|| Error::DataInvalid { + message: "MULTISET must contain non-null element/count structs with exactly two fields" + .into(), + source: None, + }) +} + +/// Java Parquet represents MULTISET as MAP and rejects NULL elements. Preserve +/// that disk layout while keeping nullable elements representable elsewhere. +pub(crate) fn parquet_schema(schema: &SchemaRef) -> SchemaRef { + Arc::new(Schema::new_with_metadata( + schema + .fields() + .iter() + .map(parquet_field) + .collect::>(), + schema.metadata().clone(), + )) +} + +fn parquet_field(field: &FieldRef) -> FieldRef { + let data_type = match field.data_type() { + DataType::List(child) if is_multiset(child) => { + let DataType::Struct(fields) = child.data_type() else { + unreachable!("MULTISET entries are structs") + }; + let fields = vec![ + Arc::new( + parquet_field(&fields[0]) + .as_ref() + .clone() + .with_nullable(false), + ), + parquet_field(&fields[1]), + ]; + DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct(fields.into()), + false, + )), + false, + ) + } + DataType::List(child) => DataType::List(parquet_field(child)), + DataType::LargeList(child) => DataType::LargeList(parquet_field(child)), + DataType::FixedSizeList(child, size) => { + DataType::FixedSizeList(parquet_field(child), *size) + } + DataType::Map(child, sorted) => DataType::Map(parquet_field(child), *sorted), + DataType::Struct(fields) => { + DataType::Struct(fields.iter().map(parquet_field).collect::>().into()) + } + data_type => data_type.clone(), + }; + Arc::new(field.as_ref().clone().with_data_type(data_type)) +} + +/// Convert logical MULTISET lists to the Parquet MAP layout, descending through +/// nested containers. Scalar conversion retains the existing timestamp casts. +pub(crate) fn to_parquet_array(array: &ArrayRef, target: &DataType) -> Result { + // NULL parents and sliced-away entries have no logical keys. Remove their + // hidden children before Parquet's non-null key validation. + let array = crate::table::normalize_write_array(array, array.data_type())?; + convert_parquet_array(&array, target) +} + +fn convert_parquet_array(array: &ArrayRef, target: &DataType) -> Result { + if array.data_type() == target { + return Ok(array.clone()); + } + let converted: ArrayRef = match (array.data_type(), target) { + (DataType::List(child), DataType::Map(entries, sorted)) if is_multiset(child) => { + let list = array.as_any().downcast_ref::().unwrap(); + let values = convert_parquet_array(list.values(), entries.data_type())?; + let values = values.as_any().downcast_ref::().unwrap(); + Arc::new( + MapArray::try_new( + entries.clone(), + list.offsets().clone(), + values.clone(), + list.nulls().cloned(), + *sorted, + ) + .map_err(storage_error)?, + ) + } + (DataType::List(_), DataType::List(child)) => { + let list = array.as_any().downcast_ref::().unwrap(); + Arc::new( + ListArray::try_new( + child.clone(), + list.offsets().clone(), + convert_parquet_array(list.values(), child.data_type())?, + list.nulls().cloned(), + ) + .map_err(storage_error)?, + ) + } + (DataType::LargeList(_), DataType::LargeList(child)) => { + let list = array.as_any().downcast_ref::().unwrap(); + Arc::new( + LargeListArray::try_new( + child.clone(), + list.offsets().clone(), + convert_parquet_array(list.values(), child.data_type())?, + list.nulls().cloned(), + ) + .map_err(storage_error)?, + ) + } + (DataType::FixedSizeList(_, size), DataType::FixedSizeList(child, _)) => { + let list = array.as_any().downcast_ref::().unwrap(); + Arc::new( + FixedSizeListArray::try_new_with_length( + child.clone(), + *size, + convert_parquet_array(list.values(), child.data_type())?, + list.nulls().cloned(), + list.len(), + ) + .map_err(storage_error)?, + ) + } + (DataType::Struct(_), DataType::Struct(fields)) => { + let structure = array.as_any().downcast_ref::().unwrap(); + let columns = structure + .columns() + .iter() + .zip(fields) + .map(|(column, field)| convert_parquet_array(column, field.data_type())) + .collect::>>()?; + Arc::new( + StructArray::try_new_with_length( + fields.clone(), + columns, + structure.nulls().cloned(), + structure.len(), + ) + .map_err(storage_error)?, + ) + } + (DataType::Map(_, _), DataType::Map(entries, sorted)) => { + let map = array.as_any().downcast_ref::().unwrap(); + let values: ArrayRef = Arc::new(map.entries().clone()); + let values = convert_parquet_array(&values, entries.data_type())?; + Arc::new( + MapArray::try_new( + entries.clone(), + map.offsets().clone(), + values + .as_any() + .downcast_ref::() + .unwrap() + .clone(), + map.nulls().cloned(), + *sorted, + ) + .map_err(storage_error)?, + ) + } + _ => arrow_cast::cast(array, target).map_err(storage_error)?, + }; + Ok(converted) +} + +fn storage_error(source: arrow_schema::ArrowError) -> Error { + Error::DataInvalid { message: format!("Cannot convert MULTISET to Parquet storage (Parquet map keys cannot be NULL): {source}"), source: Some(Box::new(source)) } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_array::{Int32Array, RecordBatch}; + use arrow_buffer::{NullBuffer, OffsetBuffer}; + use bytes::Bytes; + use parquet::arrow::{arrow_reader::ParquetRecordBatchReaderBuilder, ArrowWriter}; + + fn bag(values: Vec>, nulls: Option) -> ArrayRef { + let field = entries_field(DataType::Int32, true); + let DataType::Struct(fields) = field.data_type() else { + unreachable!() + }; + let len = values.len(); + let entries = StructArray::try_new( + fields.clone(), + vec![ + Arc::new(Int32Array::from(values)), + Arc::new(Int32Array::from(vec![3; len])), + ], + None, + ) + .unwrap(); + Arc::new( + ListArray::try_new( + field, + OffsetBuffer::new((0..=len as i32).collect::>().into()), + Arc::new(entries), + nulls, + ) + .unwrap(), + ) + } + + #[test] + fn parquet_multiset_retains_java_map_layout_and_evolves_back_to_list() { + let logical = bag(vec![Some(2), Some(4)], None); + let schema = Arc::new(Schema::new(vec![Field::new( + "bag", + logical.data_type().clone(), + true, + )])); + let physical = parquet_schema(&schema); + assert!(matches!( + physical.field(0).data_type(), + DataType::Map(_, false) + )); + let array = to_parquet_array(&logical, physical.field(0).data_type()).unwrap(); + let mut bytes = Vec::new(); + let mut writer = ArrowWriter::try_new(&mut bytes, physical.clone(), None).unwrap(); + writer + .write(&RecordBatch::try_new(physical, vec![array]).unwrap()) + .unwrap(); + writer.close().unwrap(); + let mut reader = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(bytes)) + .unwrap() + .build() + .unwrap(); + let batch = reader.next().unwrap().unwrap(); + let typ = crate::spec::DataType::Multiset(crate::spec::MultisetType::new( + crate::spec::DataType::Int(crate::spec::IntType::new()), + )); + let restored = + crate::arrow::nested_evolution::evolve_column(batch.column(0), &typ, &typ).unwrap(); + assert_eq!(&logical.to_data(), &restored.to_data()); + } + + #[test] + fn parquet_drops_hidden_null_elements_but_rejects_visible_ones() { + let source = bag( + vec![None, None, Some(4)], + Some(NullBuffer::from(vec![true, false, true])), + ); + let schema = Arc::new(Schema::new(vec![Field::new( + "bag", + source.data_type().clone(), + true, + )])); + let physical = parquet_schema(&schema); + assert!(to_parquet_array(&source, physical.field(0).data_type()).is_err()); + // A sliced-away first bag and a NULL second bag hide both NULL keys. + let converted = + to_parquet_array(&source.slice(1, 2), physical.field(0).data_type()).unwrap(); + let map = converted.as_any().downcast_ref::().unwrap(); + assert!(map.is_null(0)); + assert_eq!( + map.keys() + .as_any() + .downcast_ref::() + .unwrap() + .values(), + &[4] + ); + } + + #[test] + fn legacy_parquet_optional_keys_remain_readable() { + use parquet::arrow::arrow_writer::ArrowWriterOptions; + use parquet::schema::{parser::parse_message_type, types::SchemaDescriptor}; + for values in [vec![Some(2), Some(4)], vec![None, Some(4)]] { + let logical = bag(values, None); + let schema = Arc::new(Schema::new(vec![Field::new( + "bag", + logical.data_type().clone(), + true, + )])); + // Arrow 58 accepted nullable MULTISET keys and emitted this + // non-compliant Parquet schema even when every actual key was valid. + let descriptor = SchemaDescriptor::new(Arc::new(parse_message_type( + "message schema { optional group bag (MAP) { repeated group key_value { optional int32 key; required int32 value; } } }", + ).unwrap())); + let options = ArrowWriterOptions::new() + .with_parquet_schema(descriptor) + .with_skip_arrow_metadata(true); + let mut bytes = Vec::new(); + let mut writer = + ArrowWriter::try_new_with_options(&mut bytes, schema.clone(), options).unwrap(); + writer + .write(&RecordBatch::try_new(schema, vec![logical.clone()]).unwrap()) + .unwrap(); + writer.close().unwrap(); + let batch = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(bytes)) + .unwrap() + .build() + .unwrap() + .next() + .unwrap() + .unwrap(); + let typ = crate::spec::DataType::Multiset(crate::spec::MultisetType::new( + crate::spec::DataType::Int(crate::spec::IntType::new()), + )); + let restored = + crate::arrow::nested_evolution::evolve_column(batch.column(0), &typ, &typ).unwrap(); + assert_eq!(logical.to_data(), restored.to_data()); + } + } + + #[test] + fn multiset_detection_descends_through_rows_arrays_and_maps() { + let bag = DataType::List(entries_field(DataType::Int32, true)); + let nested = DataType::Struct( + vec![Field::new( + "bags", + DataType::List(Arc::new(Field::new("element", bag, true))), + true, + )] + .into(), + ); + assert!(contains_multiset(&nested)); + let map = DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", nested, true), + ] + .into(), + ), + false, + )), + false, + ); + assert!(contains_multiset(&map)); + } + + #[test] + fn ordinary_arrays_of_key_value_rows_keep_list_storage() { + let field = Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("key", DataType::Int32, true), + Field::new("value", DataType::Int32, false), + ] + .into(), + ), + false, + ); + let schema = Arc::new(Schema::new(vec![Field::new( + "array", + DataType::List(Arc::new(field)), + true, + )])); + assert_eq!(parquet_schema(&schema), schema); + } +} diff --git a/crates/paimon/src/arrow/nested_evolution.rs b/crates/paimon/src/arrow/nested_evolution.rs index 2e511139e..5c136f270 100644 --- a/crates/paimon/src/arrow/nested_evolution.rs +++ b/crates/paimon/src/arrow/nested_evolution.rs @@ -758,9 +758,9 @@ fn evolve_map( ) } -/// Rebuild a MULTISET whose element type evolved. Arrow renders a MULTISET as a -/// map of element -> count, so only the key side is reconciled and the counts -/// carry over untouched. +/// Rebuild a MULTISET whose element type evolved. Arrow uses a list of +/// element/count entries; Parquet and historical files can decode as a map. +/// Only the element side evolves and the counts carry over untouched. /// /// Java's read-side cast dispatcher has no MULTISET branch and raises "Cannot /// cast from type ... to type ..." for such a schema change; going through the @@ -772,20 +772,46 @@ fn evolve_multiset( source_element: &DataType, target_element: &DataType, ) -> crate::Result { - let map = visible_map(as_map_array(source, "MULTISET")?)?; - let elements = evolve_column(map.keys(), source_element, target_element)?; - let counts = map.values().clone(); - rebuild_map( - &map, - elements, - counts, - ArrowField::new( - "key", - paimon_type_to_arrow_for_read(target_element)?, - target_element.is_nullable(), - ), - ArrowField::new("value", ArrowDataType::Int32, false), + let list = if let Some(map) = source.as_any().downcast_ref::() { + visible_generic_list(&ListArray::from(map.clone()))? + } else { + visible_list(source)? + }; + let entries = list + .values() + .as_any() + .downcast_ref::() + .filter(|entries| entries.num_columns() == 2) + .ok_or_else(|| crate::Error::DataInvalid { + message: "MULTISET entries must contain an element and count".into(), + source: None, + })?; + let elements = evolve_column(entries.column(0), source_element, target_element)?; + let field = + crate::arrow::multiset::entries_field(paimon_type_to_arrow_for_read(target_element)?, true); + let ArrowDataType::Struct(fields) = field.data_type() else { + unreachable!() + }; + let entries = StructArray::try_new( + fields.clone(), + vec![elements, entries.column(1).clone()], + None, ) + .map_err(|error| crate::Error::DataInvalid { + message: format!("failed to build schema-evolved MULTISET: {error}"), + source: None, + })?; + let result = ListArray::try_new( + field, + list.offsets().clone(), + Arc::new(entries), + list.nulls().cloned(), + ) + .map_err(|error| crate::Error::DataInvalid { + message: format!("failed to build schema-evolved MULTISET: {error}"), + source: None, + })?; + Ok(Arc::new(result)) } fn as_map_array<'a>(source: &'a ArrayRef, kind: &str) -> crate::Result<&'a MapArray> { @@ -1903,15 +1929,15 @@ mod tests { #[test] fn readded_child_inside_a_multiset_does_not_serve_the_dropped_field_values() { // MULTISET> where `x` was dropped and re-added with a new id. - // Arrow renders a MULTISET as a Map of element -> count, so the source - // and target Arrow types are identical and only the field ids differ. + // A historical Parquet MULTISET decodes as Map; the + // current list representation must still reconcile the field ids. let x: ArrayRef = Arc::new(StringArray::from(vec![Some("old")])); let element_fields = Fields::from(vec![ArrowField::new("x", ArrowDataType::Utf8, true)]); let elements: ArrayRef = Arc::new(StructArray::try_new(element_fields.clone(), vec![x], None).unwrap()); let counts: ArrayRef = Arc::new(Int32Array::from(vec![Some(1)])); let entry_fields = Fields::from(vec![ - ArrowField::new("key", ArrowDataType::Struct(element_fields.clone()), true), + ArrowField::new("key", ArrowDataType::Struct(element_fields.clone()), false), ArrowField::new("value", ArrowDataType::Int32, false), ]); let entries = @@ -1945,12 +1971,8 @@ mod tests { let out = evolve_column(&source, &source_type, &target_type).unwrap(); - let map = out - .as_any() - .downcast_ref::() - .unwrap() - .clone(); - let element = as_struct(&map.keys().clone()); + let entries = crate::arrow::multiset::entries(out.as_ref(), 0).unwrap(); + let element = as_struct(entries.column(0)); assert_eq!( element.column_by_name("x").unwrap().null_count(), 1, diff --git a/crates/paimon/src/table/data_file_reader.rs b/crates/paimon/src/table/data_file_reader.rs index 7b775da3a..4cc7d7cb3 100644 --- a/crates/paimon/src/table/data_file_reader.rs +++ b/crates/paimon/src/table/data_file_reader.rs @@ -3079,7 +3079,7 @@ mod tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - batch.schema().as_ref(), + &crate::arrow::legacy::to_legacy_schema(batch.schema().as_ref()).unwrap(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -3088,7 +3088,9 @@ mod tests { }, ) .unwrap(); - writer.write_batch(batch).unwrap(); + writer + .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) + .unwrap(); writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) } @@ -3437,7 +3439,7 @@ mod tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - batches[0].schema().as_ref(), + &crate::arrow::legacy::to_legacy_schema(batches[0].schema().as_ref()).unwrap(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -3449,7 +3451,9 @@ mod tests { ) .unwrap(); for batch in batches { - writer.write_batch(batch).unwrap(); + writer + .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) + .unwrap(); } writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) diff --git a/crates/paimon/src/table/pk_full_text_read.rs b/crates/paimon/src/table/pk_full_text_read.rs index e808bc301..b230e87a7 100644 --- a/crates/paimon/src/table/pk_full_text_read.rs +++ b/crates/paimon/src/table/pk_full_text_read.rs @@ -571,7 +571,7 @@ mod read_tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - batch.schema().as_ref(), + &crate::arrow::legacy::to_legacy_schema(batch.schema().as_ref()).unwrap(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -580,7 +580,9 @@ mod read_tests { }, ) .unwrap(); - writer.write_batch(batch).unwrap(); + writer + .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) + .unwrap(); writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) } diff --git a/crates/paimon/src/table/pk_vector_indexed_split_read.rs b/crates/paimon/src/table/pk_vector_indexed_split_read.rs index 74df0f4b3..8edca20be 100644 --- a/crates/paimon/src/table/pk_vector_indexed_split_read.rs +++ b/crates/paimon/src/table/pk_vector_indexed_split_read.rs @@ -444,7 +444,7 @@ mod e2e_tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - batch.schema().as_ref(), + &crate::arrow::legacy::to_legacy_schema(batch.schema().as_ref()).unwrap(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -453,7 +453,9 @@ mod e2e_tests { }, ) .unwrap(); - writer.write_batch(batch).unwrap(); + writer + .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) + .unwrap(); writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) } @@ -462,7 +464,7 @@ mod e2e_tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - batches[0].schema().as_ref(), + &crate::arrow::legacy::to_legacy_schema(batches[0].schema().as_ref()).unwrap(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -472,7 +474,9 @@ mod e2e_tests { ) .unwrap(); for batch in batches { - writer.write_batch(batch).unwrap(); + writer + .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) + .unwrap(); } writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) diff --git a/crates/paimon/src/table/pk_vector_orchestrator.rs b/crates/paimon/src/table/pk_vector_orchestrator.rs index 0c6aa60b6..7d20ca0c1 100644 --- a/crates/paimon/src/table/pk_vector_orchestrator.rs +++ b/crates/paimon/src/table/pk_vector_orchestrator.rs @@ -988,7 +988,7 @@ mod e2e_tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - batch.schema().as_ref(), + &crate::arrow::legacy::to_legacy_schema(batch.schema().as_ref()).unwrap(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -997,7 +997,9 @@ mod e2e_tests { }, ) .unwrap(); - writer.write_batch(batch).unwrap(); + writer + .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) + .unwrap(); writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) } diff --git a/crates/paimon/src/table/pk_vector_position_read.rs b/crates/paimon/src/table/pk_vector_position_read.rs index dbc720dc6..4083fd9f5 100644 --- a/crates/paimon/src/table/pk_vector_position_read.rs +++ b/crates/paimon/src/table/pk_vector_position_read.rs @@ -346,7 +346,7 @@ mod tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - batch.schema().as_ref(), + &crate::arrow::legacy::to_legacy_schema(batch.schema().as_ref()).unwrap(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -355,7 +355,9 @@ mod tests { }, ) .unwrap(); - writer.write_batch(batch).unwrap(); + writer + .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) + .unwrap(); writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) } @@ -366,7 +368,7 @@ mod tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - batches[0].schema().as_ref(), + &crate::arrow::legacy::to_legacy_schema(batches[0].schema().as_ref()).unwrap(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -376,7 +378,9 @@ mod tests { ) .unwrap(); for batch in batches { - writer.write_batch(batch).unwrap(); + writer + .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) + .unwrap(); } writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) diff --git a/crates/paimon/src/table/pk_vector_read/residual_row_ranges_tests.rs b/crates/paimon/src/table/pk_vector_read/residual_row_ranges_tests.rs index ba533e073..482949583 100644 --- a/crates/paimon/src/table/pk_vector_read/residual_row_ranges_tests.rs +++ b/crates/paimon/src/table/pk_vector_read/residual_row_ranges_tests.rs @@ -73,7 +73,7 @@ fn id_batch(ids: Vec) -> RecordBatch { fn write_mosaic(batch: &RecordBatch) -> Bytes { let mut writer = MosaicWriter::new( MemOutputFile { data: Vec::new() }, - batch.schema().as_ref(), + &crate::arrow::legacy::to_legacy_schema(batch.schema().as_ref()).unwrap(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -82,7 +82,9 @@ fn write_mosaic(batch: &RecordBatch) -> Bytes { }, ) .unwrap(); - writer.write_batch(batch).unwrap(); + writer + .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) + .unwrap(); writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) } diff --git a/crates/paimon/src/table/postpone_bucket_planner.rs b/crates/paimon/src/table/postpone_bucket_planner.rs index e259ff6b5..fe72233e4 100644 --- a/crates/paimon/src/table/postpone_bucket_planner.rs +++ b/crates/paimon/src/table/postpone_bucket_planner.rs @@ -25,8 +25,8 @@ use crate::spec::{ use crate::table::{PostponeBucketPlan, Table}; use crate::Result; use arrow_array::{ - Array, ArrayRef, FixedSizeListArray, Int64Array, LargeListArray, ListArray, MapArray, - RecordBatch, StructArray, + Array, ArrayRef, FixedSizeListArray, Int64Array, LargeListArray, ListArray, RecordBatch, + StructArray, }; use arrow_schema::{DataType as ArrowType, Field, Schema}; use std::collections::{HashMap, HashSet}; @@ -635,11 +635,7 @@ fn variable_size(array: &ArrayRef, row: usize, data_type: &DataType) -> Result Result { - let entries = array - .as_any() - .downcast_ref::() - .ok_or_else(|| data_invalid("Expected MAP input while sizing postpone data"))? - .value(row); + let entries = crate::arrow::multiset::entries(array.as_ref(), row)?; let keys = array_size(entries.column(0), key)?; let values = array_size(entries.column(1), value)?; round_word(checked_size( @@ -824,6 +820,52 @@ mod tests { assert_eq!(planner.input_statistics(&batch).unwrap().num_rows(), 0); } + #[test] + fn input_statistics_rejects_malformed_multiset_entries() { + let schema = TableSchema::new( + 0, + &crate::spec::Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column( + "bag", + DataType::Multiset(crate::spec::MultisetType::new(DataType::Int( + IntType::new(), + ))), + ) + .primary_key(["id"]) + .option("bucket", "-2") + .option("postpone.target-size-per-bucket", "20 b") + .build() + .unwrap(), + ); + let planner = PostponeBucketPlanner::new(&schema, PostponeBucketPlan::default()).unwrap(); + let entries = StructArray::try_new( + vec![Arc::new(Field::new("key", ArrowType::Int32, true))].into(), + vec![Arc::new(Int32Array::from(vec![5]))], + None, + ) + .unwrap(); + let bag = ListArray::try_new( + Arc::new(Field::new("entries", entries.data_type().clone(), false)), + arrow_buffer::OffsetBuffer::new(vec![0, 1].into()), + Arc::new(entries), + None, + ) + .unwrap(); + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "bag", + bag.data_type().clone(), + true, + )])), + vec![Arc::new(bag)], + ) + .unwrap(); + let error = planner.input_statistics(&batch).unwrap_err(); + assert!(matches!(&error, crate::Error::DataInvalid { .. })); + assert!(error.to_string().contains("exactly two fields")); + } + #[test] fn merged_worker_statistics_are_inferred_once_and_keep_known_counts() { let schema = planner_schema(&[("postpone.target-row-num-per-bucket", "2")]); diff --git a/crates/paimon/src/table/write_batch_normalize.rs b/crates/paimon/src/table/write_batch_normalize.rs index 36bd939c7..f2bd421fc 100644 --- a/crates/paimon/src/table/write_batch_normalize.rs +++ b/crates/paimon/src/table/write_batch_normalize.rs @@ -327,6 +327,30 @@ fn normalize_visible_array( |row| (!fixed.is_null(row)).then(|| fixed.value(row)), )))) } + (DataType::Map(_, _), DataType::List(expected_child)) + if crate::arrow::multiset::is_multiset(expected_child) => + { + let map = array.as_any().downcast_ref::().unwrap(); + let nulls = NullBuffer::union(map.nulls(), parent_nulls); + let entries: ArrayRef = Arc::new(map.entries().clone()); + let (entries, offsets) = visible_children(&entries, map.offsets(), nulls.as_ref())?; + let entries = entries.as_any().downcast_ref::().unwrap(); + let DataType::Struct(fields) = expected_child.data_type() else { + unreachable!() + }; + let columns = entries + .columns() + .iter() + .zip(fields) + .map(|(column, field)| normalize_write_array(column, field.data_type())) + .collect::>>()?; + let entries = + StructArray::try_new(fields.clone(), columns, None).map_err(normalization_error)?; + Ok(Arc::new( + ListArray::try_new(expected_child.clone(), offsets, Arc::new(entries), nulls) + .map_err(normalization_error)?, + )) + } (DataType::List(_), DataType::List(expected_child)) => { let list = array.as_any().downcast_ref::().unwrap(); let nulls = NullBuffer::union(list.nulls(), parent_nulls); @@ -589,6 +613,52 @@ fn normalization_error(error: arrow_schema::ArrowError) -> Error { #[cfg(test)] mod tests { use super::*; + + #[test] + fn legacy_map_input_normalizes_to_multiset_list() { + let fields = arrow_schema::Fields::from(vec![ + Field::new("old_key", DataType::Int32, false), + Field::new("old_count", DataType::Int32, false), + ]); + let entries = StructArray::try_new( + fields.clone(), + vec![ + Arc::new(arrow_array::Int32Array::from(vec![5])), + Arc::new(arrow_array::Int32Array::from(vec![3])), + ], + None, + ) + .unwrap(); + let source: ArrayRef = Arc::new(MapArray::new( + Arc::new(Field::new("entries", DataType::Struct(fields), false)), + OffsetBuffer::new(vec![0, 1].into()), + entries, + None, + false, + )); + let target = DataType::List(crate::arrow::multiset::entries_field(DataType::Int32, true)); + let normalized = normalize_write_array(&source, &target).unwrap(); + let entries = crate::arrow::multiset::entries(normalized.as_ref(), 0).unwrap(); + assert_eq!(normalized.data_type(), &target); + assert_eq!( + entries + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 5 + ); + assert_eq!( + entries + .column(1) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 3 + ); + } use arrow_array::{FixedSizeListArray, Int32Array, LargeBinaryArray, StringArray}; use arrow_buffer::{OffsetBuffer, ScalarBuffer}; use arrow_schema::Field; @@ -1548,7 +1618,10 @@ mod tests { let output = normalize_write_array(&input, &expected).unwrap(); assert_eq!(output.data_type(), &expected); assert_eq!(output.len(), input.len()); - assert_eq!(output.nulls(), input.nulls()); + assert_eq!(output.null_count(), input.null_count()); + for row in 0..input.len() { + assert_eq!(output.is_null(row), input.is_null(row)); + } let data = output.to_data(); data.validate_full().unwrap(); } diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 5bcce1a87..b6fe051d1 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -148,6 +148,20 @@ of at most a quarter of the limit and closes the row group once its encoded size reaches the limit, so large or skewed batches stay close to it. These options affect new writes only. +## Multiset Arrow Values + +`MULTISET` uses Arrow `List>`, with a +non-null entry struct and count. The element (`key`) follows the logical element +nullability; this preserves NULL multiset elements, which Arrow Map cannot +represent. The list entry field carries `paimon.multiset=true` metadata so that +schema inference distinguishes it from an ordinary array of rows. Existing Arrow +Map write inputs with non-null keys are also accepted for multiset columns. + +Row and Avro retain their Java file encodings. Parquet retains the Java MAP +encoding, whose keys must be non-null; writing a multiset with a visible NULL +element to Parquet returns an error. Mosaic and Vortex do not support multiset +columns. + ## Parquet Page Pruning Parquet Page Index pruning is enabled by default. Set diff --git a/docs/src/sql.md b/docs/src/sql.md index f3216c355..5d87e6766 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -27,7 +27,7 @@ under the License. [dependencies] paimon = "0.6.0" paimon-datafusion = "0.6.0" -datafusion = "54.0.0" +datafusion = "55.2.0" tokio = { version = "1", features = ["full"] } ``` @@ -36,7 +36,7 @@ and writes to ordinary Paimon tables can create them with `file.format=mosaic`. ## SQL Support Scope -`paimon-datafusion` currently targets Apache DataFusion 54.x. The workspace pins `datafusion = "54.0.0"`. +`paimon-datafusion` currently targets Apache DataFusion 55.x. The workspace uses `datafusion = "55.2.0"`. SQL support has two layers: From 87a9ae4844960054cd7cadc21b450b25e9de7aa9 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 28 Oct 2026 00:52:35 +0800 Subject: [PATCH 2/6] [datafusion] Integrate DataFusion 55 scan and UDF features --- .../src/blob_descriptor_functions.rs | 8 + .../integrations/datafusion/src/blob_view.rs | 4 + .../datafusion/src/physical_plan/audit_log.rs | 5 + .../datafusion/src/physical_plan/scan.rs | 232 +++++++++++++++- .../datafusion/src/variant_functions.rs | 8 + .../datafusion/tests/datafusion55_features.rs | 159 +++++++++++ crates/paimon/src/arrow/format/parquet.rs | 259 ++++++++++++++---- crates/paimon/src/arrow/row_filter.rs | 18 ++ docs/src/sql.md | 36 +++ 9 files changed, 677 insertions(+), 52 deletions(-) create mode 100644 crates/integrations/datafusion/tests/datafusion55_features.rs diff --git a/crates/integrations/datafusion/src/blob_descriptor_functions.rs b/crates/integrations/datafusion/src/blob_descriptor_functions.rs index a49fffb27..a36ce630e 100644 --- a/crates/integrations/datafusion/src/blob_descriptor_functions.rs +++ b/crates/integrations/datafusion/src/blob_descriptor_functions.rs @@ -58,6 +58,10 @@ impl PathToDescriptorFunc { } impl ScalarUDFImpl for PathToDescriptorFunc { + fn is_strict(&self) -> bool { + true + } + fn name(&self) -> &str { PATH_TO_DESCRIPTOR } @@ -104,6 +108,10 @@ impl DescriptorToStringFunc { } impl ScalarUDFImpl for DescriptorToStringFunc { + fn is_strict(&self) -> bool { + true + } + fn name(&self) -> &str { DESCRIPTOR_TO_STRING } diff --git a/crates/integrations/datafusion/src/blob_view.rs b/crates/integrations/datafusion/src/blob_view.rs index b2fed25fe..598e619f8 100644 --- a/crates/integrations/datafusion/src/blob_view.rs +++ b/crates/integrations/datafusion/src/blob_view.rs @@ -120,6 +120,10 @@ impl Hash for BlobViewFunc { } impl ScalarUDFImpl for BlobViewFunc { + fn is_strict(&self) -> bool { + true + } + fn name(&self) -> &str { FUNCTION_NAME } diff --git a/crates/integrations/datafusion/src/physical_plan/audit_log.rs b/crates/integrations/datafusion/src/physical_plan/audit_log.rs index 7b48d58d6..6f6e98acf 100644 --- a/crates/integrations/datafusion/src/physical_plan/audit_log.rs +++ b/crates/integrations/datafusion/src/physical_plan/audit_log.rs @@ -24,6 +24,7 @@ use datafusion::common::{stats::Precision, Statistics}; use datafusion::error::Result as DFResult; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_plan::metrics::MetricsSet; use datafusion::physical_plan::{ ChildrenPropertiesMode, DisplayAs, ExecutionPlan, PlanProperties, ReplaceChildrenOptions, StatisticsArgs, @@ -57,6 +58,10 @@ impl ExecutionPlan for PaimonAuditLogScan { vec![] } + fn metrics(&self) -> Option { + self.inner.metrics() + } + fn apply_expressions( &self, f: &mut dyn FnMut(&Arc) -> DFResult, diff --git a/crates/integrations/datafusion/src/physical_plan/scan.rs b/crates/integrations/datafusion/src/physical_plan/scan.rs index 9ee660d75..80e59aa57 100644 --- a/crates/integrations/datafusion/src/physical_plan/scan.rs +++ b/crates/integrations/datafusion/src/physical_plan/scan.rs @@ -47,12 +47,13 @@ use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::filter_pushdown::{ ChildPushdownResult, FilterPushdownPhase, FilterPushdownPropagation, PushedDown, }; +use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ ChildrenPropertiesMode, DisplayAs, ExecutionPlan, Partitioning, PlanProperties, ReplaceChildrenOptions, StatisticsArgs, }; -use futures::{FutureExt, StreamExt, TryStreamExt}; +use futures::{FutureExt, Stream, StreamExt, TryStreamExt}; use paimon::arrow::ReadBudget; use paimon::spec::{ CoreOptions, DataField, Datum, MergeEngine, Predicate, PredicateBuilder, PredicateOperator, @@ -143,6 +144,39 @@ impl paimon::arrow::RowFilter for DataFusionRowFilter { } impl paimon::arrow::RowFilterFactory for DataFusionRowFilterFactory { + fn supports_row_group_pruning(&self) -> bool { + datafusion::physical_expr::DynamicFilterTracking::classify(&self.predicate) + .contains_dynamic_filter() + } + + fn pruning_predicates(&self, file_fields: &[DataField]) -> paimon::Result> { + let snapshot = datafusion::physical_expr_common::physical_expr::snapshot_physical_expr( + Arc::clone(&self.predicate), + ) + .map_err(datafusion_row_filter_error)?; + let builder = PredicateBuilder::new(file_fields); + Ok(split_conjunction(&snapshot) + .into_iter() + .filter(|expr| { + // Parquet min/max exclude NaN, whereas DataFusion compares floats + // using Arrow's total order. A finite bound cannot rule out NaN. + !collect_columns(expr).iter().any(|column| { + file_fields.iter().any(|field| { + field.name() == column.name() + && matches!( + field.data_type(), + paimon::spec::DataType::Float(_) + | paimon::spec::DataType::Double(_) + ) + }) + }) + }) + .filter_map(|expr| { + translate_physical_predicate(expr.as_ref(), file_fields, &builder, true) + }) + .collect()) + } + fn create( &self, context: paimon::arrow::RowFilterContext<'_>, @@ -797,6 +831,7 @@ pub struct PaimonTableScan { /// Wrapped in `Arc` to avoid deep-cloning `DataSplit` metadata in `execute()`. planned_partitions: Vec>, plan_properties: Arc, + metrics: ExecutionPlanMetricsSet, /// Optional limit hint pushed to paimon-core planning. limit: Option, /// Whether the pushed predicate is exact (no residual filtering needed). @@ -916,6 +951,7 @@ impl PaimonTableScan { pushed_predicate, planned_partitions, plan_properties, + metrics: ExecutionPlanMetricsSet::new(), limit, filter_exact, scan_stats, @@ -1102,9 +1138,15 @@ impl PaimonTableScan { )) }; + let mut stream = Box::pin(futures::stream::once(fut).try_flatten()); + let metrics = BaselineMetrics::new(&self.metrics, partition); + let measured = futures::stream::poll_fn(move |cx| { + let _timer = metrics.elapsed_compute().timer(); + metrics.record_poll(stream.as_mut().poll_next(cx)) + }); Ok(Box::pin(RecordBatchStreamAdapter::new( self.schema(), - futures::stream::once(fut).try_flatten(), + measured, ))) } } @@ -1118,6 +1160,10 @@ impl ExecutionPlan for PaimonTableScan { &self.plan_properties } + fn metrics(&self) -> Option { + Some(self.metrics.clone_inner()) + } + fn children(&self) -> Vec<&Arc> { vec![] } @@ -1379,6 +1425,37 @@ mod tests { )])) } + #[test] + fn runtime_pruning_refreshes_incomplete_topk_thresholds() { + use paimon::arrow::RowFilterFactory; + let column: Arc = Arc::new(Column::new("id", 0)); + let dynamic = Arc::new(DynamicFilterPhysicalExpr::new( + vec![Arc::clone(&column)], + lit(true), + )); + let filter: Arc = dynamic.clone(); + let factory = DataFusionRowFilterFactory::new(filter, test_schema()); + let fields = test_read_type(); + assert!(factory.pruning_predicates(&fields).unwrap().is_empty()); + for threshold in [100, 5] { + dynamic + .update(Arc::new(BinaryExpr::new( + Arc::clone(&column), + Operator::Lt, + lit(threshold), + ))) + .unwrap(); + assert_eq!( + factory.pruning_predicates(&fields).unwrap(), + vec![PredicateBuilder::new(&fields) + .less_than("id", Datum::Int(threshold)) + .unwrap()] + ); + } + dynamic.update(lit(true)).unwrap(); + assert!(factory.pruning_predicates(&fields).unwrap().is_empty()); + } + fn test_read_type() -> Vec { vec![DataField::new( 0, @@ -1387,6 +1464,157 @@ mod tests { )] } + #[tokio::test] + async fn runtime_row_group_pruning_preserves_nan_after_threshold_updates() { + use datafusion::arrow::array::{ArrayRef, Float32Array, Float64Array}; + use paimon::spec::{DoubleType, FloatType}; + use parquet::arrow::ArrowWriter; + use parquet::file::properties::WriterProperties; + + for single_precision in [false, true] { + for descending in [false, true] { + let dir = tempdir().unwrap(); + let bucket = dir.path().join("bucket-0"); + fs::create_dir_all(&bucket).unwrap(); + let file = bucket.join("data.parquet"); + let sign = if descending { 1.0 } else { -1.0 }; + // Floating-point arithmetic may canonicalize NaN's sign. Build + // explicit bit patterns so ascending tests really use negative NaN. + let nan = f64::from_bits(if descending { + 0x7ff8_0000_0000_0000 + } else { + 0xfff8_0000_0000_0000 + }); + let values = vec![sign * 100.0, sign * 100.0, sign, nan]; + let (data_type, array): (_, ArrayRef) = if single_precision { + ( + DataType::Float(FloatType::new()), + Arc::new(Float32Array::from( + values + .iter() + .map(|v| { + if v.is_nan() { + f32::from_bits(if descending { + 0x7fc0_0000 + } else { + 0xffc0_0000 + }) + } else { + *v as f32 + } + }) + .collect::>(), + )), + ) + } else { + ( + DataType::Double(DoubleType::new()), + Arc::new(Float64Array::from(values)), + ) + }; + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "value", + array.data_type().clone(), + true, + )])); + let batch = RecordBatch::try_new(Arc::clone(&arrow_schema), vec![array]).unwrap(); + let mut writer = ArrowWriter::try_new( + fs::File::create(&file).unwrap(), + Arc::clone(&arrow_schema), + Some( + WriterProperties::builder() + .set_max_row_group_row_count(Some(2)) + .build(), + ), + ) + .unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + let schema = PaimonSchema::builder() + .column("value", data_type) + .option("read.batch-size", "2") + .build() + .unwrap(); + let table = Table::new( + FileIOBuilder::new("file").build().unwrap(), + Identifier::new("default", "nan_topk"), + local_file_path(dir.path()), + TableSchema::new(0, &schema), + None, + ); + let split = paimon::DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path(local_file_path(&bucket)) + .with_total_buckets(1) + .with_data_files(vec![test_data_file( + "data.parquet", + 4, + fs::metadata(file).unwrap().len() as i64, + )]) + .build() + .unwrap(); + let column: Arc = Arc::new(Column::new("value", 0)); + let dynamic = Arc::new(DynamicFilterPhysicalExpr::new( + vec![Arc::clone(&column)], + lit(true), + )); + let filter: Arc = dynamic.clone(); + let read = table + .new_read_builder() + .new_read() + .unwrap() + .with_row_filter_factory(Arc::new(DataFusionRowFilterFactory::new( + filter, + arrow_schema, + ))); + let mut stream = read.to_arrow(&[split]).unwrap(); + assert_eq!(stream.try_next().await.unwrap().unwrap().num_rows(), 2); + let threshold = if single_precision { + ScalarValue::Float32(Some((sign * 100.0) as f32)) + } else { + ScalarValue::Float64(Some(sign * 100.0)) + }; + dynamic + .update(Arc::new(BinaryExpr::new( + column, + if descending { + Operator::Gt + } else { + Operator::Lt + }, + lit(threshold), + ))) + .unwrap(); + let expected = dynamic.evaluate(&batch).unwrap().into_array(4).unwrap(); + assert_eq!( + expected.as_any().downcast_ref::().unwrap(), + &BooleanArray::from(vec![false, false, false, true]) + ); + let rest = stream.try_collect::>().await.unwrap(); + assert_eq!(rest.iter().map(RecordBatch::num_rows).sum::(), 1, + "a finite Parquet bound must not discard NaN: single_precision={single_precision}, descending={descending}"); + let value = rest.iter().find(|b| b.num_rows() > 0).unwrap().column(0); + assert!(if single_precision { + value + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + .is_nan() + } else { + value + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + .is_nan() + }); + } + } + } + #[test] fn test_binary_manifest_bounds_are_not_exposed() { for data_type in [ diff --git a/crates/integrations/datafusion/src/variant_functions.rs b/crates/integrations/datafusion/src/variant_functions.rs index c94845da9..a868447e4 100644 --- a/crates/integrations/datafusion/src/variant_functions.rs +++ b/crates/integrations/datafusion/src/variant_functions.rs @@ -55,6 +55,10 @@ impl ParseJsonFunc { } impl ScalarUDFImpl for ParseJsonFunc { + fn is_strict(&self) -> bool { + true + } + fn name(&self) -> &str { if self.try_parse { "try_parse_json" @@ -181,6 +185,10 @@ impl VariantGetFunc { } impl ScalarUDFImpl for VariantGetFunc { + fn is_strict(&self) -> bool { + true + } + fn name(&self) -> &str { if self.try_get { "try_variant_get" diff --git a/crates/integrations/datafusion/tests/datafusion55_features.rs b/crates/integrations/datafusion/tests/datafusion55_features.rs new file mode 100644 index 000000000..33049cbd9 --- /dev/null +++ b/crates/integrations/datafusion/tests/datafusion55_features.rs @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod common; + +use datafusion::common::JoinType; +use datafusion::execution::TaskContext; +use datafusion::logical_expr::LogicalPlan; +use datafusion::physical_plan::collect; +use std::sync::Arc; + +fn join_type(plan: &LogicalPlan) -> Option { + if let LogicalPlan::Join(join) = plan { + return Some(join.join_type); + } + plan.inputs().into_iter().find_map(join_type) +} + +#[tokio::test] +async fn strict_udfs_enable_outer_join_elimination_without_changing_variant_null() { + let (_tmp, ctx) = common::setup_sql_context().await; + for function in [ + "path_to_descriptor(r.path)", + "descriptor_to_string(path_to_descriptor(r.path))", + "parse_json(r.json)", + "try_parse_json(r.json)", + "variant_get(parse_json(r.json), '$.a')", + "try_variant_get(parse_json(r.json), '$.a')", + "blob_view('test_db.metrics', 0, r.id)", + ] { + let sql = format!("SELECT l.id FROM (VALUES (1), (2)) l(id) LEFT JOIN (VALUES (1, 'x', '{{\"a\": 1}}')) r(id, path, json) ON l.id = r.id WHERE {function} IS NOT NULL"); + let frame = ctx.sql(&sql).await.unwrap(); + let plan = frame.into_optimized_plan().unwrap(); + assert_eq!( + join_type(&plan), + Some(JoinType::Inner), + "{function}: {plan:?}" + ); + assert_eq!(common::row_count(&ctx, &sql).await, 1); + } + let sql = "SELECT l.id FROM (VALUES (1), (2)) l(id) LEFT JOIN (VALUES (1, 'null')) r(id, json) ON l.id = r.id WHERE NOT is_variant_null(parse_json(r.json))"; + assert_eq!(common::row_count(&ctx, sql).await, 1); +} + +#[tokio::test] +async fn explain_pgjson_reports_actual_scan_output_rows() { + let (_tmp, ctx) = common::setup_sql_context().await; + common::exec(&ctx, "CREATE TABLE paimon.test_db.metrics (id INT)").await; + common::exec( + &ctx, + "INSERT INTO paimon.test_db.metrics VALUES (1), (2), (3)", + ) + .await; + for (relation, scan_name, expected_scan_rows) in [ + ("metrics", "PaimonTableScan", 2), + ("metrics$audit_log", "PaimonAuditLogScan", 2), + ] { + let query = format!("SELECT id FROM paimon.test_db.{relation} WHERE id > 1"); + let frame = ctx.sql(&query).await.unwrap(); + let plan = frame.create_physical_plan().await.unwrap(); + let batches = collect(Arc::clone(&plan), Arc::new(TaskContext::default())) + .await + .unwrap(); + assert_eq!( + batches.iter().map(|batch| batch.num_rows()).sum::(), + 2 + ); + fn scan_rows( + plan: &dyn datafusion::physical_plan::ExecutionPlan, + name: &str, + ) -> Option { + if plan.name() == name { + return plan.metrics().and_then(|metrics| metrics.output_rows()); + } + plan.children() + .into_iter() + .find_map(|child| scan_rows(child.as_ref(), name)) + } + assert_eq!( + scan_rows(plan.as_ref(), scan_name), + Some(expected_scan_rows) + ); + let output = ctx + .sql(&format!( + "EXPLAIN (ANALYZE, FORMAT pgjson, METRICS 'rows') {query}" + )) + .await + .unwrap() + .collect() + .await + .unwrap(); + let value: serde_json::Value = serde_json::from_str(common::string_value( + output[0].column_by_name("plan").unwrap().as_ref(), + 0, + )) + .unwrap(); + fn scan_node<'a>( + value: &'a serde_json::Value, + name: &str, + ) -> Option<&'a serde_json::Value> { + match value { + serde_json::Value::Object(values) => { + if values + .get("Node Type") + .and_then(|v| v.as_str()) + .is_some_and(|v| v.contains(name)) + { + return Some(value); + } + values.values().find_map(|v| scan_node(v, name)) + } + serde_json::Value::Array(values) => values.iter().find_map(|v| scan_node(v, name)), + _ => None, + } + } + let scan = scan_node(&value, scan_name).expect("Paimon scan in pgjson explain"); + assert_eq!(scan["Actual Rows"], expected_scan_rows, "{value}"); + } +} + +#[tokio::test] +async fn new_array_functions_run_on_paimon_columns() { + let (_tmp, ctx) = common::setup_sql_context().await; + common::exec( + &ctx, + "CREATE TABLE paimon.test_db.arrays (id INT, values_ ARRAY)", + ) + .await; + common::exec( + &ctx, + "INSERT INTO paimon.test_db.arrays VALUES (1, [1, 2, 3]), (2, [])", + ) + .await; + let rows = common::collect_id_value(&ctx, + "SELECT id, CAST(array_sum(array_scale(values_, 2)) AS INT) AS value FROM paimon.test_db.arrays WHERE id = 1").await; + assert_eq!(rows, vec![(1, 12)]); + assert_eq!( + common::row_count( + &ctx, + "SELECT unnest_outer(values_) FROM paimon.test_db.arrays" + ) + .await, + 4 + ); +} diff --git a/crates/paimon/src/arrow/format/parquet.rs b/crates/paimon/src/arrow/format/parquet.rs index 988e75637..6dbadf9be 100644 --- a/crates/paimon/src/arrow/format/parquet.rs +++ b/crates/paimon/src/arrow/format/parquet.rs @@ -906,7 +906,10 @@ impl FormatFileReader for ParquetFormatReader { Some(fp) => (&fp.predicates, &fp.file_fields), None => (&empty_predicates, &[]), }; - let row_filter_factory = predicates.and_then(|fp| fp.row_filter_factory.as_deref()); + let row_filter_factory = predicates.and_then(|fp| fp.row_filter_factory.clone()); + let runtime_pruning = row_filter_factory + .as_ref() + .is_some_and(|factory| factory.supports_row_group_pruning()); // Predicates need both indexes for page-stat pruning. Row selection only // needs OffsetIndex so arrow-rs can avoid fetching unselected pages. @@ -983,7 +986,7 @@ impl FormatFileReader for ParquetFormatReader { .map(ParquetRowFilter::into_predicates) .unwrap_or_default(); - if let Some(factory) = row_filter_factory { + if let Some(factory) = row_filter_factory.as_deref() { let file_schema = batch_stream_builder.schema(); match factory.create(RowFilterContext { file_schema }) { Ok(filters) => { @@ -1043,14 +1046,15 @@ impl FormatFileReader for ParquetFormatReader { // Resource-aware sequential reads build one row-group stream at a time, // so admission precedes its data I/O. Preserve stateful decoder filters // across those streams instead of recreating the engine's factory. - let shared_filters = if self.read_budget.as_ref().is_some_and(|b| b.has_resources()) { - std::mem::take(&mut decoder_predicates) - .into_iter() - .map(SharedParquetPredicate::new) - .collect::>() - } else { - Vec::new() - }; + let shared_filters = + if runtime_pruning || self.read_budget.as_ref().is_some_and(|b| b.has_resources()) { + std::mem::take(&mut decoder_predicates) + .into_iter() + .map(SharedParquetPredicate::new) + .collect::>() + } else { + Vec::new() + }; if !decoder_predicates.is_empty() { batch_stream_builder = batch_stream_builder.with_row_filter(ParquetRowFilter::new(decoder_predicates)); @@ -1107,45 +1111,46 @@ impl FormatFileReader for ParquetFormatReader { .min(batch_stream_builder.metadata().num_row_groups()) }) .unwrap_or(1); - let selected_row_groups = self - .read_budget - .as_ref() - .filter(|budget| { + let selected_row_groups = (runtime_pruning + || self.read_budget.as_ref().is_some_and(|budget| { row_group_parallelism > 1 || budget.diagnostics_enabled() || budget.has_resources() - }) - .map(|budget| { - let mut row_group_selection = combined_selection; - let selected_row_groups = batch_stream_builder - .metadata() - .row_groups() - .iter() - .enumerate() - .filter_map(|(row_group_index, row_group)| { - let selection = row_group_selection - .as_mut() - .map(|selection| selection.split_off(row_group.num_rows() as usize)); - if selection - .as_ref() - .is_some_and(|selection| !selection.selects_any()) - { - return None; - } - let projection = if budget.has_resources() { - &memory_mask - } else { - &mask - }; - let projected_bytes = projected_row_group_bytes(row_group, projection); - Some((row_group_index, selection, projected_bytes)) - }) - .collect::>(); - let projected_bytes = selected_row_groups - .iter() - .map(|(_, _, projected_bytes)| *projected_bytes) - .collect::>(); + })) + .then(|| { + let budget = self.read_budget.as_ref(); + let mut row_group_selection = combined_selection; + let selected_row_groups = batch_stream_builder + .metadata() + .row_groups() + .iter() + .enumerate() + .filter_map(|(row_group_index, row_group)| { + let selection = row_group_selection + .as_mut() + .map(|selection| selection.split_off(row_group.num_rows() as usize)); + if selection + .as_ref() + .is_some_and(|selection| !selection.selects_any()) + { + return None; + } + let projection = if budget.is_some_and(|budget| budget.has_resources()) { + &memory_mask + } else { + &mask + }; + let projected_bytes = projected_row_group_bytes(row_group, projection); + Some((row_group_index, selection, projected_bytes)) + }) + .collect::>(); + let projected_bytes = selected_row_groups + .iter() + .map(|(_, _, projected_bytes)| *projected_bytes) + .collect::>(); + if let Some(budget) = budget { budget.record_projected_row_groups(&projected_bytes); - selected_row_groups - }); + } + selected_row_groups + }); if row_group_parallelism > 1 { let selected_row_groups = selected_row_groups.expect("parallel row-group reads need a selection plan"); @@ -1245,8 +1250,12 @@ impl FormatFileReader for ParquetFormatReader { return Ok(stream.boxed()); } - if let Some(budget) = self.read_budget.as_ref().filter(|b| b.has_resources()) { - let budget = Arc::clone(budget); + let budget = self + .read_budget + .as_ref() + .filter(|budget| budget.has_resources()) + .cloned(); + if budget.is_some() || runtime_pruning { let selected = selected_row_groups.expect("resource-aware reads need a selection plan"); let metadata = ArrowReaderMetadata::try_new( batch_stream_builder.metadata().clone(), @@ -1257,12 +1266,38 @@ impl FormatFileReader for ParquetFormatReader { row_filter_factory: None, file_fields: file_fields.to_vec(), }); + let runtime_fields = file_fields.to_vec(); + let pruning_metadata = Arc::clone(batch_stream_builder.metadata()); + let column_indices = pruning_metadata + .row_groups() + .first() + .map(|group| build_row_group_column_indices(group.columns(), &runtime_fields)) + .unwrap_or_default(); + let identity_mapping = (0..runtime_fields.len()).map(Some).collect::>(); return Ok(async_stream::try_stream! { for (index, selection, estimated_bytes) in selected { + // Refresh immediately before data I/O, never while building the + // whole-file selection: a TopK threshold can tighten after a batch. + if runtime_pruning { + let factory = row_filter_factory.as_ref().expect("runtime pruning has a factory"); + let may_match = match factory.pruning_predicates(&runtime_fields) { + Ok(predicates) => predicates_may_match_with_schema( + &predicates, + &ParquetRowGroupStats { row_group: pruning_metadata.row_group(index), column_indices: &column_indices }, + &identity_mapping, + &runtime_fields, + ), + Err(error) => { + log::warn!("failed to refresh runtime Parquet pruning: {error}"); + true + } + }; + if !may_match { continue; } + } // Foreground and merge reads only try memory admission. They // never hold a shared scheduling slot while awaiting another // merge input, nor wait for output buffers owned downstream. - let _permit = budget.reserve_memory(estimated_bytes)?; + let _permit = budget.as_ref().map(|budget| budget.reserve_memory(estimated_bytes)).transpose()?; let mut stream = build_row_group_stream( Arc::clone(&shared_reader), file_size, metadata.clone(), mask.clone(), index, batch_size, selection, shared_filters.clone(), @@ -7185,6 +7220,130 @@ mod tests { assert_eq!(ids, vec![1, 2, 3, 4, 5]); } + #[tokio::test] + async fn runtime_row_group_pruning_refreshes_before_io_and_falls_open() { + #[derive(Debug)] + struct ThresholdFactory { + threshold: Arc, + calls: Arc, + fail: bool, + } + impl crate::arrow::RowFilterFactory for ThresholdFactory { + fn create( + &self, + _: crate::arrow::RowFilterContext<'_>, + ) -> crate::Result>> { + Ok(vec![]) + } + fn supports_row_group_pruning(&self) -> bool { + true + } + fn pruning_predicates(&self, fields: &[DataField]) -> crate::Result> { + self.calls.fetch_add(1, AtomicOrdering::SeqCst); + if self.fail { + return Err(Error::DataInvalid { + message: "unavailable runtime threshold".into(), + source: None, + }); + } + let limit = self.threshold.load(AtomicOrdering::SeqCst); + if limit == usize::MAX { + return Ok(vec![]); + } + Ok(vec![PredicateBuilder::new(fields) + .less_than("id", Datum::Int(limit as i32)) + .unwrap()]) + } + } + for (resources, selected_ranges) in + [(false, false), (true, false), (false, true), (true, true)] + { + let mut read_bytes = Vec::new(); + for (statistics, fail) in [ + (EnabledStatistics::Chunk, false), + (EnabledStatistics::None, false), + (EnabledStatistics::Chunk, true), + ] { + let data = + Bytes::from(write_multi_row_group_parquet(64, 512, statistics, true).await); + let tracker = TrackingFileRead::new(data.clone()); + let threshold = Arc::new(AtomicUsize::new(usize::MAX)); + let calls = Arc::new(AtomicUsize::new(0)); + let fields = vec![ + residual_test_field(0, "id", DataType::Int(IntType::new())), + residual_test_field(1, "value", DataType::Int(IntType::new())), + ]; + let predicates = FilePredicates { + predicates: vec![], + file_fields: fields.clone(), + row_filter_factory: Some(Arc::new(ThresholdFactory { + threshold: Arc::clone(&threshold), + calls: Arc::clone(&calls), + fail, + })), + }; + let reader = if resources { + ParquetFormatReader::with_read_budget(Arc::new( + ReadBudget::default().with_resources( + crate::resource::ResourceContext::builder() + .memory_limit(1024 * 1024) + .build() + .unwrap(), + ), + )) + } else { + ParquetFormatReader::default() + }; + let mut stream = reader + .read_batch_stream( + Box::new(tracker.clone()), + data.len() as u64, + &fields, + Some(&predicates), + Some(64), + selected_ranges + .then(|| vec![RowRange::new(3, 68), RowRange::new(130, 260)]), + ) + .await + .unwrap(); + tracker.reset(); + let first = stream.next().await.unwrap().unwrap(); + assert_eq!(first.num_rows(), if selected_ranges { 61 } else { 64 }); + assert_eq!( + first + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(first.num_rows() - 1), + 63 + ); + threshold.store(64, AtomicOrdering::SeqCst); + let rest = stream.try_collect::>().await.unwrap(); + let rows: usize = rest.iter().map(RecordBatch::num_rows).sum(); + assert_eq!( + rows, + if statistics == EnabledStatistics::Chunk && !fail { + 0 + } else if selected_ranges { + 136 + } else { + 448 + } + ); + assert_eq!( + calls.load(AtomicOrdering::SeqCst), + if selected_ranges { 5 } else { 8 } + ); + read_bytes.push(tracker.bytes_read()); + } + assert!( + read_bytes[0] < read_bytes[1], + "pruning must avoid data I/O: {read_bytes:?}" + ); + } + } + #[tokio::test] async fn test_external_row_filters_are_ordered_by_parquet_cost() { use std::sync::Mutex; diff --git a/crates/paimon/src/arrow/row_filter.rs b/crates/paimon/src/arrow/row_filter.rs index ff973e101..ace249888 100644 --- a/crates/paimon/src/arrow/row_filter.rs +++ b/crates/paimon/src/arrow/row_filter.rs @@ -50,4 +50,22 @@ pub trait RowFilter: Send + 'static { /// particular file. pub trait RowFilterFactory: Debug + Send + Sync { fn create(&self, context: RowFilterContext<'_>) -> crate::Result>>; + + /// Whether to refresh pruning predicates before reading each Parquet row group. + fn supports_row_group_pruning(&self) -> bool { + false + } + + /// A conservative snapshot of this filter, expressed against the historical + /// file fields. Called before fetching a row group's data, so changing TopK + /// thresholds can skip unread groups. Unsupported expressions must return + /// no predicates; errors fall back to reading the group. Exact row filtering + /// remains the caller's responsibility. Once a snapshot excludes a row, + /// later snapshots must not admit it again: skipped groups cannot be replayed. + fn pruning_predicates( + &self, + _file_fields: &[crate::spec::DataField], + ) -> crate::Result> { + Ok(Vec::new()) + } } diff --git a/docs/src/sql.md b/docs/src/sql.md index 5d87e6766..156f52895 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -47,6 +47,42 @@ Not every DataFusion DDL/DML statement maps to a Paimon table operation. For Pai For the exact delegated SQL grammar, see the [DataFusion SQL Reference](https://datafusion.apache.org/user-guide/sql/index.html). +### DataFusion 55 integration + +Paimon Parquet scans refresh supported dynamic predicates before reading each +remaining row group. This lets an `ORDER BY ... LIMIT` threshold skip groups +that can no longer contribute. Missing statistics, unsupported predicates, and +snapshot errors keep the group readable; the original runtime filter still +checks decoded rows. Floating-point predicates stay with DataFusion's row +filter because Parquet bounds do not account for NaN ordering. + +The NULL-propagating `parse_json`, `try_parse_json`, `variant_get`, +`try_variant_get`, `path_to_descriptor`, `descriptor_to_string`, and `blob_view` +functions declare DataFusion's strict UDF metadata. This enables outer join +elimination when a filter rejects their NULL results. `is_variant_null` keeps +its distinct SQL NULL behavior. + +Regular and audit-log scans report actual output rows, batches, bytes, and +compute time to DataFusion. For example, PostgreSQL-compatible JSON plans can +be requested with: + +```sql +EXPLAIN (ANALYZE, FORMAT pgjson, METRICS 'rows') +SELECT * FROM paimon.default.my_table WHERE id > 10; +``` + +DataFusion's built-in array functions (including `array_scale`, `array_sum`, +and `unnest_outer`) and its planner, aggregation, and spill improvements are +available through the normal session. Custom spill backends can be configured +on the `RuntimeEnv` passed to `SQLContext::builder`. + +Paimon `MERGE INTO` continues to use `SQLContext`'s Paimon write implementation. +DataFusion's generic `TableProvider::merge_into` hook, `file_row_index()` and +`input_file_name()` for Paimon scans, range-partition declarations, and custom +query-planner FFI are not provided by this upgrade. In particular, Paimon's +logical row IDs are not physical file row offsets, and ordinary scan splits +do not guarantee disjoint sorted ranges. + ## Registering Catalog Register an entire Paimon catalog so all databases and tables are accessible via `paimon.database.table` syntax: From e63669f7e245d13e0cc7a3f3859d48c8905fc990 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 28 Oct 2026 14:03:52 +0800 Subject: [PATCH 3/6] [test] Adapt incremental diff fixtures to multiset lists --- .../tests/incremental_diff_extended_test.rs | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/crates/paimon/tests/incremental_diff_extended_test.rs b/crates/paimon/tests/incremental_diff_extended_test.rs index cf67ce42d..3f728d092 100644 --- a/crates/paimon/tests/incremental_diff_extended_test.rs +++ b/crates/paimon/tests/incremental_diff_extended_test.rs @@ -118,12 +118,14 @@ fn nested_schema() -> TableSchema { TableSchema::new(0, &schema) } -fn map_array(arrow_type: &ArrowDataType, values: &[StringMap]) -> MapArray { - let ArrowDataType::Map(entries_field, sorted) = arrow_type else { - panic!("expected Arrow Map") +fn map_or_multiset_array(arrow_type: &ArrowDataType, values: &[StringMap]) -> ArrayRef { + let (entries_field, sorted) = match arrow_type { + ArrowDataType::Map(entries_field, sorted) => (entries_field, Some(*sorted)), + ArrowDataType::List(entries_field) => (entries_field, None), + _ => panic!("expected Arrow Map or multiset List"), }; let ArrowDataType::Struct(entry_fields) = entries_field.data_type() else { - panic!("expected Map entries struct") + panic!("expected entries struct") }; let mut offsets = vec![0_i32]; let mut keys = Vec::new(); @@ -148,14 +150,17 @@ fn map_array(arrow_type: &ArrowDataType, values: &[StringMap]) -> MapArray { None, ) .unwrap(); - MapArray::try_new( - Arc::clone(entries_field), - OffsetBuffer::new(ScalarBuffer::from(offsets)), - entries, - Some(NullBuffer::from(valid)), - *sorted, - ) - .unwrap() + let offsets = OffsetBuffer::new(ScalarBuffer::from(offsets)); + let nulls = Some(NullBuffer::from(valid)); + match sorted { + Some(sorted) => Arc::new( + MapArray::try_new(Arc::clone(entries_field), offsets, entries, nulls, sorted).unwrap(), + ), + None => Arc::new( + ListArray::try_new(Arc::clone(entries_field), offsets, Arc::new(entries), nulls) + .unwrap(), + ), + } } fn nested_batch(table: &Table, rows: &[NestedRow]) -> RecordBatch { @@ -222,8 +227,8 @@ fn nested_batch(table: &Table, rows: &[NestedRow]) -> RecordBatch { )), Arc::new(payload), Arc::new(items), - Arc::new(map_array(schema.field(3).data_type(), &attributes)), - Arc::new(map_array(schema.field(4).data_type(), &bag)), + map_or_multiset_array(schema.field(3).data_type(), &attributes), + map_or_multiset_array(schema.field(4).data_type(), &bag), ], ) .unwrap() From fc00f08d666a97ad6286821967c4181832035b5d Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Tue, 3 Nov 2026 14:24:51 +0800 Subject: [PATCH 4/6] [core] Upgrade Mosaic and Vortex to native Arrow 59 Use Mosaic 0.3.0 and Vortex 0.90.0 on the Rust 1.95 main branch and remove the Arrow 58 C Data bridge. Migrate Vortex sessions and logical layout slots while preserving nested nulls and old-file reads. Align Spark aggregates with DataFusion 55, preserve any_value null semantics, and repair the vector C test schema fixture. --- Cargo.lock | 868 +++++++++++------- Cargo.toml | 2 +- DEPENDENCIES.rust.tsv | 206 +++-- benchmarks/tpcds/DEPENDENCIES.rust.tsv | 105 ++- bindings/c/DEPENDENCIES.rust.tsv | 8 +- bindings/c/src/tests.rs | 7 + bindings/go/DEPENDENCIES.rust.tsv | 8 +- bindings/python/DEPENDENCIES.rust.tsv | 124 +-- .../integration_tests/DEPENDENCIES.rust.tsv | 8 +- .../datafusion/DEPENDENCIES.rust.tsv | 187 ++-- .../datafusion/src/selection_aggregate.rs | 11 +- .../datafusion/src/spark_functions.rs | 21 +- .../datafusion/tests/remaining_compat.rs | 138 +-- .../paimon-rest-server/DEPENDENCIES.rust.tsv | 8 +- crates/paimon/Cargo.toml | 7 +- crates/paimon/DEPENDENCIES.rust.tsv | 96 +- crates/paimon/src/arrow/format/mosaic.rs | 17 +- .../paimon/src/arrow/format/mosaic_write.rs | 5 +- crates/paimon/src/arrow/format/vortex.rs | 168 +++- .../paimon/src/arrow/format/vortex/nested.rs | 55 +- crates/paimon/src/arrow/legacy.rs | 189 ---- crates/paimon/src/arrow/mod.rs | 1 - crates/paimon/src/table/data_file_reader.rs | 12 +- crates/paimon/src/table/pk_full_text_read.rs | 6 +- .../src/table/pk_vector_indexed_split_read.rs | 12 +- .../src/table/pk_vector_orchestrator.rs | 6 +- .../src/table/pk_vector_position_read.rs | 12 +- .../residual_row_ranges_tests.rs | 6 +- .../paimon/testdata/vortex_compat/generate.py | 33 + .../testdata/vortex_compat/vortex_0_75.vortex | Bin 0 -> 5984 bytes deny.toml | 3 - 31 files changed, 1278 insertions(+), 1051 deletions(-) delete mode 100644 crates/paimon/src/arrow/legacy.rs create mode 100644 crates/paimon/testdata/vortex_compat/generate.py create mode 100644 crates/paimon/testdata/vortex_compat/vortex_0_75.vortex diff --git a/Cargo.lock b/Cargo.lock index 2f0999cb2..321544845 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,6 +80,17 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "alp" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b23203d5e87df8d8393c8021b53f9193b697ba13fc4ef653911dc1235e87bfd7" +dependencies = [ + "fastlanes 0.6.1", + "itertools 0.15.0", + "num-traits", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -159,7 +170,7 @@ dependencies = [ "liblzma", "log", "miniz_oxide", - "num-bigint", + "num-bigint 0.4.8", "quad-rand", "rand 0.9.5", "regex-lite", @@ -221,9 +232,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arrow" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" +checksum = "3f04e0a89cc175eaf3740282537851e6fd9d7276f97730b9d0ea2859c11a85df" dependencies = [ "arrow-arith", "arrow-array", @@ -243,9 +254,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" +checksum = "157943c720919e8f5b6731407ce5cd4c9a5ef47a75bf10edaf24b7081b5763ef" dependencies = [ "arrow-array", "arrow-buffer", @@ -257,9 +268,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" +checksum = "72b5b1682924f11ecf6929689c16cbc69efbb6dd141b06ad98ee1cb10d858c43" dependencies = [ "ahash", "arrow-buffer", @@ -269,6 +280,7 @@ dependencies = [ "chrono-tz", "half", "hashbrown 0.17.1", + "libc", "num-complex", "num-integer", "num-traits", @@ -276,21 +288,21 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" +checksum = "b5d687afc6c927965a8a759f3b68de91933d9dfa2e1bb6e156bac84b85665e92" dependencies = [ "bytes", "half", - "num-bigint", + "num-bigint 0.5.1", "num-traits", ] [[package]] name = "arrow-cast" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" +checksum = "3f23c3379feede42135ef27ca60e67f6e643cd4343ae74e4c8b16c83f55afd85" dependencies = [ "arrow-array", "arrow-buffer", @@ -299,7 +311,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64 0.22.1", + "base64 0.23.1", "chrono", "comfy-table", "half", @@ -310,9 +322,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26" +checksum = "85b1984628b091b8319b3c305fce21d04f9e9b2578bb733b0d0108462ae1d698" dependencies = [ "arrow-array", "arrow-cast", @@ -325,9 +337,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" +checksum = "bb0a031c7ff5e2bdb443117054fb466af1d4df5b7f0ecbe197713dc8b113b1ba" dependencies = [ "arrow-buffer", "arrow-schema", @@ -338,9 +350,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" +checksum = "f5dd1afadda2eb821aaf851bb4c0adef60c30cc0f1a29bd02fb99dd07fb9f58d" dependencies = [ "arrow-array", "arrow-buffer", @@ -348,15 +360,15 @@ dependencies = [ "arrow-schema", "arrow-select", "flatbuffers", - "lz4_flex 0.13.1", + "lz4_flex 0.14.0", "zstd 0.13.3", ] [[package]] name = "arrow-json" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14" +checksum = "e2bc4d56a0dad84cc51c5812ccff48b642047a1f3be80349cd84318983e79edb" dependencies = [ "arrow-array", "arrow-buffer", @@ -379,9 +391,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" +checksum = "561b8209faa2cd489764b7fd07d7cb9e8d274d78048c38de59f77c41140cb834" dependencies = [ "arrow-array", "arrow-buffer", @@ -392,9 +404,9 @@ dependencies = [ [[package]] name = "arrow-pyarrow" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ffb9be5a873590f825aef50df20e0f8dff5fd42a77058e28bbe7bd44bb53dec" +checksum = "ba037102b3f52163f96db970209ada931c2a6f46c4607d3170950d075db68773" dependencies = [ "arrow-array", "arrow-data", @@ -404,9 +416,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" +checksum = "0d8c68b87a569f92b90c46cb0a09d86bd9609ddc9b1bf1fc5a503c8c6907c14e" dependencies = [ "arrow-array", "arrow-buffer", @@ -417,9 +429,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" +checksum = "e87acb0c6bf6237fca7a50d7b51e801c98a18a5fe6f3bb34f7d77b85ca165388" dependencies = [ "bitflags 2.13.1", "serde_core", @@ -428,9 +440,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" +checksum = "c844141958d805b47d1208e0d12aac53fcf388ce8011639fe82aa492adde0bb4" dependencies = [ "ahash", "arrow-array", @@ -442,9 +454,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" +checksum = "bc130892b1a9552da77971924b390e06cf375d1f7e4344a7b7b40ca4426fa470" dependencies = [ "arrow-array", "arrow-buffer", @@ -624,7 +636,7 @@ checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.9", ] [[package]] @@ -787,7 +799,7 @@ checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" dependencies = [ "autocfg", "libm", - "num-bigint", + "num-bigint 0.4.8", "num-integer", "num-traits", "serde", @@ -813,10 +825,11 @@ dependencies = [ [[package]] name = "bit-vec" -version = "0.9.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269" dependencies = [ + "borsh", "serde", ] @@ -950,6 +963,30 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "brotli" version = "8.0.4" @@ -1333,12 +1370,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core_detect" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" - [[package]] name = "cpubits" version = "0.1.1" @@ -1563,9 +1594,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" +checksum = "4cd5317db77ba6bca25c761a8091b9d294a9f4188e4f9dcd0243e6a57de862ac" dependencies = [ "arrow", "arrow-schema", @@ -1600,7 +1631,7 @@ dependencies = [ "flate2", "futures", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -1616,9 +1647,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" +checksum = "f35d36794509b7bfcbf90e11a1b5e4d047123c34610105e2e80a2048351cdb2f" dependencies = [ "arrow", "async-trait", @@ -1632,7 +1663,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -1641,9 +1672,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" +checksum = "a6454bfb7f43b83942e2ca2576fd09342eedbd0d323e1eef72c3c8e4d57d7203" dependencies = [ "arrow", "async-trait", @@ -1657,16 +1688,17 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", + "percent-encoding", ] [[package]] name = "datafusion-common" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" +checksum = "d26626e12b224b83351239465dcd00f72d47b02fb5e5b65a643e2dc8ed37f687" dependencies = [ "arrow", "arrow-ipc", @@ -1676,9 +1708,10 @@ dependencies = [ "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "libc", "log", + "num-traits", "object_store", "parquet", "recursive", @@ -1690,9 +1723,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" +checksum = "a106420cbfa980f242c22243ae9a6cce661d4d86ce96cff87dacabf7754a4470" dependencies = [ "futures", "log", @@ -1701,9 +1734,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" +checksum = "210aa1e9525e9454f2399dbb8b0b95510145bd8c1fdc35de74e9ccbf81bcb699" dependencies = [ "arrow", "async-compression", @@ -1719,11 +1752,12 @@ dependencies = [ "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "flate2", "futures", "glob", - "itertools 0.14.0", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -1737,9 +1771,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" +checksum = "65b1b041688cad707fa103c3e7f4c6d89b3134ebf67da0129070dad3381851eb" dependencies = [ "arrow", "arrow-ipc", @@ -1752,18 +1786,19 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "object_store", "tokio", ] [[package]] name = "datafusion-datasource-csv" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" +checksum = "2a7b34e253c4fa5251f174977af910a2f2ec831ec39dbbb56b29ce505cff278b" dependencies = [ "arrow", "async-trait", @@ -1775,6 +1810,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", "object_store", @@ -1784,9 +1820,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" +checksum = "1663338f43be6a5746473008e2d0fbde282577ddf0bae90d6c7386e988d71b29" dependencies = [ "arrow", "async-trait", @@ -1798,6 +1834,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", "object_store", @@ -1807,11 +1844,12 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" +checksum = "7c521ae91b6d899d7190812881f45fe9463c222c11d90149f5332a2e56555904" dependencies = [ "arrow", + "arrow-schema", "async-trait", "bytes", "datafusion-common", @@ -1825,10 +1863,11 @@ dependencies = [ "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-pruning", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -1838,19 +1877,20 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" +checksum = "0c2569c2e74a34fcae3a369ab3912d61e41a062592510f11e31b1dae2725ecdc" [[package]] name = "datafusion-execution" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" +checksum = "39cfdd5ac0769d1e74a885450e49fde8f63ca7ab41eb479a068564aaea80b837" dependencies = [ "arrow", "arrow-buffer", "async-trait", + "bytes", "dashmap", "datafusion-common", "datafusion-expr", @@ -1859,16 +1899,19 @@ dependencies = [ "log", "object_store", "parking_lot", + "pin-project-lite", "rand 0.9.5", "tempfile", + "tokio", + "tokio-util", "url", ] [[package]] name = "datafusion-expr" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" +checksum = "86560fd7df4dda924f5474b588c56e7ae8e96dcf158c7327c9cfe9daff448ec5" dependencies = [ "arrow", "arrow-schema", @@ -1880,30 +1923,33 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "recursive", + "serde", "serde_json", "sqlparser", ] [[package]] name = "datafusion-expr-common" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" +checksum = "e82e313cf655cf6bc2dce6cbd23bcb0d20236a54a1b09c436615e4bac4d8c49f" dependencies = [ "arrow", "datafusion-common", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-ffi" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5660e8fa79fd51e29ce46f3026b67317ef738ebd633e106beb1a1907a406152" +checksum = "58a2da6b361eae9452cf8e2cafadb05b44501a4d965b26760327cb18e0d26c4f" dependencies = [ "arrow", "arrow-schema", @@ -1934,13 +1980,13 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" +checksum = "939ae2771da8a3fdc23ab4de610ae8fef9515476626b49d96d0d5c782b105771" dependencies = [ "arrow", "arrow-buffer", - "base64 0.22.1", + "base64 0.23.1", "blake2", "blake3", "chrono", @@ -1953,7 +1999,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hex", - "itertools 0.14.0", + "itertools 0.15.0", "log", "md-5 0.11.0", "memchr", @@ -1966,9 +2012,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" +checksum = "fd6c3aee9f2bc65716787d651e023743a90b4efbc57a2f679f036632e16fb110" dependencies = [ "arrow", "datafusion-common", @@ -1979,17 +2025,17 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", - "foldhash 0.2.0", "half", + "hashbrown 0.17.1", "log", "num-traits", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" +checksum = "988b5d3dfb4939d81c2f9fab267063610fe52a843620f2fa6cb2a1598e43369c" dependencies = [ "arrow", "datafusion-common", @@ -1999,9 +2045,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" +checksum = "08149d433a97c9ea3cdf80de06480ab5c7f5453485f1e452d5b3583c81d5c0dd" dependencies = [ "arrow", "arrow-ord", @@ -2016,7 +2062,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hashbrown 0.17.1", - "itertools 0.14.0", + "itertools 0.15.0", "itoa", "log", "memchr", @@ -2024,9 +2070,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" +checksum = "2c767567cce2782d4d0bdfa55fab010aaba0bdbe533509e9e0b9a6a3ee29ed59" dependencies = [ "arrow", "async-trait", @@ -2040,9 +2086,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" +checksum = "0a311c1f558d1f792f1f631c3670da12ee72a20fbf5d6ca151325d66191c03c1" dependencies = [ "arrow", "datafusion-common", @@ -2057,9 +2103,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" +checksum = "fffd54b8acf6ecef284cc0677e01e1b8c71be64fec50612ba00a0ff0d9b41a52" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2067,20 +2113,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" +checksum = "14d6df9763c8fb968b2417d797aef4c55ac2e4a895aaef38c72c16a287aa4a40" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] name = "datafusion-optimizer" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" +checksum = "acfb8cea6e05e2e75bae183340cf7d53cccb3db0449e2f1fbc07654d980fd19f" dependencies = [ "arrow", "chrono", @@ -2089,7 +2135,7 @@ dependencies = [ "datafusion-expr-common", "datafusion-physical-expr", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "recursive", "regex", @@ -2098,20 +2144,22 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" +checksum = "9f002cef849e10c0597a217ec7891c2796c0f8b42d739f5d24abc24496373406" dependencies = [ "arrow", + "arrow-schema", "datafusion-common", "datafusion-expr", "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", + "datafusion-proto-models", "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "petgraph", "recursive", @@ -2120,9 +2168,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" +checksum = "82761522c4536e7059d3b185cd99f10525c2bf48f1bd26a0a57e73e19eadc0fb" dependencies = [ "arrow", "datafusion-common", @@ -2130,31 +2178,32 @@ dependencies = [ "datafusion-functions", "datafusion-physical-expr", "datafusion-physical-expr-common", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-physical-expr-common" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" +checksum = "f8c5c67aa252886f4d6b331c8114797f5b140464e774c4da36889269a1bf1bd8" dependencies = [ "arrow", "chrono", "datafusion-common", "datafusion-expr-common", + "datafusion-proto-models", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" +checksum = "f1a49dd3a76e4879012682c59dc3735282e60640c97483546a4113564a60bf8f" dependencies = [ "arrow", "datafusion-common", @@ -2165,15 +2214,16 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", - "itertools 0.14.0", + "datafusion-session", + "itertools 0.15.0", "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" +checksum = "b1451d6817e5ac0dd21819b19949d1c420ab51d748f55dd9a96ecb5418a24c97" dependencies = [ "arrow", "arrow-data", @@ -2181,6 +2231,7 @@ dependencies = [ "arrow-ord", "arrow-schema", "async-trait", + "bytes", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", @@ -2190,26 +2241,29 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", "futures", "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "num-traits", "parking_lot", "pin-project-lite", + "serde", + "serde_json", "tokio", ] [[package]] name = "datafusion-proto" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae" +checksum = "481881a3422072554c8c26a177a4f0a67f4d8c0e2a394a79e551e6bd9a5cec4b" dependencies = [ "arrow", - "chrono", "datafusion-catalog", "datafusion-catalog-listing", "datafusion-common", @@ -2225,26 +2279,38 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-proto-common", + "datafusion-proto-models", "object_store", "prost 0.14.4", ] [[package]] name = "datafusion-proto-common" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530" +checksum = "a95763d5fa64a88b658ba4ec7314d9c538e2aea749dc88d5ce2afaafbb4c69b8" dependencies = [ "arrow", "datafusion-common", "prost 0.14.4", ] +[[package]] +name = "datafusion-proto-models" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c4982d05dee1e0b90b7f3b39845dca7eee5635c550db09158d385b8181bdc6a" +dependencies = [ + "datafusion-common", + "datafusion-proto-common", + "prost 0.14.4", +] + [[package]] name = "datafusion-pruning" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" +checksum = "f11f0dbe2b5a642f6106e44de8bbd6654fa7220ca14f32313d4efd5514a86968" dependencies = [ "arrow", "datafusion-common", @@ -2258,10 +2324,11 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" +checksum = "62dc5ce37a5ce1ff5362949776829aded2529a11dbaa13d24b0ffa6a059e106a" dependencies = [ + "arrow-schema", "async-trait", "datafusion-common", "datafusion-execution", @@ -2272,23 +2339,22 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957" +checksum = "0c3d6f76463984c4ed8587c52ed4f21f142fd96fa55d8f128f673f2db4c46942" dependencies = [ "arrow", "bigdecimal", "chrono", "crc32fast", "datafusion", - "datafusion-catalog", "datafusion-common", - "datafusion-execution", "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-aggregate-common", "datafusion-functions-nested", + "datafusion-session", "log", "num-traits", "percent-encoding", @@ -2302,9 +2368,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "54.0.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" +checksum = "816c224a0ef5469a32a562b5d4830544f714f97c5f50142109286d40a288c7db" dependencies = [ "arrow", "bigdecimal", @@ -2317,6 +2383,7 @@ dependencies = [ "recursive", "regex", "sqlparser", + "stacker", ] [[package]] @@ -2597,14 +2664,26 @@ checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" [[package]] name = "fastlanes" -version = "0.5.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9272a674b446f53253f66582596614e41552d14968a3d1aaa590576a91c1188" +checksum = "4a516528425408429de7661e7101f7e7ad54f7d3fab859a3a6de239f508d17e2" dependencies = [ "const_for", - "core_detect", "num-traits", - "paste", + "pastey", + "seq-macro", +] + +[[package]] +name = "fastlanes" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcb34a73abd2872401096bf65d13e8d26e0c9ca6166d5e2b11a314ba017cdef" +dependencies = [ + "const_for", + "cpufeatures 0.3.0", + "num-traits", + "pastey", "seq-macro", ] @@ -2614,6 +2693,23 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "fearless_simd" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629c9d6b6f30b5604089e0980f4d924a013aacb1654a6cdf5d3d852224f68651" + +[[package]] +name = "fearless_simd_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e91d94c6885390b81782d7ee60fd6583b33907333a8b340de25ccf468353315" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "filetime" version = "0.2.29" @@ -2717,9 +2813,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst-rs" -version = "0.5.11" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b13ac798afc0d9194eb4efefef8b9332efbd80b43f302a968cb8cb23b9d5360" +checksum = "03b614a2a7f2efc50d76c1e47d64b57c55acf6e0e718b0cae78236d147c487d0" dependencies = [ "rustc-hash", ] @@ -3443,12 +3539,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "inventory" version = "0.3.24" @@ -3488,6 +3578,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -3916,9 +4015,6 @@ name = "lz4_flex" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" -dependencies = [ - "twox-hash", -] [[package]] name = "lz4_flex" @@ -4179,7 +4275,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ - "num-bigint", + "num-bigint 0.4.8", "num-complex", "num-integer", "num-iter", @@ -4198,6 +4294,16 @@ dependencies = [ "serde", ] +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint-dig" version = "0.8.6" @@ -4254,7 +4360,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "num-bigint", + "num-bigint 0.4.8", "num-integer", "num-traits", ] @@ -4365,10 +4471,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" [[package]] -name = "oneshot" +name = "onpair" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe21416a02c693fb9f980befcb230ecc70b0b3d1cc4abf88b9675c4c1457f0c" +checksum = "34272d1ea63266148a9a5b2b96db80a45245d034862302735f2f93413a6f9172" +dependencies = [ + "hashbrown 0.16.1", + "rand 0.9.5", +] [[package]] name = "opendal-core" @@ -4619,9 +4729,9 @@ dependencies = [ [[package]] name = "orc-rust" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b9867c4e7343218682ba11aceae1310c80735ec2beaf6124a0f8f848dad197" +checksum = "e54c163f843c4f0fcfca65b9b73ca6f93c37c8e86736d5a1e578c6e843eed0a0" dependencies = [ "arrow", "async-trait", @@ -4644,15 +4754,6 @@ dependencies = [ "zstd 0.13.3", ] -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - [[package]] name = "ordered-float" version = "5.3.0" @@ -4725,7 +4826,7 @@ dependencies = [ "lz4_flex 0.14.0", "lzokay-native", "md-5 0.10.6", - "num-bigint", + "num-bigint 0.4.8", "num-traits", "opendal-core", "opendal-http-transport-reqwest", @@ -4859,13 +4960,14 @@ dependencies = [ [[package]] name = "paimon-mosaic-core" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d2799ba43edf953feaf552dc48aee63c5923f774b7277473f4884e055442de" +checksum = "6127799590f9e9d6c552d73690b79e60b43bba5a72a949c9cd44e2064ec6d245" dependencies = [ "arrow-array", "arrow-buffer", "arrow-schema", + "libc", "zstd 0.13.3", ] @@ -4947,9 +5049,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.4.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" +checksum = "8eb260d566f2fbdabe356d26aaa5d13403aac29b5b4e44463b8f9bd7b2b12680" dependencies = [ "ahash", "arrow-array", @@ -4958,7 +5060,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.22.1", + "base64 0.23.1", "brotli", "bytes", "chrono", @@ -4966,27 +5068,78 @@ dependencies = [ "futures", "half", "hashbrown 0.17.1", - "lz4_flex 0.13.1", - "num-bigint", + "lz4_flex 0.14.0", + "num-bigint 0.5.1", "num-integer", "num-traits", "object_store", - "paste", "seq-macro", "simdutf8", "snap", - "thrift", "tokio", "twox-hash", "zstd 0.13.3", ] +[[package]] +name = "parquet-variant" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d0085fde95ccfae0b8d703acd1245bda8855a0bf996eba848e8e2f9995f8606" +dependencies = [ + "arrow", + "arrow-schema", + "chrono", + "half", + "indexmap 2.14.0", + "num-traits", + "simdutf8", + "uuid", +] + +[[package]] +name = "parquet-variant-compute" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1f9036efdea8177c84895f409a442e3960a8ccc642792760d8d48415542bb3a" +dependencies = [ + "arrow", + "arrow-schema", + "chrono", + "half", + "indexmap 2.14.0", + "parquet-variant", + "parquet-variant-json", + "serde_json", + "uuid", +] + +[[package]] +name = "parquet-variant-json" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50ae7a6fdff86cdd5075653039b62e2ba674b0aae9228022cd74dc9cf71d7353" +dependencies = [ + "arrow-schema", + "base64 0.23.1", + "chrono", + "parquet-variant", + "serde_json", + "uuid", +] + [[package]] name = "paste" version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "pbkdf2" version = "0.12.2" @@ -5370,9 +5523,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.28.3" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +checksum = "700d18fa267b73b9b521fd7e13580e2f446916f176cacee1ab63fcc8191f1655" dependencies = [ "chrono", "libc", @@ -5385,18 +5538,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.3" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +checksum = "7b3fc0c4d08f6bb10e71fe39dfb9e2f59c6eb6854e22ec8092f50c69a4499adb" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.28.3" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +checksum = "dfc0b8e19df29aad7086cf977bb0c2a2f143e30567eb113e9cf72b62ca698330" dependencies = [ "libc", "pyo3-build-config", @@ -5404,9 +5557,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.3" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" +checksum = "6100e8a4b5eba53afaa5ed078364851a0b2499c553a44c31026b929049b49dc6" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -5416,13 +5569,12 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.28.3" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" +checksum = "6143877a16e82b5a727b7127ff4cd86858a24a28d745d72f43e6f227c7b1bdb3" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", "quote", "syn 2.0.119", ] @@ -5724,7 +5876,7 @@ checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.9", ] [[package]] @@ -6341,7 +6493,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.9", ] [[package]] @@ -6376,7 +6528,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.9", ] [[package]] @@ -6799,9 +6951,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.2" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" dependencies = [ "proc-macro2", "quote", @@ -6884,7 +7036,7 @@ dependencies = [ "measure_time", "memmap2", "once_cell", - "oneshot 0.1.13", + "oneshot", "rayon", "regex", "rust-stemmers", @@ -6976,7 +7128,7 @@ checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82" dependencies = [ "fnv", "nom", - "ordered-float 5.3.0", + "ordered-float", "serde", "serde_json", ] @@ -7093,18 +7245,7 @@ checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", -] - -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "ordered-float 2.10.1", + "syn 3.0.9", ] [[package]] @@ -7425,7 +7566,7 @@ checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.9", ] [[package]] @@ -7508,79 +7649,72 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vortex" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b722c8baa7328544d3ce22763321a9be97a75f11acdb630bb71abbe61def2a" +checksum = "00e778cec317a66ad43111508c9dfcc782af9bbac8ace68097651d687f864204" dependencies = [ "vortex-alp", "vortex-array", + "vortex-arrow", "vortex-btrblocks", "vortex-buffer", "vortex-bytebool", "vortex-datetime-parts", "vortex-decimal-byte-parts", + "vortex-edition", "vortex-error", "vortex-fastlanes", "vortex-file", - "vortex-flatbuffers", "vortex-fsst", "vortex-io", "vortex-ipc", "vortex-layout", "vortex-mask", "vortex-metrics", + "vortex-parquet-variant", "vortex-pco", - "vortex-proto", "vortex-runend", "vortex-scan", "vortex-sequence", "vortex-session", "vortex-sparse", "vortex-utils", + "vortex-uuid", "vortex-zigzag", "vortex-zstd", ] [[package]] name = "vortex-alp" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f1c242a300d0bef65ebcbaaf0086ccff778ac45e62f0bdcefbfcea5b078c979" +checksum = "1c0f717b7cc81f1cd0e94f017c04d6c91ffc9240143d14830415aad948e1e90e" dependencies = [ + "alp", "itertools 0.14.0", "num-traits", "prost 0.14.4", - "rustc-hash", "vortex-array", "vortex-buffer", "vortex-error", "vortex-fastlanes", "vortex-mask", "vortex-session", - "vortex-utils", ] [[package]] name = "vortex-array" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d33d2bed6547a9a30ddd4ae35259ffaca66bd3deac734f340fa4f98c046eb297" +checksum = "454098724288d8e27e99e0c8bd8875570ea7ce6add928024e701e99bd936d9bb" dependencies = [ - "arc-swap", "arcref", - "arrow-arith", - "arrow-array", "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ord", - "arrow-schema", - "arrow-select", - "arrow-string", - "async-lock", "bytes", "cfg-if", "enum-iterator", + "fearless_simd", + "fearless_simd_macros", "flatbuffers", "futures", "half", @@ -7588,49 +7722,73 @@ dependencies = [ "inventory", "itertools 0.14.0", "jiff", + "memchr", "num-traits", "num_enum", "parking_lot", "paste", "pin-project-lite", "prost 0.14.4", + "prost-types", "rand 0.10.2", + "regex", + "regex-syntax", "rustc-hash", "simdutf8", "smallvec", "static_assertions", - "termtree", "tracing", - "uuid", "vortex-array-macros", "vortex-buffer", + "vortex-compute", "vortex-error", - "vortex-flatbuffers", "vortex-mask", - "vortex-proto", "vortex-session", "vortex-utils", ] [[package]] name = "vortex-array-macros" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57dba54aebd8dc42230bb4345ede660741c34fc10439fdb1369824cb86bae907" +checksum = "a2210cabfd84df126829b7607c09a16ebce7eb79fe9f346567f9321e24e787a8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] -name = "vortex-btrblocks" -version = "0.75.0" +name = "vortex-arrow" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "110427e1a65221519c8127e92a8080ee922b5f76a311568d227fd227bfbdfa10" +checksum = "1647f385ca0a718f4722fc6cdfa54dde716b2172a36d9f49ba812eb6be149b1d" dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", "itertools 0.14.0", "num-traits", + "simdutf8", + "tracing", + "vortex-array", + "vortex-buffer", + "vortex-error", + "vortex-mask", + "vortex-runend", + "vortex-session", +] + +[[package]] +name = "vortex-btrblocks" +version = "0.90.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1138d084c1b3988936312ba65262caafcfedeb1991c9937dabeb1051cab6dfb" +dependencies = [ + "itertools 0.14.0", "pco", "rand 0.10.2", "vortex-alp", @@ -7639,12 +7797,15 @@ dependencies = [ "vortex-compressor", "vortex-datetime-parts", "vortex-decimal-byte-parts", + "vortex-edition", "vortex-error", "vortex-fastlanes", "vortex-fsst", + "vortex-onpair", "vortex-pco", "vortex-runend", "vortex-sequence", + "vortex-session", "vortex-sparse", "vortex-utils", "vortex-zigzag", @@ -7653,37 +7814,40 @@ dependencies = [ [[package]] name = "vortex-buffer" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8da47e7e8afbaa18e4471a8f99f7c33e909d07f1e9cdfeae6df3c1e4f71070" +checksum = "db52df963f2caa4380abb5ed4405476b2fb5d1675ae60b35cb80791a2de209dc" dependencies = [ + "allocator-api2", "arrow-buffer", "bitvec", "bytes", + "fearless_simd", + "fearless_simd_macros", "itertools 0.14.0", + "num-traits", "simdutf8", "vortex-error", ] [[package]] name = "vortex-bytebool" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b26f84e77f32726ec826ac70582b37e34d89bf8f007dc0b3426a188d823f45a5" +checksum = "32269208b92c2aa76ee81fe629e0bc45698d8cca051712a97b5e6326b31a1f67" dependencies = [ "num-traits", "vortex-array", "vortex-buffer", "vortex-error", - "vortex-mask", "vortex-session", ] [[package]] name = "vortex-compressor" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daa039a03a9350e8ca3219bd141e852a139c0a89811c3a7caecc22fcc8d07d35" +checksum = "80d16c2cb63ce298a3a2858e43f7ae29105fd74b86bcecb52311f3a9fa8ae1b0" dependencies = [ "itertools 0.14.0", "num-traits", @@ -7698,16 +7862,26 @@ dependencies = [ "vortex-utils", ] +[[package]] +name = "vortex-compute" +version = "0.90.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1b6bb64c801e393f509a17c2afcea639f04baed3eca96af76d2b41bef91b870" +dependencies = [ + "vortex-buffer", +] + [[package]] name = "vortex-datetime-parts" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bade3b947e68b638ad184e24b1367c3146e394e4dc837486444eb0bab7cf578" +checksum = "a67e541dca82970bf5ea3ad5ea47b02ab46f60cd53a2ada94e561aeabef74d2d" dependencies = [ "num-traits", "prost 0.14.4", "vortex-array", "vortex-buffer", + "vortex-compute", "vortex-error", "vortex-mask", "vortex-session", @@ -7715,9 +7889,9 @@ dependencies = [ [[package]] name = "vortex-decimal-byte-parts" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196c22cb318d0cf2a67eb7cf9b93550eb394a0f4f56775c119abbc3fa5e9c4c2" +checksum = "08e3371e16e9ee8cc24734d9e9e67cf8013ec2a4a443fe499916edb11aba780b" dependencies = [ "num-traits", "prost 0.14.4", @@ -7728,11 +7902,22 @@ dependencies = [ "vortex-session", ] +[[package]] +name = "vortex-edition" +version = "0.90.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecf0598d26fffb33ed48cb4d5e2ad8cf7a11ac929965d20a6959047c65ba7d1" +dependencies = [ + "parking_lot", + "vortex-error", + "vortex-session", +] + [[package]] name = "vortex-error" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80158c6f4397a08dc27c14ec46c3ffbecf18d0690841e5ffe9aef2d3a0a241ab" +checksum = "ade3532eb43eb26ccbf0a648165b2e2c3e8f64843b0648744bac9f51aa08a18c" dependencies = [ "arrow-schema", "flatbuffers", @@ -7743,17 +7928,18 @@ dependencies = [ [[package]] name = "vortex-fastlanes" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312a4446fbd20cbbb395a16ec3428d8b113c9b1188c51fd4c0b6265ccc487149" +checksum = "85b39609df49674c7df3841fba541cd61775d15a07aaad2fb124821d0ab44892" dependencies = [ - "fastlanes", + "fastlanes 0.7.2", "itertools 0.14.0", "lending-iterator", "num-traits", "prost 0.14.4", "vortex-array", "vortex-buffer", + "vortex-compute", "vortex-error", "vortex-mask", "vortex-session", @@ -7761,9 +7947,9 @@ dependencies = [ [[package]] name = "vortex-file" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1b30147596131d56ebaeb0838372faa09621302e916da553e4413aad96ce0c" +checksum = "225844b04952557b9ca8622867fd2f0749a2577385148b695f71d9f9f0a9af1d" dependencies = [ "async-trait", "bytes", @@ -7772,11 +7958,11 @@ dependencies = [ "itertools 0.14.0", "kanal", "moka", - "oneshot 0.2.1", "parking_lot", "pin-project-lite", "tokio", "tracing", + "url", "vortex-alp", "vortex-array", "vortex-btrblocks", @@ -7784,14 +7970,15 @@ dependencies = [ "vortex-bytebool", "vortex-datetime-parts", "vortex-decimal-byte-parts", + "vortex-edition", "vortex-error", "vortex-fastlanes", - "vortex-flatbuffers", "vortex-fsst", "vortex-io", "vortex-layout", "vortex-mask", "vortex-metrics", + "vortex-onpair", "vortex-pco", "vortex-runend", "vortex-scan", @@ -7803,24 +7990,14 @@ dependencies = [ "vortex-zstd", ] -[[package]] -name = "vortex-flatbuffers" -version = "0.75.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ecc9db16991f005079050cd2245dac110eb2cc321e9c087a530acc09b423bb0" -dependencies = [ - "flatbuffers", - "vortex-buffer", - "vortex-error", -] - [[package]] name = "vortex-fsst" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "570382994bd8bc21836550f04427a8d85f419f9e897c9ec042c1486ab34aa063" +checksum = "e2a198ab5fcd335db05dab36506ae37988acde7e916d4bbf3cbac6519ae6ae27" dependencies = [ "fsst-rs", + "num-traits", "prost 0.14.4", "vortex-array", "vortex-buffer", @@ -7831,19 +8008,20 @@ dependencies = [ [[package]] name = "vortex-io" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1747e9615151214cd31ec00a854cd6dd4bf677d9f649b5b32f0d4d5f40c1e268" +checksum = "2157da7dbfa60b1303616b939e90f54faf4a02a777c245ed2d0a227edc42e5d4" dependencies = [ + "async-executor", "async-fs", "async-stream", "async-trait", "bytes", + "crossbeam-channel", "custom-labels", "futures", "glob", "kanal", - "oneshot 0.2.1", "parking_lot", "pin-project-lite", "smol", @@ -7860,9 +8038,9 @@ dependencies = [ [[package]] name = "vortex-ipc" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4bd60c8735ea6e0f71a27a16f44f9fbb8d293c341d30ad6f042103a2b4808b8" +checksum = "e8119836dbbfde9243895fc915eb64d17b4d52a63a48b5746fd17e112601c907" dependencies = [ "bytes", "flatbuffers", @@ -7872,19 +8050,35 @@ dependencies = [ "vortex-array", "vortex-buffer", "vortex-error", - "vortex-flatbuffers", + "vortex-session", +] + +[[package]] +name = "vortex-json" +version = "0.90.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab575eaab239140d1dce28783bffa8ec96642b5b7ca8370e311bd6f9a4930e3" +dependencies = [ + "arrow-array", + "arrow-schema", + "prost 0.14.4", + "vortex-array", + "vortex-arrow", + "vortex-edition", + "vortex-error", "vortex-session", ] [[package]] name = "vortex-layout" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "526570e7cfdf252798015e27cf175f2cac40aae1e164337fd382501193ec4274" +checksum = "2ae8a8e5da4d59590c5db1d580988c74619bbfe39cd18df47660f2fcbf6f4f7e" dependencies = [ "arcref", "arrow-array", "arrow-schema", + "async-lock", "async-stream", "async-trait", "bit-vec", @@ -7894,21 +8088,24 @@ dependencies = [ "kanal", "moka", "once_cell", - "oneshot 0.2.1", "parking_lot", "paste", "pin-project-lite", "prost 0.14.4", + "roaring", "rustc-hash", "sketches-ddsketch", + "smallvec", "termtree", "tokio", "tracing", + "twox-hash", + "uuid", "vortex-array", + "vortex-arrow", "vortex-btrblocks", "vortex-buffer", "vortex-error", - "vortex-flatbuffers", "vortex-io", "vortex-mask", "vortex-metrics", @@ -7921,9 +8118,9 @@ dependencies = [ [[package]] name = "vortex-mask" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5b43ee2a50c722d9abdaed454e836b044a3a69acbb5f5d95e28b89e520b11d9" +checksum = "d7b17f6601def0aad3bebebb175219f09b59c2e137860962af078e4e55e52e9c" dependencies = [ "itertools 0.14.0", "vortex-buffer", @@ -7932,21 +8129,22 @@ dependencies = [ [[package]] name = "vortex-metrics" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e079c9fe032af05b2e38bf91e63d2bdf36bd01e3920bc4a352648b2b21e9f70" +checksum = "8f0d03b9b48c72005f71f60c7134d1fcac3985e034b914a56f54e2a223097bb7" dependencies = [ "parking_lot", "sketches-ddsketch", ] [[package]] -name = "vortex-pco" -version = "0.75.0" +name = "vortex-onpair" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8902b2df3068b37267c927cdec8905ea4e0e693d81274ba9713000929d1e514e" +checksum = "d7b86706bb011bd76084e5fb0f34137a10fa210a8364b28cdfb53dd7222a2ab3" dependencies = [ - "pco", + "num-traits", + "onpair", "prost 0.14.4", "vortex-array", "vortex-buffer", @@ -7956,22 +8154,48 @@ dependencies = [ ] [[package]] -name = "vortex-proto" -version = "0.75.0" +name = "vortex-parquet-variant" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f018abac72332c320ba3b8b59b6ce72f6c8a6caf0552b0b5c255f336b7f502" +checksum = "253d55452deb61da604207bbd557de421a962c933197682cfe4ee72a3e9934e2" dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "chrono", + "parquet-variant", + "parquet-variant-compute", "prost 0.14.4", - "prost-types", + "vortex-array", + "vortex-arrow", + "vortex-buffer", + "vortex-error", + "vortex-json", + "vortex-mask", + "vortex-session", +] + +[[package]] +name = "vortex-pco" +version = "0.90.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d54b4a848b899a70fcbbd86da1e20b4ec0527722a08dbf3cf000610f7f02260" +dependencies = [ + "pco", + "prost 0.14.4", + "vortex-array", + "vortex-buffer", + "vortex-error", + "vortex-mask", + "vortex-session", ] [[package]] name = "vortex-runend" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73dd8c96206fd4c0110e9ac304fd41e78f6e85659975c9e63911dddd982de5fd" +checksum = "2828cdf7c3096227136ca26b93c1ec5b55c4b74f45a2bff0ea98ec334adf2bd1" dependencies = [ - "arrow-array", "itertools 0.14.0", "num-traits", "prost 0.14.4", @@ -7984,9 +8208,9 @@ dependencies = [ [[package]] name = "vortex-scan" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34deb979fc801bbc0e015ceccca1870f1dbbc72426fc040d277ea86cd9df5fd" +checksum = "c966f342f8251c796dee968ad1a88221f9bf1e94fda1744b5094ab6e05c6b3b3" dependencies = [ "async-trait", "futures", @@ -7995,34 +8219,36 @@ dependencies = [ "vortex-array", "vortex-buffer", "vortex-error", + "vortex-io", "vortex-mask", "vortex-session", + "vortex-utils", ] [[package]] name = "vortex-sequence" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "933091989818d88836430fb68ca0b3ef9e054bf2740f94051377231cc287d427" +checksum = "f0913adc59e97fb965b9bcf5feee11bf090027b7c7b711494c4b72282365e1bd" dependencies = [ "num-traits", "prost 0.14.4", "smallvec", "vortex-array", "vortex-buffer", + "vortex-compute", "vortex-error", "vortex-mask", - "vortex-proto", "vortex-session", ] [[package]] name = "vortex-session" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a62810f3a266cdd8bc4812235b8d2b1b586a909dc69107ba51f4dbfee01d9f2" +checksum = "03fcffcfac13c4d854f836dc584d1611cc8aba473e728f804c0607e58ee88238" dependencies = [ - "dashmap", + "arc-swap", "lasso", "parking_lot", "vortex-error", @@ -8031,9 +8257,9 @@ dependencies = [ [[package]] name = "vortex-sparse" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe0d321f522d325a811bfc6506b320db12f57bd7710d558520d75dd7313b9dd1" +checksum = "b9c445dddd2c49ce562444382be4d4cac8a36d2832a0a1b2728acbd4b74b107a" dependencies = [ "itertools 0.14.0", "num-traits", @@ -8047,23 +8273,39 @@ dependencies = [ [[package]] name = "vortex-utils" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebe05be4a3e14c562c80f78cbdec0e5a05030e4d1299a6ca6609441758414202" +checksum = "8fd7be99f740319f84ddd5176535b3616172c99fb7b3e3f2301ec7eb8de053d1" dependencies = [ "dashmap", "hashbrown 0.17.1", +] + +[[package]] +name = "vortex-uuid" +version = "0.90.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3252fb4740197481d86249581a6fe536db6879af5d17a64fdd82a624d16245e" +dependencies = [ + "arrow-array", + "arrow-schema", + "uuid", + "vortex-array", + "vortex-arrow", + "vortex-buffer", "vortex-error", + "vortex-session", ] [[package]] name = "vortex-zigzag" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6635049187e7e46413fc2d760a2836b73c0b1ffcbbc54b657b6c338a5551c5c6" +checksum = "83e07baa4d5952093dbd271cda75ae01e787ecdca1e59394519b30411ed28303" dependencies = [ "vortex-array", "vortex-buffer", + "vortex-compute", "vortex-error", "vortex-mask", "vortex-session", @@ -8072,14 +8314,16 @@ dependencies = [ [[package]] name = "vortex-zstd" -version = "0.75.0" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dc464ba75cff8510a7f31e8c5aa7ca4f3ce34d15e87e80a91880d3afa7458df" +checksum = "16ed140fdf797d28c36b4d4b73223a7874d794ecd3c9cd3a79e16977bd6b03b4" dependencies = [ "itertools 0.14.0", + "num-traits", "prost 0.14.4", "vortex-array", "vortex-buffer", + "vortex-edition", "vortex-error", "vortex-mask", "vortex-session", diff --git a/Cargo.toml b/Cargo.toml index 95f5d337a..6574b5754 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,7 @@ arrow-select = "59.2" arrow-string = "59.2" datafusion = "55.2.0" datafusion-ffi = "55.2.0" -datafusion-spark = { version = "55.0.0", features = ["core"] } +datafusion-spark = { version = "55.2.0", features = ["core"] } paimon = { version = "0.6.0", path = "crates/paimon" } parquet = "59.2" constant_time_eq = ">=0.4.0, <0.5.0" diff --git a/DEPENDENCIES.rust.tsv b/DEPENDENCIES.rust.tsv index 810dccaed..0e39b53ff 100644 --- a/DEPENDENCIES.rust.tsv +++ b/DEPENDENCIES.rust.tsv @@ -8,6 +8,7 @@ aho-corasick@1.1.4 X X alloc-no-stdlib@2.0.4 X alloc-stdlib@0.2.4 X allocator-api2@0.2.21 X X +alp@0.0.4 X android_system_properties@0.1.5 X X anstream@1.0.0 X X anstyle@1.0.14 X X @@ -22,21 +23,21 @@ arc-swap@1.9.2 X X arcref@0.2.0 X arrayref@0.3.9 X arrayvec@0.7.8 X X -arrow@58.4.0 X -arrow-arith@58.4.0 X -arrow-array@58.4.0 X X -arrow-buffer@58.4.0 X -arrow-cast@58.4.0 X -arrow-csv@58.4.0 X -arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X -arrow-ord@58.4.0 X -arrow-pyarrow@58.4.0 X -arrow-row@58.4.0 X -arrow-schema@58.4.0 X -arrow-select@58.4.0 X -arrow-string@58.4.0 X +arrow@59.3.2 X +arrow-arith@59.3.2 X +arrow-array@59.3.2 X X +arrow-buffer@59.3.2 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X +arrow-ord@59.3.2 X +arrow-pyarrow@59.3.2 X +arrow-row@59.3.2 X +arrow-schema@59.3.2 X +arrow-select@59.3.2 X +arrow-string@59.3.2 X async-channel@2.5.0 X X async-compression@0.4.42 X X async-executor@1.14.0 X X @@ -67,7 +68,7 @@ base64ct@1.8.3 X X better_io@0.2.0 X bigdecimal@0.4.10 X X bindgen@0.72.1 X -bit-vec@0.9.1 X X +bit-vec@0.10.1 X X bitflags@2.13.1 X X bitpacking@0.9.3 X bitvec@1.1.1 X @@ -122,7 +123,6 @@ constant_time_eq@0.4.2 X X X core-foundation@0.10.1 X X core-foundation@0.9.4 X X core-foundation-sys@0.8.7 X X -core_detect@1.0.0 X X cpubits@0.1.1 X X cpufeatures@0.2.17 X X cpufeatures@0.3.0 X X @@ -147,41 +147,42 @@ darling_core@0.23.0 X darling_macro@0.23.0 X dary_heap@0.3.9 X X dashmap@6.2.1 X -datafusion@54.0.0 X -datafusion-catalog@54.0.0 X -datafusion-catalog-listing@54.0.0 X -datafusion-common@54.0.0 X -datafusion-common-runtime@54.0.0 X -datafusion-datasource@54.0.0 X -datafusion-datasource-arrow@54.0.0 X -datafusion-datasource-csv@54.0.0 X -datafusion-datasource-json@54.0.0 X -datafusion-datasource-parquet@54.0.0 X -datafusion-doc@54.0.0 X -datafusion-execution@54.0.0 X -datafusion-expr@54.0.0 X -datafusion-expr-common@54.0.0 X -datafusion-ffi@54.0.0 X -datafusion-functions@54.0.0 X -datafusion-functions-aggregate@54.0.0 X -datafusion-functions-aggregate-common@54.0.0 X -datafusion-functions-nested@54.0.0 X -datafusion-functions-table@54.0.0 X -datafusion-functions-window@54.0.0 X -datafusion-functions-window-common@54.0.0 X -datafusion-macros@54.0.0 X -datafusion-optimizer@54.0.0 X -datafusion-physical-expr@54.0.0 X -datafusion-physical-expr-adapter@54.0.0 X -datafusion-physical-expr-common@54.0.0 X -datafusion-physical-optimizer@54.0.0 X -datafusion-physical-plan@54.0.0 X -datafusion-proto@54.0.0 X -datafusion-proto-common@54.0.0 X -datafusion-pruning@54.0.0 X -datafusion-session@54.0.0 X -datafusion-spark@54.0.0 X -datafusion-sql@54.0.0 X +datafusion@55.2.0 X +datafusion-catalog@55.2.0 X +datafusion-catalog-listing@55.2.0 X +datafusion-common@55.2.0 X +datafusion-common-runtime@55.2.0 X +datafusion-datasource@55.2.0 X +datafusion-datasource-arrow@55.2.0 X +datafusion-datasource-csv@55.2.0 X +datafusion-datasource-json@55.2.0 X +datafusion-datasource-parquet@55.2.0 X +datafusion-doc@55.2.0 X +datafusion-execution@55.2.0 X +datafusion-expr@55.2.0 X +datafusion-expr-common@55.2.0 X +datafusion-ffi@55.2.0 X +datafusion-functions@55.2.0 X +datafusion-functions-aggregate@55.2.0 X +datafusion-functions-aggregate-common@55.2.0 X +datafusion-functions-nested@55.2.0 X +datafusion-functions-table@55.2.0 X +datafusion-functions-window@55.2.0 X +datafusion-functions-window-common@55.2.0 X +datafusion-macros@55.2.0 X +datafusion-optimizer@55.2.0 X +datafusion-physical-expr@55.2.0 X +datafusion-physical-expr-adapter@55.2.0 X +datafusion-physical-expr-common@55.2.0 X +datafusion-physical-optimizer@55.2.0 X +datafusion-physical-plan@55.2.0 X +datafusion-proto@55.2.0 X +datafusion-proto-common@55.2.0 X +datafusion-proto-models@55.2.0 X +datafusion-pruning@55.2.0 X +datafusion-session@55.2.0 X +datafusion-spark@55.2.0 X +datafusion-sql@55.2.0 X datasketches@0.2.0 X der@0.7.10 X X deranged@0.5.8 X X @@ -209,8 +210,11 @@ ext-trait-proc_macros@1.0.1 X X X extension-traits@1.0.1 X X X fallible-streaming-iterator@0.1.9 X X fastdivide@0.4.2 X X -fastlanes@0.5.2 X +fastlanes@0.6.1 X +fastlanes@0.7.2 X fastrand@2.5.0 X X +fearless_simd@1.2.0 X X +fearless_simd_macros@0.1.0 X X find-msvc-tools@0.1.9 X X fixedbitset@0.5.7 X X flatbuffers@25.12.19 X @@ -223,7 +227,7 @@ foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs4@0.13.1 X X fs_extra@1.3.0 X -fsst-rs@0.5.11 X +fsst-rs@0.6.0 X funty@2.0.0 X futures@0.3.33 X X futures-channel@0.3.33 X X @@ -287,12 +291,12 @@ include-flate-compress@0.3.4 X indexmap@2.14.0 X X inout@0.1.4 X X inout@0.2.2 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X is_terminal_polyfill@1.70.2 X X itertools@0.13.0 X X itertools@0.14.0 X X +itertools@0.15.0 X X itoa@1.0.18 X X jieba-macros@0.10.3 X jieba-rs@0.10.3 X @@ -364,6 +368,7 @@ nougat@0.2.4 X X X nougat-proc_macros@0.2.4 X X X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-bigint-dig@0.8.6 X X num-complex@0.4.6 X X num-conv@0.2.2 X X @@ -380,7 +385,7 @@ object_store@0.13.2 X X once_cell@1.21.4 X X once_cell_polyfill@1.70.2 X X oneshot@0.1.13 X X -oneshot@0.2.1 X X +onpair@0.2.1 X opendal-core@0.59.5 X opendal-http-transport-reqwest@0.59.5 X opendal-layer-retry@0.59.5 X @@ -397,8 +402,7 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X @@ -407,15 +411,19 @@ paimon-c@0.6.0 X paimon-datafusion@0.6.0 X paimon-ftindex-core@0.1.0 X paimon-integration-tests@0.6.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-rest-server@0.6.0 X paimon-tpcds-bench@0.6.0 X paimon-vindex-core@0.5.0 X parking@2.2.1 X X parking_lot@0.12.5 X X parking_lot_core@0.9.12 X X -parquet@58.4.0 X +parquet@59.3.2 X +parquet-variant@59.3.2 X +parquet-variant-compute@59.3.2 X +parquet-variant-json@59.3.2 X paste@1.0.15 X X +pastey@0.2.3 X X pbkdf2@0.12.2 X X pco@1.0.2 X pem@4.0.0 X @@ -455,11 +463,11 @@ prost-derive@0.13.5 X prost-derive@0.14.4 X prost-types@0.14.4 X psm@0.1.31 X X -pyo3@0.28.3 X X -pyo3-build-config@0.28.3 X X -pyo3-ffi@0.28.3 X X -pyo3-macros@0.28.3 X X -pyo3-macros-backend@0.28.3 X X +pyo3@0.29.3 X X +pyo3-build-config@0.29.3 X X +pyo3-ffi@0.29.3 X X +pyo3-macros@0.29.3 X X +pyo3-macros-backend@0.29.3 X X pypaimon_rust@0.6.0 X quad-rand@0.2.3 X quick-xml@0.41.0 X @@ -576,7 +584,7 @@ strum_macros@0.27.2 X subtle@2.6.1 X syn@1.0.109 X X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X @@ -600,7 +608,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X @@ -640,36 +647,41 @@ utf8parse@0.2.2 X X uuid@1.24.0 X X vcpkg@0.2.15 X X version_check@0.9.5 X X -vortex@0.75.0 X -vortex-alp@0.75.0 X -vortex-array@0.75.0 X -vortex-array-macros@0.75.0 X -vortex-btrblocks@0.75.0 X -vortex-buffer@0.75.0 X -vortex-bytebool@0.75.0 X -vortex-compressor@0.75.0 X -vortex-datetime-parts@0.75.0 X -vortex-decimal-byte-parts@0.75.0 X -vortex-error@0.75.0 X -vortex-fastlanes@0.75.0 X -vortex-file@0.75.0 X -vortex-flatbuffers@0.75.0 X -vortex-fsst@0.75.0 X -vortex-io@0.75.0 X -vortex-ipc@0.75.0 X -vortex-layout@0.75.0 X -vortex-mask@0.75.0 X -vortex-metrics@0.75.0 X -vortex-pco@0.75.0 X -vortex-proto@0.75.0 X -vortex-runend@0.75.0 X -vortex-scan@0.75.0 X -vortex-sequence@0.75.0 X -vortex-session@0.75.0 X -vortex-sparse@0.75.0 X -vortex-utils@0.75.0 X -vortex-zigzag@0.75.0 X -vortex-zstd@0.75.0 X +vortex@0.90.0 X +vortex-alp@0.90.0 X +vortex-array@0.90.0 X +vortex-array-macros@0.90.0 X +vortex-arrow@0.90.0 X +vortex-btrblocks@0.90.0 X +vortex-buffer@0.90.0 X +vortex-bytebool@0.90.0 X +vortex-compressor@0.90.0 X +vortex-compute@0.90.0 X +vortex-datetime-parts@0.90.0 X +vortex-decimal-byte-parts@0.90.0 X +vortex-edition@0.90.0 X +vortex-error@0.90.0 X +vortex-fastlanes@0.90.0 X +vortex-file@0.90.0 X +vortex-fsst@0.90.0 X +vortex-io@0.90.0 X +vortex-ipc@0.90.0 X +vortex-json@0.90.0 X +vortex-layout@0.90.0 X +vortex-mask@0.90.0 X +vortex-metrics@0.90.0 X +vortex-onpair@0.90.0 X +vortex-parquet-variant@0.90.0 X +vortex-pco@0.90.0 X +vortex-runend@0.90.0 X +vortex-scan@0.90.0 X +vortex-sequence@0.90.0 X +vortex-session@0.90.0 X +vortex-sparse@0.90.0 X +vortex-utils@0.90.0 X +vortex-uuid@0.90.0 X +vortex-zigzag@0.90.0 X +vortex-zstd@0.90.0 X walkdir@2.5.0 X X want@0.3.1 X wasi@0.11.1+wasi-snapshot-preview1 X X X diff --git a/benchmarks/tpcds/DEPENDENCIES.rust.tsv b/benchmarks/tpcds/DEPENDENCIES.rust.tsv index e1821d263..f63518a34 100644 --- a/benchmarks/tpcds/DEPENDENCIES.rust.tsv +++ b/benchmarks/tpcds/DEPENDENCIES.rust.tsv @@ -19,20 +19,20 @@ ar_archive_writer@0.5.2 X arc-swap@1.9.2 X X arrayref@0.3.9 X arrayvec@0.7.8 X X -arrow@58.4.0 X -arrow-arith@58.4.0 X -arrow-array@58.4.0 X X -arrow-buffer@58.4.0 X -arrow-cast@58.4.0 X -arrow-csv@58.4.0 X -arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X -arrow-ord@58.4.0 X -arrow-row@58.4.0 X -arrow-schema@58.4.0 X -arrow-select@58.4.0 X -arrow-string@58.4.0 X +arrow@59.3.2 X +arrow-arith@59.3.2 X +arrow-array@59.3.2 X X +arrow-buffer@59.3.2 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X +arrow-ord@59.3.2 X +arrow-row@59.3.2 X +arrow-schema@59.3.2 X +arrow-select@59.3.2 X +arrow-string@59.3.2 X async-compression@0.4.42 X X async-stream@0.3.6 X async-stream-impl@0.3.6 X @@ -106,38 +106,38 @@ darling_core@0.23.0 X darling_macro@0.23.0 X dary_heap@0.3.9 X X dashmap@6.2.1 X -datafusion@54.0.0 X -datafusion-catalog@54.0.0 X -datafusion-catalog-listing@54.0.0 X -datafusion-common@54.0.0 X -datafusion-common-runtime@54.0.0 X -datafusion-datasource@54.0.0 X -datafusion-datasource-arrow@54.0.0 X -datafusion-datasource-csv@54.0.0 X -datafusion-datasource-json@54.0.0 X -datafusion-datasource-parquet@54.0.0 X -datafusion-doc@54.0.0 X -datafusion-execution@54.0.0 X -datafusion-expr@54.0.0 X -datafusion-expr-common@54.0.0 X -datafusion-functions@54.0.0 X -datafusion-functions-aggregate@54.0.0 X -datafusion-functions-aggregate-common@54.0.0 X -datafusion-functions-nested@54.0.0 X -datafusion-functions-table@54.0.0 X -datafusion-functions-window@54.0.0 X -datafusion-functions-window-common@54.0.0 X -datafusion-macros@54.0.0 X -datafusion-optimizer@54.0.0 X -datafusion-physical-expr@54.0.0 X -datafusion-physical-expr-adapter@54.0.0 X -datafusion-physical-expr-common@54.0.0 X -datafusion-physical-optimizer@54.0.0 X -datafusion-physical-plan@54.0.0 X -datafusion-pruning@54.0.0 X -datafusion-session@54.0.0 X -datafusion-spark@54.0.0 X -datafusion-sql@54.0.0 X +datafusion@55.2.0 X +datafusion-catalog@55.2.0 X +datafusion-catalog-listing@55.2.0 X +datafusion-common@55.2.0 X +datafusion-common-runtime@55.2.0 X +datafusion-datasource@55.2.0 X +datafusion-datasource-arrow@55.2.0 X +datafusion-datasource-csv@55.2.0 X +datafusion-datasource-json@55.2.0 X +datafusion-datasource-parquet@55.2.0 X +datafusion-doc@55.2.0 X +datafusion-execution@55.2.0 X +datafusion-expr@55.2.0 X +datafusion-expr-common@55.2.0 X +datafusion-functions@55.2.0 X +datafusion-functions-aggregate@55.2.0 X +datafusion-functions-aggregate-common@55.2.0 X +datafusion-functions-nested@55.2.0 X +datafusion-functions-table@55.2.0 X +datafusion-functions-window@55.2.0 X +datafusion-functions-window-common@55.2.0 X +datafusion-macros@55.2.0 X +datafusion-optimizer@55.2.0 X +datafusion-physical-expr@55.2.0 X +datafusion-physical-expr-adapter@55.2.0 X +datafusion-physical-expr-common@55.2.0 X +datafusion-physical-optimizer@55.2.0 X +datafusion-physical-plan@55.2.0 X +datafusion-pruning@55.2.0 X +datafusion-session@55.2.0 X +datafusion-spark@55.2.0 X +datafusion-sql@55.2.0 X datasketches@0.2.0 X deranged@0.5.8 X X diff@0.1.13 X X @@ -220,11 +220,11 @@ include-flate@0.3.4 X include-flate-codegen@0.3.4 X include-flate-compress@0.3.4 X indexmap@2.14.0 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X is_terminal_polyfill@1.70.2 X X itertools@0.14.0 X X +itertools@0.15.0 X X itoa@1.0.18 X X jieba-macros@0.10.3 X jieba-rs@0.10.3 X @@ -283,6 +283,7 @@ no_std_io2@0.9.4 X X nom@7.1.3 X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-complex@0.4.6 X X num-conv@0.2.2 X X num-integer@0.1.46 X X @@ -303,20 +304,19 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X paimon@0.6.0 X paimon-datafusion@0.6.0 X paimon-ftindex-core@0.1.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-tpcds-bench@0.6.0 X paimon-vindex-core@0.5.0 X parking_lot@0.12.5 X X parking_lot_core@0.9.12 X X -parquet@58.4.0 X +parquet@59.3.2 X paste@1.0.15 X X percent-encoding@2.3.2 X X petgraph@0.8.3 X X @@ -431,7 +431,7 @@ strum@0.27.2 X strum_macros@0.27.2 X subtle@2.6.1 X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X @@ -451,7 +451,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X diff --git a/bindings/c/DEPENDENCIES.rust.tsv b/bindings/c/DEPENDENCIES.rust.tsv index 01cdc0d8c..872dd10d7 100644 --- a/bindings/c/DEPENDENCIES.rust.tsv +++ b/bindings/c/DEPENDENCIES.rust.tsv @@ -14,19 +14,15 @@ approx@0.5.1 X arc-swap@1.9.2 X X arrow@59.3.2 X arrow-arith@59.3.2 X -arrow-array@58.4.0 X X arrow-array@59.3.2 X X -arrow-buffer@58.4.0 X arrow-buffer@59.3.2 X arrow-cast@59.3.2 X arrow-csv@59.3.2 X -arrow-data@58.4.0 X arrow-data@59.3.2 X arrow-ipc@59.3.2 X arrow-json@59.3.2 X arrow-ord@59.3.2 X arrow-row@59.3.2 X -arrow-schema@58.4.0 X arrow-schema@59.3.2 X arrow-select@59.3.2 X arrow-string@59.3.2 X @@ -270,7 +266,7 @@ ownedbytes@0.9.0 X paimon@0.6.0 X paimon-c@0.6.0 X paimon-ftindex-core@0.1.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-vindex-core@0.5.0 X parquet@59.3.2 X paste@1.0.15 X X @@ -394,7 +390,7 @@ strum@0.27.2 X strum_macros@0.27.2 X subtle@2.6.1 X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X diff --git a/bindings/c/src/tests.rs b/bindings/c/src/tests.rs index 108078d4e..91a0f8e69 100644 --- a/bindings/c/src/tests.rs +++ b/bindings/c/src/tests.rs @@ -3324,6 +3324,13 @@ fn build_pk_vector_table(path: &str, vectors: &[[f32; PK_DIM]]) -> Table { ); crate::runtime().block_on(async { + // Automatic compaction reloads the schema from storage after committing. + file_io + .new_output(&table.schema_manager().schema_path(table.schema().id())) + .unwrap() + .write(serde_json::to_vec(table.schema()).unwrap().into()) + .await + .unwrap(); // Write a real data file via the public write path to obtain a genuine // DataFileMeta (name, row count, stats, file size). let write_builder = table.new_write_builder(); diff --git a/bindings/go/DEPENDENCIES.rust.tsv b/bindings/go/DEPENDENCIES.rust.tsv index 01cdc0d8c..872dd10d7 100644 --- a/bindings/go/DEPENDENCIES.rust.tsv +++ b/bindings/go/DEPENDENCIES.rust.tsv @@ -14,19 +14,15 @@ approx@0.5.1 X arc-swap@1.9.2 X X arrow@59.3.2 X arrow-arith@59.3.2 X -arrow-array@58.4.0 X X arrow-array@59.3.2 X X -arrow-buffer@58.4.0 X arrow-buffer@59.3.2 X arrow-cast@59.3.2 X arrow-csv@59.3.2 X -arrow-data@58.4.0 X arrow-data@59.3.2 X arrow-ipc@59.3.2 X arrow-json@59.3.2 X arrow-ord@59.3.2 X arrow-row@59.3.2 X -arrow-schema@58.4.0 X arrow-schema@59.3.2 X arrow-select@59.3.2 X arrow-string@59.3.2 X @@ -270,7 +266,7 @@ ownedbytes@0.9.0 X paimon@0.6.0 X paimon-c@0.6.0 X paimon-ftindex-core@0.1.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-vindex-core@0.5.0 X parquet@59.3.2 X paste@1.0.15 X X @@ -394,7 +390,7 @@ strum@0.27.2 X strum_macros@0.27.2 X subtle@2.6.1 X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X diff --git a/bindings/python/DEPENDENCIES.rust.tsv b/bindings/python/DEPENDENCIES.rust.tsv index 642f4d3a9..a18d0382c 100644 --- a/bindings/python/DEPENDENCIES.rust.tsv +++ b/bindings/python/DEPENDENCIES.rust.tsv @@ -16,21 +16,21 @@ ar_archive_writer@0.5.2 X arc-swap@1.9.2 X X arrayref@0.3.9 X arrayvec@0.7.8 X X -arrow@58.4.0 X -arrow-arith@58.4.0 X -arrow-array@58.4.0 X X -arrow-buffer@58.4.0 X -arrow-cast@58.4.0 X -arrow-csv@58.4.0 X -arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X -arrow-ord@58.4.0 X -arrow-pyarrow@58.4.0 X -arrow-row@58.4.0 X -arrow-schema@58.4.0 X -arrow-select@58.4.0 X -arrow-string@58.4.0 X +arrow@59.3.2 X +arrow-arith@59.3.2 X +arrow-array@59.3.2 X X +arrow-buffer@59.3.2 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X +arrow-ord@59.3.2 X +arrow-pyarrow@59.3.2 X +arrow-row@59.3.2 X +arrow-schema@59.3.2 X +arrow-select@59.3.2 X +arrow-string@59.3.2 X async-compression@0.4.42 X X async-ffi@0.5.0 X async-stream@0.3.6 X @@ -113,41 +113,42 @@ darling_core@0.23.0 X darling_macro@0.23.0 X dary_heap@0.3.9 X X dashmap@6.2.1 X -datafusion@54.0.0 X -datafusion-catalog@54.0.0 X -datafusion-catalog-listing@54.0.0 X -datafusion-common@54.0.0 X -datafusion-common-runtime@54.0.0 X -datafusion-datasource@54.0.0 X -datafusion-datasource-arrow@54.0.0 X -datafusion-datasource-csv@54.0.0 X -datafusion-datasource-json@54.0.0 X -datafusion-datasource-parquet@54.0.0 X -datafusion-doc@54.0.0 X -datafusion-execution@54.0.0 X -datafusion-expr@54.0.0 X -datafusion-expr-common@54.0.0 X -datafusion-ffi@54.0.0 X -datafusion-functions@54.0.0 X -datafusion-functions-aggregate@54.0.0 X -datafusion-functions-aggregate-common@54.0.0 X -datafusion-functions-nested@54.0.0 X -datafusion-functions-table@54.0.0 X -datafusion-functions-window@54.0.0 X -datafusion-functions-window-common@54.0.0 X -datafusion-macros@54.0.0 X -datafusion-optimizer@54.0.0 X -datafusion-physical-expr@54.0.0 X -datafusion-physical-expr-adapter@54.0.0 X -datafusion-physical-expr-common@54.0.0 X -datafusion-physical-optimizer@54.0.0 X -datafusion-physical-plan@54.0.0 X -datafusion-proto@54.0.0 X -datafusion-proto-common@54.0.0 X -datafusion-pruning@54.0.0 X -datafusion-session@54.0.0 X -datafusion-spark@54.0.0 X -datafusion-sql@54.0.0 X +datafusion@55.2.0 X +datafusion-catalog@55.2.0 X +datafusion-catalog-listing@55.2.0 X +datafusion-common@55.2.0 X +datafusion-common-runtime@55.2.0 X +datafusion-datasource@55.2.0 X +datafusion-datasource-arrow@55.2.0 X +datafusion-datasource-csv@55.2.0 X +datafusion-datasource-json@55.2.0 X +datafusion-datasource-parquet@55.2.0 X +datafusion-doc@55.2.0 X +datafusion-execution@55.2.0 X +datafusion-expr@55.2.0 X +datafusion-expr-common@55.2.0 X +datafusion-ffi@55.2.0 X +datafusion-functions@55.2.0 X +datafusion-functions-aggregate@55.2.0 X +datafusion-functions-aggregate-common@55.2.0 X +datafusion-functions-nested@55.2.0 X +datafusion-functions-table@55.2.0 X +datafusion-functions-window@55.2.0 X +datafusion-functions-window-common@55.2.0 X +datafusion-macros@55.2.0 X +datafusion-optimizer@55.2.0 X +datafusion-physical-expr@55.2.0 X +datafusion-physical-expr-adapter@55.2.0 X +datafusion-physical-expr-common@55.2.0 X +datafusion-physical-optimizer@55.2.0 X +datafusion-physical-plan@55.2.0 X +datafusion-proto@55.2.0 X +datafusion-proto-common@55.2.0 X +datafusion-proto-models@55.2.0 X +datafusion-pruning@55.2.0 X +datafusion-session@55.2.0 X +datafusion-spark@55.2.0 X +datafusion-sql@55.2.0 X datasketches@0.2.0 X der@0.7.10 X X deranged@0.5.8 X X @@ -239,10 +240,10 @@ include-flate-compress@0.3.4 X indexmap@2.14.0 X X inout@0.1.4 X X inout@0.2.2 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X itertools@0.14.0 X X +itertools@0.15.0 X X itoa@1.0.18 X X jieba-macros@0.10.3 X jieba-rs@0.10.3 X @@ -302,6 +303,7 @@ no_std_io2@0.9.4 X X nom@7.1.3 X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-bigint-dig@0.8.6 X X num-complex@0.4.6 X X num-conv@0.2.2 X X @@ -331,19 +333,18 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X paimon@0.6.0 X paimon-datafusion@0.6.0 X paimon-ftindex-core@0.1.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-vindex-core@0.5.0 X parking_lot@0.12.5 X X parking_lot_core@0.9.12 X X -parquet@58.4.0 X +parquet@59.3.2 X paste@1.0.15 X X pbkdf2@0.12.2 X X pem@4.0.0 X @@ -380,11 +381,11 @@ prost-derive@0.13.5 X prost-derive@0.14.4 X prost-types@0.14.4 X psm@0.1.31 X X -pyo3@0.28.3 X X -pyo3-build-config@0.28.3 X X -pyo3-ffi@0.28.3 X X -pyo3-macros@0.28.3 X X -pyo3-macros-backend@0.28.3 X X +pyo3@0.29.3 X X +pyo3-build-config@0.29.3 X X +pyo3-ffi@0.29.3 X X +pyo3-macros@0.29.3 X X +pyo3-macros-backend@0.29.3 X X pypaimon_rust@0.6.0 X quad-rand@0.2.3 X quick-xml@0.41.0 X @@ -493,7 +494,7 @@ strum@0.27.2 X strum_macros@0.27.2 X subtle@2.6.1 X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X @@ -514,7 +515,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X diff --git a/crates/integration_tests/DEPENDENCIES.rust.tsv b/crates/integration_tests/DEPENDENCIES.rust.tsv index ec13d5433..c3b8cd87f 100644 --- a/crates/integration_tests/DEPENDENCIES.rust.tsv +++ b/crates/integration_tests/DEPENDENCIES.rust.tsv @@ -13,19 +13,15 @@ approx@0.5.1 X arc-swap@1.9.2 X X arrow@59.3.2 X arrow-arith@59.3.2 X -arrow-array@58.4.0 X X arrow-array@59.3.2 X X -arrow-buffer@58.4.0 X arrow-buffer@59.3.2 X arrow-cast@59.3.2 X arrow-csv@59.3.2 X -arrow-data@58.4.0 X arrow-data@59.3.2 X arrow-ipc@59.3.2 X arrow-json@59.3.2 X arrow-ord@59.3.2 X arrow-row@59.3.2 X -arrow-schema@58.4.0 X arrow-schema@59.3.2 X arrow-select@59.3.2 X arrow-string@59.3.2 X @@ -254,7 +250,7 @@ ownedbytes@0.9.0 X paimon@0.6.0 X paimon-ftindex-core@0.1.0 X paimon-integration-tests@0.6.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-vindex-core@0.5.0 X parquet@59.3.2 X paste@1.0.15 X X @@ -358,7 +354,7 @@ strum@0.27.2 X strum_macros@0.27.2 X subtle@2.6.1 X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X diff --git a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv index 9e016eab7..773ee4247 100644 --- a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv +++ b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv @@ -6,6 +6,7 @@ aho-corasick@1.1.4 X X alloc-no-stdlib@2.0.4 X alloc-stdlib@0.2.4 X allocator-api2@0.2.21 X X +alp@0.0.4 X android_system_properties@0.1.5 X X anyhow@1.0.104 X X apache-avro@0.21.0 X @@ -15,20 +16,20 @@ arc-swap@1.9.2 X X arcref@0.2.0 X arrayref@0.3.9 X arrayvec@0.7.8 X X -arrow@58.4.0 X -arrow-arith@58.4.0 X -arrow-array@58.4.0 X X -arrow-buffer@58.4.0 X -arrow-cast@58.4.0 X -arrow-csv@58.4.0 X -arrow-data@58.4.0 X -arrow-ipc@58.4.0 X -arrow-json@58.4.0 X -arrow-ord@58.4.0 X -arrow-row@58.4.0 X -arrow-schema@58.4.0 X -arrow-select@58.4.0 X -arrow-string@58.4.0 X +arrow@59.3.2 X +arrow-arith@59.3.2 X +arrow-array@59.3.2 X X +arrow-buffer@59.3.2 X +arrow-cast@59.3.2 X +arrow-csv@59.3.2 X +arrow-data@59.3.2 X +arrow-ipc@59.3.2 X +arrow-json@59.3.2 X +arrow-ord@59.3.2 X +arrow-row@59.3.2 X +arrow-schema@59.3.2 X +arrow-select@59.3.2 X +arrow-string@59.3.2 X async-channel@2.5.0 X X async-compression@0.4.42 X X async-executor@1.14.0 X X @@ -54,7 +55,7 @@ base64@0.23.1 X X better_io@0.2.0 X bigdecimal@0.4.10 X X bindgen@0.72.1 X -bit-vec@0.9.1 X X +bit-vec@0.10.1 X X bitflags@2.13.1 X X bitpacking@0.9.3 X bitvec@1.1.1 X @@ -97,7 +98,6 @@ constant_time_eq@0.4.2 X X X core-foundation@0.10.1 X X core-foundation@0.9.4 X X core-foundation-sys@0.8.7 X X -core_detect@1.0.0 X X cpufeatures@0.2.17 X X cpufeatures@0.3.0 X X crc32fast@1.5.0 X X @@ -117,38 +117,38 @@ darling_core@0.23.0 X darling_macro@0.23.0 X dary_heap@0.3.9 X X dashmap@6.2.1 X -datafusion@54.0.0 X -datafusion-catalog@54.0.0 X -datafusion-catalog-listing@54.0.0 X -datafusion-common@54.0.0 X -datafusion-common-runtime@54.0.0 X -datafusion-datasource@54.0.0 X -datafusion-datasource-arrow@54.0.0 X -datafusion-datasource-csv@54.0.0 X -datafusion-datasource-json@54.0.0 X -datafusion-datasource-parquet@54.0.0 X -datafusion-doc@54.0.0 X -datafusion-execution@54.0.0 X -datafusion-expr@54.0.0 X -datafusion-expr-common@54.0.0 X -datafusion-functions@54.0.0 X -datafusion-functions-aggregate@54.0.0 X -datafusion-functions-aggregate-common@54.0.0 X -datafusion-functions-nested@54.0.0 X -datafusion-functions-table@54.0.0 X -datafusion-functions-window@54.0.0 X -datafusion-functions-window-common@54.0.0 X -datafusion-macros@54.0.0 X -datafusion-optimizer@54.0.0 X -datafusion-physical-expr@54.0.0 X -datafusion-physical-expr-adapter@54.0.0 X -datafusion-physical-expr-common@54.0.0 X -datafusion-physical-optimizer@54.0.0 X -datafusion-physical-plan@54.0.0 X -datafusion-pruning@54.0.0 X -datafusion-session@54.0.0 X -datafusion-spark@54.0.0 X -datafusion-sql@54.0.0 X +datafusion@55.2.0 X +datafusion-catalog@55.2.0 X +datafusion-catalog-listing@55.2.0 X +datafusion-common@55.2.0 X +datafusion-common-runtime@55.2.0 X +datafusion-datasource@55.2.0 X +datafusion-datasource-arrow@55.2.0 X +datafusion-datasource-csv@55.2.0 X +datafusion-datasource-json@55.2.0 X +datafusion-datasource-parquet@55.2.0 X +datafusion-doc@55.2.0 X +datafusion-execution@55.2.0 X +datafusion-expr@55.2.0 X +datafusion-expr-common@55.2.0 X +datafusion-functions@55.2.0 X +datafusion-functions-aggregate@55.2.0 X +datafusion-functions-aggregate-common@55.2.0 X +datafusion-functions-nested@55.2.0 X +datafusion-functions-table@55.2.0 X +datafusion-functions-window@55.2.0 X +datafusion-functions-window-common@55.2.0 X +datafusion-macros@55.2.0 X +datafusion-optimizer@55.2.0 X +datafusion-physical-expr@55.2.0 X +datafusion-physical-expr-adapter@55.2.0 X +datafusion-physical-expr-common@55.2.0 X +datafusion-physical-optimizer@55.2.0 X +datafusion-physical-plan@55.2.0 X +datafusion-pruning@55.2.0 X +datafusion-session@55.2.0 X +datafusion-spark@55.2.0 X +datafusion-sql@55.2.0 X datasketches@0.2.0 X deranged@0.5.8 X X diff@0.1.13 X X @@ -173,8 +173,11 @@ ext-trait-proc_macros@1.0.1 X X X extension-traits@1.0.1 X X X fallible-streaming-iterator@0.1.9 X X fastdivide@0.4.2 X X -fastlanes@0.5.2 X +fastlanes@0.6.1 X +fastlanes@0.7.2 X fastrand@2.5.0 X X +fearless_simd@1.2.0 X X +fearless_simd_macros@0.1.0 X X find-msvc-tools@0.1.9 X X fixedbitset@0.5.7 X X flatbuffers@25.12.19 X @@ -187,7 +190,7 @@ foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs4@0.13.1 X X fs_extra@1.3.0 X -fsst-rs@0.5.11 X +fsst-rs@0.6.0 X funty@2.0.0 X futures@0.3.33 X X futures-channel@0.3.33 X X @@ -245,11 +248,11 @@ include-flate@0.3.4 X include-flate-codegen@0.3.4 X include-flate-compress@0.3.4 X indexmap@2.14.0 X X -integer-encoding@3.0.4 X inventory@0.3.24 X X ipnet@2.12.0 X X itertools@0.13.0 X X itertools@0.14.0 X X +itertools@0.15.0 X X itoa@1.0.18 X X jieba-macros@0.10.3 X jieba-rs@0.10.3 X @@ -319,6 +322,7 @@ nougat@0.2.4 X X X nougat-proc_macros@0.2.4 X X X num@0.4.3 X X num-bigint@0.4.8 X X +num-bigint@0.5.1 X X num-complex@0.4.6 X X num-conv@0.2.2 X X num-integer@0.1.46 X X @@ -331,7 +335,7 @@ object@0.37.3 X X object_store@0.13.2 X X once_cell@1.21.4 X X oneshot@0.1.13 X X -oneshot@0.2.1 X X +onpair@0.2.1 X opendal-core@0.59.5 X opendal-http-transport-reqwest@0.59.5 X opendal-layer-retry@0.59.5 X @@ -341,21 +345,24 @@ openssl@0.10.81 X openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X openssl-sys@0.9.117 X -orc-rust@0.8.0 X -ordered-float@2.10.1 X +orc-rust@0.9.0 X ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.9.0 X paimon@0.6.0 X paimon-datafusion@0.6.0 X paimon-ftindex-core@0.1.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-vindex-core@0.5.0 X parking@2.2.1 X X parking_lot@0.12.5 X X parking_lot_core@0.9.12 X X -parquet@58.4.0 X +parquet@59.3.2 X +parquet-variant@59.3.2 X +parquet-variant-compute@59.3.2 X +parquet-variant-json@59.3.2 X paste@1.0.15 X X +pastey@0.2.3 X X pco@1.0.2 X percent-encoding@2.3.2 X X petgraph@0.8.3 X X @@ -483,7 +490,7 @@ strum_macros@0.27.2 X subtle@2.6.1 X syn@1.0.109 X X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X @@ -506,7 +513,6 @@ thiserror@1.0.69 X X thiserror@2.0.19 X X thiserror-impl@1.0.69 X X thiserror-impl@2.0.19 X X -thrift@0.17.0 X time@0.3.54 X X time-core@0.1.9 X X tiny-keccak@2.0.2 X @@ -542,36 +548,41 @@ utf8_iter@1.0.4 X X uuid@1.24.0 X X vcpkg@0.2.15 X X version_check@0.9.5 X X -vortex@0.75.0 X -vortex-alp@0.75.0 X -vortex-array@0.75.0 X -vortex-array-macros@0.75.0 X -vortex-btrblocks@0.75.0 X -vortex-buffer@0.75.0 X -vortex-bytebool@0.75.0 X -vortex-compressor@0.75.0 X -vortex-datetime-parts@0.75.0 X -vortex-decimal-byte-parts@0.75.0 X -vortex-error@0.75.0 X -vortex-fastlanes@0.75.0 X -vortex-file@0.75.0 X -vortex-flatbuffers@0.75.0 X -vortex-fsst@0.75.0 X -vortex-io@0.75.0 X -vortex-ipc@0.75.0 X -vortex-layout@0.75.0 X -vortex-mask@0.75.0 X -vortex-metrics@0.75.0 X -vortex-pco@0.75.0 X -vortex-proto@0.75.0 X -vortex-runend@0.75.0 X -vortex-scan@0.75.0 X -vortex-sequence@0.75.0 X -vortex-session@0.75.0 X -vortex-sparse@0.75.0 X -vortex-utils@0.75.0 X -vortex-zigzag@0.75.0 X -vortex-zstd@0.75.0 X +vortex@0.90.0 X +vortex-alp@0.90.0 X +vortex-array@0.90.0 X +vortex-array-macros@0.90.0 X +vortex-arrow@0.90.0 X +vortex-btrblocks@0.90.0 X +vortex-buffer@0.90.0 X +vortex-bytebool@0.90.0 X +vortex-compressor@0.90.0 X +vortex-compute@0.90.0 X +vortex-datetime-parts@0.90.0 X +vortex-decimal-byte-parts@0.90.0 X +vortex-edition@0.90.0 X +vortex-error@0.90.0 X +vortex-fastlanes@0.90.0 X +vortex-file@0.90.0 X +vortex-fsst@0.90.0 X +vortex-io@0.90.0 X +vortex-ipc@0.90.0 X +vortex-json@0.90.0 X +vortex-layout@0.90.0 X +vortex-mask@0.90.0 X +vortex-metrics@0.90.0 X +vortex-onpair@0.90.0 X +vortex-parquet-variant@0.90.0 X +vortex-pco@0.90.0 X +vortex-runend@0.90.0 X +vortex-scan@0.90.0 X +vortex-sequence@0.90.0 X +vortex-session@0.90.0 X +vortex-sparse@0.90.0 X +vortex-utils@0.90.0 X +vortex-uuid@0.90.0 X +vortex-zigzag@0.90.0 X +vortex-zstd@0.90.0 X walkdir@2.5.0 X X want@0.3.1 X wasi@0.11.1+wasi-snapshot-preview1 X X X diff --git a/crates/integrations/datafusion/src/selection_aggregate.rs b/crates/integrations/datafusion/src/selection_aggregate.rs index a01deeaa4..37a0f05ed 100644 --- a/crates/integrations/datafusion/src/selection_aggregate.rs +++ b/crates/integrations/datafusion/src/selection_aggregate.rs @@ -222,11 +222,18 @@ impl GroupsAccumulator for SelectionGroupsAccumulator { &mut self, values: &[ArrayRef], group_indices: &[usize], - opt_filter: Option<&BooleanArray>, total_num_groups: usize, ) -> Result<()> { self.native - .merge_batch(values, group_indices, opt_filter, total_num_groups) + .merge_batch(values, group_indices, total_num_groups) + } + fn convert_to_state( + &self, + values: &[ArrayRef], + opt_filter: Option<&BooleanArray>, + ) -> Result> { + let mask = eligibility(values, self.ignore_null_values, opt_filter)?; + self.native.convert_to_state(values, mask.as_ref()) } fn size(&self) -> usize { self.native.size() + std::mem::size_of::() diff --git a/crates/integrations/datafusion/src/spark_functions.rs b/crates/integrations/datafusion/src/spark_functions.rs index 317d695b2..f727a527b 100644 --- a/crates/integrations/datafusion/src/spark_functions.rs +++ b/crates/integrations/datafusion/src/spark_functions.rs @@ -36,6 +36,9 @@ use std::sync::Arc; // Adapt input and return types while retaining upstream behavior. pub(crate) fn apply_compatibility_fixes(ctx: &SessionContext) { + // Use the compatibility rewrite for Spark's optional ignoreNulls argument + // and default RESPECT NULLS semantics. DataFusion 55's native UDAF ignores nulls. + ctx.deregister_udaf("any_value"); ctx.register_udf(date_part().as_ref().clone()); for bits in [false, true] { ctx.register_udf(ScalarUDF::from(ByteLength { @@ -297,9 +300,6 @@ impl FilteredGroups { } } impl GroupsAccumulator for FilteredGroups { - fn supports_convert_to_state(&self) -> bool { - self.0.supports_convert_to_state() - } fn convert_to_state( &self, values: &[ArrayRef], @@ -321,19 +321,8 @@ impl GroupsAccumulator for FilteredGroups { self.0.update_batch(values, groups, None, total) } } - fn merge_batch( - &mut self, - values: &[ArrayRef], - groups: &[usize], - predicate: Option<&BooleanArray>, - total: usize, - ) -> Result<()> { - if let Some(predicate) = predicate { - let (values, groups) = Self::filter(values, groups, predicate)?; - self.0.merge_batch(&values, &groups, None, total) - } else { - self.0.merge_batch(values, groups, None, total) - } + fn merge_batch(&mut self, values: &[ArrayRef], groups: &[usize], total: usize) -> Result<()> { + self.0.merge_batch(values, groups, total) } fn evaluate(&mut self, emit: EmitTo) -> Result { self.0.evaluate(emit) diff --git a/crates/integrations/datafusion/tests/remaining_compat.rs b/crates/integrations/datafusion/tests/remaining_compat.rs index 850346010..e940ab90a 100644 --- a/crates/integrations/datafusion/tests/remaining_compat.rs +++ b/crates/integrations/datafusion/tests/remaining_compat.rs @@ -98,44 +98,59 @@ async fn selection_aggregates_merge_native_state_across_batches_and_partitions() }; use datafusion::datasource::MemTable; use std::sync::Arc; - let ctx = SQLContext::new(); - let schema = Arc::new(Schema::new(vec![ - Field::new("g", DataType::Int64, false), - Field::new("v", DataType::Int64, true), - Field::new("k", DataType::Int64, true), - ])); - let batch = |g: Vec, v: Vec>, k: Vec>| { - RecordBatch::try_new( - Arc::clone(&schema), + // Exercise both ordinary partial aggregation and direct state conversion. + for bypass in [false, true] { + let ctx = SQLContext::new(); + ctx.sql("SET datafusion.execution.target_partitions = 2") + .await + .unwrap(); + if bypass { + ctx.sql( + "SET datafusion.execution.skip_partial_aggregation_probe_ratio_threshold = 0.0", + ) + .await + .unwrap(); + ctx.sql("SET datafusion.execution.skip_partial_aggregation_probe_rows_threshold = 0") + .await + .unwrap(); + } + let schema = Arc::new(Schema::new(vec![ + Field::new("g", DataType::Int64, false), + Field::new("v", DataType::Int64, true), + Field::new("k", DataType::Int64, true), + ])); + let batch = |g: Vec, v: Vec>, k: Vec>| { + RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(Int64Array::from(g)), + Arc::new(Int64Array::from(v)), + Arc::new(Int64Array::from(k)), + ], + ) + .unwrap() + }; + let partitions = vec![ vec![ - Arc::new(Int64Array::from(g)), - Arc::new(Int64Array::from(v)), - Arc::new(Int64Array::from(k)), + batch(vec![1, 2], vec![Some(10), Some(200)], vec![Some(1), None]), + batch(vec![1, 3], vec![None, Some(300)], vec![Some(4), Some(10)]), ], - ) - .unwrap() - }; - let partitions = vec![ - vec![ - batch(vec![1, 2], vec![Some(10), Some(200)], vec![Some(1), None]), - batch(vec![1, 3], vec![None, Some(300)], vec![Some(4), Some(10)]), - ], - vec![ - batch( - vec![1, 2, 3], - vec![Some(30), Some(201), None], - vec![Some(3), None, Some(11)], - ), - batch(vec![1, 3], vec![Some(999), Some(301)], vec![None, Some(12)]), - ], - ]; - ctx.ctx() - .register_table( - "selection_inputs", - Arc::new(MemTable::try_new(schema, partitions).unwrap()), - ) - .unwrap(); - assert_eq!( + vec![ + batch( + vec![1, 2, 3], + vec![Some(30), Some(201), None], + vec![Some(3), None, Some(11)], + ), + batch(vec![1, 3], vec![Some(999), Some(301)], vec![None, Some(12)]), + ], + ]; + ctx.ctx() + .register_table( + "selection_inputs", + Arc::new(MemTable::try_new(schema, partitions).unwrap()), + ) + .unwrap(); + assert_eq!( rows( &ctx, "SELECT g,arg_max(CAST(NULL AS BIGINT),1) FROM selection_inputs GROUP BY g ORDER BY g" @@ -143,15 +158,18 @@ async fn selection_aggregates_merge_native_state_across_batches_and_partitions() .await, vec![vec!["1", "NULL"], vec!["2", "NULL"], vec!["3", "NULL"]] ); - assert_eq!(rows(&ctx,"SELECT g,max_by(v,k),min_by(v,k),arg_max(v,k),max_by(v,k) FILTER (WHERE v<300 OR v IS NULL) FROM selection_inputs GROUP BY g ORDER BY g").await,vec![vec!["1","NULL","10","30","NULL"],vec!["2","NULL","NULL","NULL","NULL"],vec!["3","301","300","301","NULL"]]); - assert_eq!( - rows( - &ctx, - "SELECT max_by(v,k),min_by(v,k),arg_max(v,k) FROM selection_inputs" - ) - .await, - vec![vec!["301", "10", "301"]] - ); + assert_eq!(rows(&ctx,"SELECT g,max_by(v,k),min_by(v,k),arg_max(v,k),max_by(v,k) FILTER (WHERE v<300 OR v IS NULL) FROM selection_inputs GROUP BY g ORDER BY g").await,vec![vec!["1","NULL","10","30","NULL"],vec!["2","NULL","NULL","NULL","NULL"],vec!["3","301","300","301","NULL"]]); + assert_eq!( + rows( + &ctx, + "SELECT max_by(v,k),min_by(v,k),arg_max(v,k) FROM selection_inputs" + ) + .await, + vec![vec!["301", "10", "301"]] + ); + assert_eq!(rows(&ctx, "SELECT g,arg_max(v,k) FILTER (WHERE k>2) FROM selection_inputs GROUP BY g ORDER BY g").await, + vec![vec!["1", "30"],vec!["2", "NULL"],vec!["3", "301"]]); + } } #[tokio::test] @@ -229,17 +247,25 @@ async fn aggregate_rewrites_keep_filters_constant_keys_and_local_registrations() vec![vec!["a", "a", "a"]] ); assert_eq!(rows(&ctx,"SELECT array_length(collect_list(p OR q)) FROM (VALUES (NULL,FALSE),(FALSE,TRUE)) t(p,q)").await,vec![vec!["1"]]); - ctx.ctx().register_udf(create_udf( - "first", - vec![DataType::Int64], - DataType::Int64, - Volatility::Immutable, - Arc::new(|args| Ok(args[0].clone())), - )); - assert_eq!( - rows(&ctx, "SELECT first(v) FROM (VALUES (1),(2)) t(v)").await, - vec![vec!["1"], vec!["2"]] - ); + assert_eq!(rows(&ctx, "SELECT any_value(v),any_value(v,false),any_value(v,true) FROM (VALUES (CAST(NULL AS BIGINT)),(7)) t(v)").await, + vec![vec!["NULL", "NULL", "7"]]); + for name in ["first", "any_value"] { + ctx.ctx().register_udf(create_udf( + name, + vec![DataType::Int64], + DataType::Int64, + Volatility::Immutable, + Arc::new(|args| Ok(args[0].clone())), + )); + assert_eq!( + rows( + &ctx, + &format!("SELECT {name}(v) FROM (VALUES (1),(2)) t(v)") + ) + .await, + vec![vec!["1"], vec!["2"]] + ); + } } #[tokio::test] diff --git a/crates/paimon-rest-server/DEPENDENCIES.rust.tsv b/crates/paimon-rest-server/DEPENDENCIES.rust.tsv index ecdb37da4..bca31d1d7 100644 --- a/crates/paimon-rest-server/DEPENDENCIES.rust.tsv +++ b/crates/paimon-rest-server/DEPENDENCIES.rust.tsv @@ -13,19 +13,15 @@ approx@0.5.1 X arc-swap@1.9.2 X X arrow@59.3.2 X arrow-arith@59.3.2 X -arrow-array@58.4.0 X X arrow-array@59.3.2 X X -arrow-buffer@58.4.0 X arrow-buffer@59.3.2 X arrow-cast@59.3.2 X arrow-csv@59.3.2 X -arrow-data@58.4.0 X arrow-data@59.3.2 X arrow-ipc@59.3.2 X arrow-json@59.3.2 X arrow-ord@59.3.2 X arrow-row@59.3.2 X -arrow-schema@58.4.0 X arrow-schema@59.3.2 X arrow-select@59.3.2 X arrow-string@59.3.2 X @@ -257,7 +253,7 @@ ordered-multimap@0.7.3 X ownedbytes@0.9.0 X paimon@0.6.0 X paimon-ftindex-core@0.1.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-rest-server@0.6.0 X paimon-vindex-core@0.5.0 X parquet@59.3.2 X @@ -364,7 +360,7 @@ strum@0.27.2 X strum_macros@0.27.2 X subtle@2.6.1 X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X diff --git a/crates/paimon/Cargo.toml b/crates/paimon/Cargo.toml index 3bc041655..a92f02ab6 100644 --- a/crates/paimon/Cargo.toml +++ b/crates/paimon/Cargo.toml @@ -127,9 +127,6 @@ bzip2 = "0.6" liblzma = { version = "0.4", default-features = false, features = ["static"] } lzokay-native = { version = "0.1", default-features = false, features = ["decompress"] } arrow-array = { workspace = true } -# Mosaic and Vortex still expose Arrow 58; bridge their arrays through the C Data Interface. -arrow-array-58 = { package = "arrow-array", version = "58.4", features = ["ffi"] } -arrow-schema-58 = { package = "arrow-schema", version = "58.4" } arrow-arith = { workspace = true } arrow-buffer = { workspace = true } arrow-cast = { workspace = true } @@ -159,11 +156,11 @@ twox-hash = { version = "2.1.3", default-features = false, features = ["xxhash64 md-5 = "0.10" regex = "1" uuid = { version = "1", features = ["v4"] } -paimon-mosaic-core = "0.2.0" +paimon-mosaic-core = "0.3.0" paimon-ftindex-core = "0.1.0" tempfile = "3" paimon-vindex-core = "0.5.0" -vortex = { version = "0.75.0", features = ["tokio"], optional = true } +vortex = { version = "0.90.0", features = ["tokio"], optional = true } libloading = "0.9" log = "0.4" # Keep CI on the dependency set that passed before unicode-segmentation 1.13.3. diff --git a/crates/paimon/DEPENDENCIES.rust.tsv b/crates/paimon/DEPENDENCIES.rust.tsv index 0630619e3..a8a771744 100644 --- a/crates/paimon/DEPENDENCIES.rust.tsv +++ b/crates/paimon/DEPENDENCIES.rust.tsv @@ -8,6 +8,7 @@ aho-corasick@1.1.4 X X alloc-no-stdlib@2.0.4 X alloc-stdlib@0.2.4 X allocator-api2@0.2.21 X X +alp@0.0.4 X android_system_properties@0.1.5 X X anyhow@1.0.104 X X apache-avro@0.21.0 X @@ -15,27 +16,18 @@ approx@0.5.1 X arc-swap@1.9.2 X X arcref@0.2.0 X arrow@59.3.2 X -arrow-arith@58.4.0 X arrow-arith@59.3.2 X -arrow-array@58.4.0 X X arrow-array@59.3.2 X X -arrow-buffer@58.4.0 X arrow-buffer@59.3.2 X -arrow-cast@58.4.0 X arrow-cast@59.3.2 X arrow-csv@59.3.2 X -arrow-data@58.4.0 X arrow-data@59.3.2 X arrow-ipc@59.3.2 X arrow-json@59.3.2 X -arrow-ord@58.4.0 X arrow-ord@59.3.2 X arrow-row@59.3.2 X -arrow-schema@58.4.0 X arrow-schema@59.3.2 X -arrow-select@58.4.0 X arrow-select@59.3.2 X -arrow-string@58.4.0 X arrow-string@59.3.2 X async-channel@2.5.0 X X async-compression@0.4.42 X X @@ -63,7 +55,7 @@ base64ct@1.8.3 X X better_io@0.2.0 X bigdecimal@0.4.10 X X bindgen@0.72.1 X -bit-vec@0.9.1 X X +bit-vec@0.10.1 X X bitflags@2.13.1 X X bitpacking@0.9.3 X bitvec@1.1.1 X @@ -110,7 +102,6 @@ const_for@0.1.6 X core-foundation@0.10.1 X X core-foundation@0.9.4 X X core-foundation-sys@0.8.7 X X -core_detect@1.0.0 X X cpubits@0.1.1 X X cpufeatures@0.2.17 X X cpufeatures@0.3.0 X X @@ -162,8 +153,11 @@ ext-trait-proc_macros@1.0.1 X X X extension-traits@1.0.1 X X X fallible-streaming-iterator@0.1.9 X X fastdivide@0.4.2 X X -fastlanes@0.5.2 X +fastlanes@0.6.1 X +fastlanes@0.7.2 X fastrand@2.5.0 X X +fearless_simd@1.2.0 X X +fearless_simd_macros@0.1.0 X X find-msvc-tools@0.1.9 X X flatbuffers@25.12.19 X flate2@1.1.9 X X @@ -174,7 +168,7 @@ foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs4@0.13.1 X X fs_extra@1.3.0 X -fsst-rs@0.5.11 X +fsst-rs@0.6.0 X funty@2.0.0 X futures@0.3.33 X X futures-channel@0.3.33 X X @@ -240,6 +234,7 @@ inventory@0.3.24 X X ipnet@2.12.0 X X itertools@0.13.0 X X itertools@0.14.0 X X +itertools@0.15.0 X X itoa@1.0.18 X X jieba-macros@0.10.3 X jieba-rs@0.10.3 X @@ -324,7 +319,7 @@ objc2-core-foundation@0.3.2 X X X objc2-system-configuration@0.3.2 X X X once_cell@1.21.4 X X oneshot@0.1.13 X X -oneshot@0.2.1 X X +onpair@0.2.1 X opendal-core@0.59.5 X opendal-http-transport-reqwest@0.59.5 X opendal-layer-retry@0.59.5 X @@ -347,13 +342,17 @@ ordered-multimap@0.7.3 X ownedbytes@0.9.0 X paimon@0.6.0 X paimon-ftindex-core@0.1.0 X -paimon-mosaic-core@0.2.0 X +paimon-mosaic-core@0.3.0 X paimon-vindex-core@0.5.0 X parking@2.2.1 X X parking_lot@0.12.5 X X parking_lot_core@0.9.12 X X parquet@59.3.2 X +parquet-variant@59.3.2 X +parquet-variant-compute@59.3.2 X +parquet-variant-json@59.3.2 X paste@1.0.15 X X +pastey@0.2.3 X X pbkdf2@0.12.2 X X pco@1.0.2 X pem@4.0.0 X @@ -493,7 +492,7 @@ strum_macros@0.27.2 X subtle@2.6.1 X syn@1.0.109 X X syn@2.0.119 X X -syn@3.0.2 X X +syn@3.0.9 X X sync_wrapper@1.0.2 X synstructure@0.13.2 X system-configuration@0.7.0 X X @@ -550,36 +549,41 @@ utf8_iter@1.0.4 X X uuid@1.24.0 X X vcpkg@0.2.15 X X version_check@0.9.5 X X -vortex@0.75.0 X -vortex-alp@0.75.0 X -vortex-array@0.75.0 X -vortex-array-macros@0.75.0 X -vortex-btrblocks@0.75.0 X -vortex-buffer@0.75.0 X -vortex-bytebool@0.75.0 X -vortex-compressor@0.75.0 X -vortex-datetime-parts@0.75.0 X -vortex-decimal-byte-parts@0.75.0 X -vortex-error@0.75.0 X -vortex-fastlanes@0.75.0 X -vortex-file@0.75.0 X -vortex-flatbuffers@0.75.0 X -vortex-fsst@0.75.0 X -vortex-io@0.75.0 X -vortex-ipc@0.75.0 X -vortex-layout@0.75.0 X -vortex-mask@0.75.0 X -vortex-metrics@0.75.0 X -vortex-pco@0.75.0 X -vortex-proto@0.75.0 X -vortex-runend@0.75.0 X -vortex-scan@0.75.0 X -vortex-sequence@0.75.0 X -vortex-session@0.75.0 X -vortex-sparse@0.75.0 X -vortex-utils@0.75.0 X -vortex-zigzag@0.75.0 X -vortex-zstd@0.75.0 X +vortex@0.90.0 X +vortex-alp@0.90.0 X +vortex-array@0.90.0 X +vortex-array-macros@0.90.0 X +vortex-arrow@0.90.0 X +vortex-btrblocks@0.90.0 X +vortex-buffer@0.90.0 X +vortex-bytebool@0.90.0 X +vortex-compressor@0.90.0 X +vortex-compute@0.90.0 X +vortex-datetime-parts@0.90.0 X +vortex-decimal-byte-parts@0.90.0 X +vortex-edition@0.90.0 X +vortex-error@0.90.0 X +vortex-fastlanes@0.90.0 X +vortex-file@0.90.0 X +vortex-fsst@0.90.0 X +vortex-io@0.90.0 X +vortex-ipc@0.90.0 X +vortex-json@0.90.0 X +vortex-layout@0.90.0 X +vortex-mask@0.90.0 X +vortex-metrics@0.90.0 X +vortex-onpair@0.90.0 X +vortex-parquet-variant@0.90.0 X +vortex-pco@0.90.0 X +vortex-runend@0.90.0 X +vortex-scan@0.90.0 X +vortex-sequence@0.90.0 X +vortex-session@0.90.0 X +vortex-sparse@0.90.0 X +vortex-utils@0.90.0 X +vortex-uuid@0.90.0 X +vortex-zigzag@0.90.0 X +vortex-zstd@0.90.0 X walkdir@2.5.0 X X want@0.3.1 X wasi@0.11.1+wasi-snapshot-preview1 X X X diff --git a/crates/paimon/src/arrow/format/mosaic.rs b/crates/paimon/src/arrow/format/mosaic.rs index 136c27d85..c646447a3 100644 --- a/crates/paimon/src/arrow/format/mosaic.rs +++ b/crates/paimon/src/arrow/format/mosaic.rs @@ -511,7 +511,6 @@ fn read_planned_row_group( .row_group_reader_by_names(plan.index, &names) .map_err(mosaic_read_error)?; let batch = row_group_reader.read_columns().map_err(mosaic_read_error)?; - let batch = crate::arrow::legacy::from_legacy_batch(&batch)?; take_row_slices(batch, plan.selected_slices.as_deref(), read_schema) } @@ -822,9 +821,7 @@ fn validate_mosaic_arrow_type(data_type: &ArrowDataType) -> Result<(), String> { if crate::arrow::multiset::contains_multiset(data_type) { return Err("MULTISET is not supported by Mosaic".into()); } - let data_type = - crate::arrow::legacy::to_legacy_data_type(data_type).map_err(|error| error.to_string())?; - paimon_mosaic_core::types::validate_data_type(&data_type) + paimon_mosaic_core::types::validate_data_type(data_type) } fn selected_slices_for_row_group( @@ -1225,7 +1222,7 @@ mod tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - &crate::arrow::legacy::to_legacy_schema(batches[0].schema().as_ref()).unwrap(), + batches[0].schema().as_ref(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -1236,9 +1233,7 @@ mod tests { ) .unwrap(); for batch in batches { - writer - .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) - .unwrap(); + writer.write_batch(batch).unwrap(); } writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) @@ -1442,7 +1437,7 @@ mod tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - &crate::arrow::legacy::to_legacy_schema(batches[0].schema().as_ref()).unwrap(), + batches[0].schema().as_ref(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 1, @@ -1453,9 +1448,7 @@ mod tests { ) .unwrap(); for batch in batches { - writer - .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) - .unwrap(); + writer.write_batch(batch).unwrap(); } writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) diff --git a/crates/paimon/src/arrow/format/mosaic_write.rs b/crates/paimon/src/arrow/format/mosaic_write.rs index 9a026da08..23b6267e7 100644 --- a/crates/paimon/src/arrow/format/mosaic_write.rs +++ b/crates/paimon/src/arrow/format/mosaic_write.rs @@ -166,8 +166,7 @@ impl MosaicFormatWriter { }) }) .collect::>>()?; - let legacy_schema = crate::arrow::legacy::to_legacy_schema(&physical_schema)?; - let encoder = MosaicWriter::new(PendingOutput::default(), &legacy_schema, options) + let encoder = MosaicWriter::new(PendingOutput::default(), &physical_schema, options) .map_err(mosaic_write_error)?; Ok(Self { output: output.writer().await?, @@ -331,7 +330,7 @@ impl FormatFileWriter for MosaicFormatWriter { }; let groups_before = self.encoder.num_row_groups(); self.encoder - .write_batch(&crate::arrow::legacy::to_legacy_batch(&batch)?) + .write_batch(&batch) .map_err(mosaic_write_error)?; self.pending_rows = if self.encoder.num_row_groups() > groups_before { 0 diff --git a/crates/paimon/src/arrow/format/vortex.rs b/crates/paimon/src/arrow/format/vortex.rs index 9ad65e852..7246c1cf7 100644 --- a/crates/paimon/src/arrow/format/vortex.rs +++ b/crates/paimon/src/arrow/format/vortex.rs @@ -28,10 +28,9 @@ use arrow_schema::{DataType as ArrowDataType, SchemaRef}; use async_trait::async_trait; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, OnceLock}; -use vortex::array::arrow::{ArrowSessionExt, FromArrowArray}; -use vortex::array::dtype::arrow::FromArrowType; use vortex::array::dtype::DType; use vortex::array::{ArrayRef, VortexSessionExecute}; +use vortex::arrow::ArrowSessionExt; use vortex::buffer::ByteBuffer; use vortex::file::{OpenOptionsSessionExt, WriteOptionsSessionExt}; use vortex::io::runtime::current::CurrentThreadRuntime; @@ -181,9 +180,9 @@ fn read_vortex_batches( .iter() .any(|field| matches!(field.data_type(), ArrowDataType::Struct(_))) { - // Vortex 0.75 can apply a child ROW's validity to its parent while - // projecting recursive struct layouts. Read complete ROWs and their - // independent validity before applying selection and outer projection. + // Preserve each ROW's independent validity when walking recursive + // layouts, including files written by older Vortex versions. Apply + // selection and outer projection after reconstructing complete ROWs. let names = scan_schema .fields() .iter() @@ -197,8 +196,10 @@ fn read_vortex_batches( unreachable!() }; let indices = arrow_array::UInt64Array::from_iter_values(rows.iter()); - let indices = - ArrayRef::from_arrow(&indices, false).map_err(|error| Error::DataInvalid { + let indices = session + .arrow() + .from_arrow_array(Arc::new(indices), false) + .map_err(|error| Error::DataInvalid { message: format!("Failed to build Vortex row selection: {error}"), source: None, })?; @@ -230,7 +231,14 @@ fn read_vortex_batches( // Apply column projection. { use vortex::array::expr::{root, select}; - scan_builder = scan_builder.with_projection(select(projected_names, root())); + scan_builder = scan_builder.with_projection( + select(projected_names, root()) + .bind(vortex_file.dtype()) + .map_err(|e| Error::DataInvalid { + message: format!("Failed to bind Vortex projection: {e}"), + source: None, + })?, + ); } // Vortex 0.68 filtered scans can block indefinitely on some runtimes. @@ -424,9 +432,9 @@ fn vortex_array_to_record_batch( vortex_array: ArrayRef, schema: &SchemaRef, ) -> crate::Result { - let target_field = arrow_schema_58::Field::new( + let target_field = arrow_schema::Field::new( "", - crate::arrow::legacy::to_legacy_data_type(&ArrowDataType::Struct(schema.fields().clone()))?, + ArrowDataType::Struct(schema.fields().clone()), vortex_array.dtype().is_nullable(), ); let mut ctx = session.create_execution_ctx(); @@ -438,7 +446,6 @@ fn vortex_array_to_record_batch( source: None, })?; - let arrow_array = crate::arrow::legacy::from_legacy_array(&arrow_array)?; let struct_array = arrow_array .as_any() .downcast_ref::() @@ -476,6 +483,8 @@ fn vortex_array_to_record_batch( /// writes all accumulated arrays through Vortex into an in-memory buffer before /// flushing that buffer to Paimon's output file. pub(crate) struct VortexFormatWriter { + /// Reuse the Arrow conversion registry across input batches. + session: VortexSession, /// Vortex dtype derived from the target Arrow schema. dtype: DType, /// Converted arrays pending final Vortex write. @@ -491,10 +500,17 @@ pub(crate) struct VortexFormatWriter { impl VortexFormatWriter { pub(crate) async fn new(output: &OutputFile, schema: SchemaRef) -> crate::Result { validate_vortex_schema(&schema)?; - let dtype = DType::from_arrow(Arc::new(crate::arrow::legacy::to_legacy_schema(&schema)?)); + let session = VortexSession::default(); + let dtype = session + .arrow() + .from_arrow_schema(schema.as_ref()) + .map_err(|e| Error::Unsupported { + message: format!("Failed to convert Vortex schema: {e}"), + })?; let bytes_written = Arc::new(AtomicU64::new(0)); Ok(Self { + session, dtype, arrays: Vec::new(), output: output.clone(), @@ -504,8 +520,8 @@ impl VortexFormatWriter { } } -/// `DType::from_arrow` ends in `unimplemented!` for a type it has no arm for, so screen -/// the schema first. MAP and MULTISET remain unsupported at any depth, so the +/// Screen the schema before Arrow conversion. MAP and MULTISET remain +/// unsupported at any depth, so the /// walk also checks the metadata marking the Arrow list used for MULTISET. fn validate_vortex_schema(schema: &SchemaRef) -> crate::Result<()> { for field in schema.fields() { @@ -544,7 +560,10 @@ fn contains_unconvertible_type(data_type: &ArrowDataType) -> bool { impl FormatFileWriter for VortexFormatWriter { async fn write(&mut self, batch: &RecordBatch) -> crate::Result<()> { let staged_bytes = batch.get_array_memory_size(); - let vortex_arr = ArrayRef::from_arrow(crate::arrow::legacy::to_legacy_batch(batch)?, false) + let vortex_arr = self + .session + .arrow() + .from_arrow_record_batch(batch.clone(), batch.schema_ref()) .map_err(|e| Error::DataInvalid { message: format!("Failed to convert RecordBatch to Vortex: {e}"), source: None, @@ -572,6 +591,7 @@ impl FormatFileWriter for VortexFormatWriter { async fn close(self: Box) -> crate::Result { let this = *self; let VortexFormatWriter { + session: _, dtype, arrays, output, @@ -653,6 +673,116 @@ mod tests { use futures::StreamExt; use std::ops::Range; + #[tokio::test] + async fn reads_vortex_0_75_file_with_nested_nulls_projection_and_selection() { + use crate::spec::{IntType, RowType}; + use arrow_array::{Array, StructArray}; + use futures::TryStreamExt; + + let bytes = Bytes::from_static(include_bytes!( + "../../../testdata/vortex_compat/vortex_0_75.vortex" + )); + let fields = vec![ + DataField::new(0, "id".into(), DataType::Int(IntType::new())), + DataField::new( + 1, + "name".into(), + DataType::VarChar(VarCharType::string_type()), + ), + DataField::new( + 2, + "profile".into(), + DataType::Row(RowType::new(vec![DataField::new( + 3, + "detail".into(), + DataType::Row(RowType::new(vec![DataField::new( + 4, + "keep".into(), + DataType::Int(IntType::new()), + )])), + )])), + ), + ]; + for sparse in [false, true] { + let projection = if sparse { + vec![fields[2].clone(), fields[0].clone()] + } else { + fields.clone() + }; + let selection = sparse.then(|| vec![RowRange::new(1, 2), RowRange::new(511, 511)]); + let batches = VortexFormatReader + .read_batch_stream( + Box::new(WholeFileOnlyRead { + bytes: bytes.clone(), + }), + bytes.len() as u64, + &projection, + None, + None, + selection, + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let schema = crate::arrow::build_target_arrow_schema(&projection).unwrap(); + let batch = arrow_select::concat::concat_batches(&schema, &batches).unwrap(); + let expected_ids = if sparse { + vec![1, 2, 511] + } else { + (0..512).collect() + }; + let ids = batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(ids.values().as_ref(), expected_ids); + let profile = batch + .column_by_name("profile") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let detail = profile + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let keep = detail + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for (row, id) in expected_ids.iter().enumerate() { + assert_eq!(profile.is_valid(row), id % 4 != 2); + if profile.is_valid(row) { + assert_eq!(detail.is_valid(row), id % 4 != 1); + if detail.is_valid(row) { + assert_eq!(keep.value(row), if id % 4 == 0 { 7 } else { 9 }); + } + } + if !sparse { + let name = batch + .column_by_name("name") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(name.is_null(row), id % 4 == 1); + if name.is_valid(row) { + assert_eq!( + name.value(row), + ["one", "", "three", "four"][*id as usize % 4] + ); + } + } + } + } + } + fn test_arrow_schema() -> Arc { Arc::new(ArrowSchema::new(vec![ ArrowField::new("id", ArrowDataType::Int32, false), @@ -697,9 +827,13 @@ mod tests { Some(NullBuffer::from(vec![true, true, false])), ); let original = RecordBatch::try_new(schema.clone(), vec![Arc::new(profile)]).unwrap(); + let session = VortexSession::default(); let memory = vortex_array_to_record_batch( - &VortexSession::default(), - ArrayRef::from_arrow(original.clone(), false).unwrap(), + &session, + session + .arrow() + .from_arrow_record_batch(original.clone(), original.schema_ref()) + .unwrap(), &schema, ) .unwrap(); diff --git a/crates/paimon/src/arrow/format/vortex/nested.rs b/crates/paimon/src/arrow/format/vortex/nested.rs index b19dfd2a0..a9287b59e 100644 --- a/crates/paimon/src/arrow/format/vortex/nested.rs +++ b/crates/paimon/src/arrow/format/vortex/nested.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -//! Read complete ROWs without Vortex 0.75's recursive StructReader projection. -//! Each ROW has independent validity; a null child must not null its parent. +//! Read complete ROWs with independent validity, including older Vortex layouts. +//! A null child must not null its parent during recursive field projection. use super::CurrentThreadRuntime; use crate::{Error, Result}; @@ -56,6 +56,13 @@ pub(super) fn read_array( ) } +fn child(layout: &LayoutRef, slot: usize) -> Result { + layout + .slot(slot) + .map_err(invalid)? + .ok_or_else(|| invalid(format!("missing layout child slot {slot}"))) +} + fn projected_dtype(dtype: &DType, names: Option<&[String]>) -> Result { let Some(names) = names else { return Ok(dtype.clone()); @@ -101,7 +108,7 @@ fn read_layout( runtime, session, source.clone(), - &layout.child(0).map_err(invalid)?, + &child(layout, 0)?, None, )?) } else { @@ -112,9 +119,7 @@ fn read_layout( let index = original .find(name) .ok_or_else(|| invalid(format!("missing root field {name}")))?; - let child = layout - .child(index + usize::from(nullable)) - .map_err(invalid)?; + let child = child(layout, index + 1)?; columns.push(read_layout(runtime, session, source.clone(), &child, None)?); } return StructArray::try_new( @@ -133,7 +138,7 @@ fn read_layout( runtime, session, source.clone(), - &layout.child(index).map_err(invalid)?, + &child(layout, index)?, roots, )?); } @@ -143,28 +148,19 @@ fn read_layout( } if layout.is::() { // Values use dictionary coordinates, codes use row coordinates. - let values = read_layout( - runtime, - session, - source.clone(), - &layout.child(0).map_err(invalid)?, - roots, - )?; - let codes = read_layout( - runtime, - session, - source, - &layout.child(1).map_err(invalid)?, - None, - )?; + let values = read_layout(runtime, session, source.clone(), &child(layout, 0)?, roots)?; + let codes = read_layout(runtime, session, source, &child(layout, 1)?, None)?; return DictArray::try_new(codes, values) .map(IntoArray::into_array) .map_err(invalid); } // Zone maps wrap data without changing its schema or row coordinates. - for (index, kind) in layout.child_types().enumerate() { - if matches!(kind, LayoutChildType::Transparent(_)) { - let child = layout.child(index).map_err(invalid)?; + for index in 0..layout.nslots() { + if matches!( + layout.slot_type(index), + Some(LayoutChildType::Transparent(_)) + ) { + let child = child(layout, index)?; if child.dtype() == layout.dtype() && child.row_count() == layout.row_count() { return read_layout(runtime, session, source, &child, roots); } @@ -217,11 +213,10 @@ fn read_layout( #[cfg(test)] mod tests { use super::*; - use vortex::array::dtype::arrow::FromArrowType; + use vortex::arrow::ArrowSessionExt; use vortex::io::session::RuntimeSessionExt; use vortex::layout::layouts::chunked::ChunkedLayout; use vortex::layout::segments::{SegmentFuture, SegmentId}; - use vortex::layout::IntoLayout; use vortex::layout::LayoutChildren; use vortex::VortexSessionDefault; @@ -258,10 +253,14 @@ mod tests { true, ), ])); - let layout = - ChunkedLayout::new(0, DType::from_arrow(schema.clone()), Arc::new(Empty)).into_layout(); let runtime = CurrentThreadRuntime::new(); let session = VortexSession::default().with_handle(runtime.handle()); + let layout = ChunkedLayout::new( + 0, + session.arrow().from_arrow_schema(&schema).unwrap(), + Arc::new(Empty), + ) + .into_layout(); let array = read_layout( &runtime, &session, diff --git a/crates/paimon/src/arrow/legacy.rs b/crates/paimon/src/arrow/legacy.rs deleted file mode 100644 index 6ff2d18e9..000000000 --- a/crates/paimon/src/arrow/legacy.rs +++ /dev/null @@ -1,189 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Zero-copy C Data Interface bridge for Mosaic and Vortex's Arrow 58 APIs. -//! Release callbacks keep ownership of buffers in the exporting Arrow version. - -use std::sync::Arc; - -use arrow_array::{Array, ArrayRef, RecordBatch, RecordBatchOptions}; -use arrow_schema::{DataType, Schema}; - -use crate::{Error, Result}; - -fn bridge_error(source: impl std::error::Error + Send + Sync + 'static) -> Error { - Error::DataInvalid { - message: "Cannot bridge Arrow 58 and Arrow 59 data".into(), - source: Some(Box::new(source)), - } -} - -pub(crate) fn to_legacy_schema(schema: &Schema) -> Result { - let schema = arrow_schema::ffi::FFI_ArrowSchema::try_from(schema).map_err(bridge_error)?; - // SAFETY: Both versions implement the same repr(C) ArrowSchema ABI. The - // exporter remains alive while the importer copies the schema and metadata. - let schema = - unsafe { &*std::ptr::from_ref(&schema).cast::() }; - arrow_schema_58::Schema::try_from(schema).map_err(bridge_error) -} - -fn from_legacy_schema(schema: &arrow_schema_58::Schema) -> Result { - let schema = arrow_schema_58::ffi::FFI_ArrowSchema::try_from(schema).map_err(bridge_error)?; - // SAFETY: ArrowSchema has the same C ABI in both versions, and the exported - // schema outlives this import, which copies its fields and metadata. - let schema = - unsafe { &*std::ptr::from_ref(&schema).cast::() }; - Schema::try_from(schema).map_err(bridge_error) -} - -pub(crate) fn to_legacy_data_type(data_type: &DataType) -> Result { - let schema = Schema::new(vec![arrow_schema::Field::new("", data_type.clone(), true)]); - Ok(to_legacy_schema(&schema)?.field(0).data_type().clone()) -} - -fn to_legacy_array(array: &ArrayRef) -> Result { - let (mut array, schema) = arrow_array::ffi::to_ffi(&array.to_data()).map_err(bridge_error)?; - // SAFETY: The C Data Interface structs share a stable ABI. from_raw moves - // ownership and marks the source array released, preventing a double drop. - // from_ffi retains the exporter's release callback until all buffers die. - let data = unsafe { - let array = - arrow_array_58::ffi::FFI_ArrowArray::from_raw(std::ptr::from_mut(&mut array).cast()); - let schema = &*std::ptr::from_ref(&schema).cast(); - arrow_array_58::ffi::from_ffi(array, schema) - } - .map_err(bridge_error)?; - Ok(arrow_array_58::make_array(data)) -} - -pub(crate) fn from_legacy_array(array: &arrow_array_58::ArrayRef) -> Result { - let (mut array, schema) = - arrow_array_58::ffi::to_ffi(&array.to_data()).map_err(bridge_error)?; - // SAFETY: This is the inverse of to_legacy_array: the shared C ABI moves - // ownership via from_raw, and imported buffers retain the release callback. - let data = unsafe { - let array = - arrow_array::ffi::FFI_ArrowArray::from_raw(std::ptr::from_mut(&mut array).cast()); - let schema = &*std::ptr::from_ref(&schema).cast(); - arrow_array::ffi::from_ffi(array, schema) - } - .map_err(bridge_error)?; - Ok(arrow_array::make_array(data)) -} - -pub(crate) fn to_legacy_batch(batch: &RecordBatch) -> Result { - let schema = Arc::new(to_legacy_schema(batch.schema().as_ref())?); - let columns = batch - .columns() - .iter() - .map(to_legacy_array) - .collect::>()?; - arrow_array_58::RecordBatch::try_new_with_options( - schema, - columns, - &arrow_array_58::RecordBatchOptions::new().with_row_count(Some(batch.num_rows())), - ) - .map_err(bridge_error) -} - -pub(crate) fn from_legacy_batch(batch: &arrow_array_58::RecordBatch) -> Result { - let schema = Arc::new(from_legacy_schema(batch.schema().as_ref())?); - let columns = batch - .columns() - .iter() - .map(from_legacy_array) - .collect::>()?; - RecordBatch::try_new_with_options( - schema, - columns, - &RecordBatchOptions::new().with_row_count(Some(batch.num_rows())), - ) - .map_err(bridge_error) -} - -#[cfg(test)] -mod tests { - use super::*; - use arrow_array::{types::Int32Type, Int32Array, ListArray, StringArray}; - use arrow_schema::Field; - - #[test] - fn round_trip_preserves_slices_nulls_nested_values_metadata_and_buffer_ownership() { - let ids = Arc::new(Int32Array::from(vec![Some(0), Some(1), None, Some(3)])); - let names = Arc::new(StringArray::from(vec![ - Some("skip"), - Some("one"), - None, - Some("三"), - ])); - let lists = Arc::new(ListArray::from_iter_primitive::(vec![ - Some(vec![Some(0)]), - Some(vec![Some(1), None]), - None, - Some(vec![]), - ])); - let schema = Arc::new(Schema::new_with_metadata( - vec![ - Field::new("id", DataType::Int32, true) - .with_metadata([("field_id".into(), "42".into())].into()), - Field::new("name", DataType::Utf8, true), - Field::new("values", lists.data_type().clone(), true), - ], - [("table".into(), "bridge".into())].into(), - )); - let original = RecordBatch::try_new(schema, vec![ids, names, lists]) - .unwrap() - .slice(1, 3); - let expected = original.clone(); - let legacy = to_legacy_batch(&original).unwrap(); - drop(original); - let imported = from_legacy_batch(&legacy).unwrap(); - drop(legacy); - assert_eq!(imported, expected); - let ids = imported - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let original_ids = expected - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(ids.values().as_ptr(), original_ids.values().as_ptr()); - drop(expected); - assert_eq!(ids.value(0), 1); - assert!(ids.is_null(1)); - assert_eq!(ids.value(2), 3); - } - - #[test] - fn round_trip_preserves_row_counts_without_columns() { - for rows in [0, 7] { - let batch = RecordBatch::try_new_with_options( - Arc::new(Schema::empty()), - vec![], - &RecordBatchOptions::new().with_row_count(Some(rows)), - ) - .unwrap(); - assert_eq!( - from_legacy_batch(&to_legacy_batch(&batch).unwrap()).unwrap(), - batch - ); - } - } -} diff --git a/crates/paimon/src/arrow/mod.rs b/crates/paimon/src/arrow/mod.rs index fc353c6c4..403301482 100644 --- a/crates/paimon/src/arrow/mod.rs +++ b/crates/paimon/src/arrow/mod.rs @@ -16,7 +16,6 @@ // under the License. mod date_extract; -pub(crate) mod legacy; pub(crate) mod multiset; pub(crate) use date_extract::evaluate_date_extract; mod cast_width; diff --git a/crates/paimon/src/table/data_file_reader.rs b/crates/paimon/src/table/data_file_reader.rs index 4cc7d7cb3..7b775da3a 100644 --- a/crates/paimon/src/table/data_file_reader.rs +++ b/crates/paimon/src/table/data_file_reader.rs @@ -3079,7 +3079,7 @@ mod tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - &crate::arrow::legacy::to_legacy_schema(batch.schema().as_ref()).unwrap(), + batch.schema().as_ref(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -3088,9 +3088,7 @@ mod tests { }, ) .unwrap(); - writer - .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) - .unwrap(); + writer.write_batch(batch).unwrap(); writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) } @@ -3439,7 +3437,7 @@ mod tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - &crate::arrow::legacy::to_legacy_schema(batches[0].schema().as_ref()).unwrap(), + batches[0].schema().as_ref(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -3451,9 +3449,7 @@ mod tests { ) .unwrap(); for batch in batches { - writer - .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) - .unwrap(); + writer.write_batch(batch).unwrap(); } writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) diff --git a/crates/paimon/src/table/pk_full_text_read.rs b/crates/paimon/src/table/pk_full_text_read.rs index b230e87a7..e808bc301 100644 --- a/crates/paimon/src/table/pk_full_text_read.rs +++ b/crates/paimon/src/table/pk_full_text_read.rs @@ -571,7 +571,7 @@ mod read_tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - &crate::arrow::legacy::to_legacy_schema(batch.schema().as_ref()).unwrap(), + batch.schema().as_ref(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -580,9 +580,7 @@ mod read_tests { }, ) .unwrap(); - writer - .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) - .unwrap(); + writer.write_batch(batch).unwrap(); writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) } diff --git a/crates/paimon/src/table/pk_vector_indexed_split_read.rs b/crates/paimon/src/table/pk_vector_indexed_split_read.rs index 8edca20be..74df0f4b3 100644 --- a/crates/paimon/src/table/pk_vector_indexed_split_read.rs +++ b/crates/paimon/src/table/pk_vector_indexed_split_read.rs @@ -444,7 +444,7 @@ mod e2e_tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - &crate::arrow::legacy::to_legacy_schema(batch.schema().as_ref()).unwrap(), + batch.schema().as_ref(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -453,9 +453,7 @@ mod e2e_tests { }, ) .unwrap(); - writer - .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) - .unwrap(); + writer.write_batch(batch).unwrap(); writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) } @@ -464,7 +462,7 @@ mod e2e_tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - &crate::arrow::legacy::to_legacy_schema(batches[0].schema().as_ref()).unwrap(), + batches[0].schema().as_ref(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -474,9 +472,7 @@ mod e2e_tests { ) .unwrap(); for batch in batches { - writer - .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) - .unwrap(); + writer.write_batch(batch).unwrap(); } writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) diff --git a/crates/paimon/src/table/pk_vector_orchestrator.rs b/crates/paimon/src/table/pk_vector_orchestrator.rs index 7d20ca0c1..0c6aa60b6 100644 --- a/crates/paimon/src/table/pk_vector_orchestrator.rs +++ b/crates/paimon/src/table/pk_vector_orchestrator.rs @@ -988,7 +988,7 @@ mod e2e_tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - &crate::arrow::legacy::to_legacy_schema(batch.schema().as_ref()).unwrap(), + batch.schema().as_ref(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -997,9 +997,7 @@ mod e2e_tests { }, ) .unwrap(); - writer - .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) - .unwrap(); + writer.write_batch(batch).unwrap(); writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) } diff --git a/crates/paimon/src/table/pk_vector_position_read.rs b/crates/paimon/src/table/pk_vector_position_read.rs index 4083fd9f5..dbc720dc6 100644 --- a/crates/paimon/src/table/pk_vector_position_read.rs +++ b/crates/paimon/src/table/pk_vector_position_read.rs @@ -346,7 +346,7 @@ mod tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - &crate::arrow::legacy::to_legacy_schema(batch.schema().as_ref()).unwrap(), + batch.schema().as_ref(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -355,9 +355,7 @@ mod tests { }, ) .unwrap(); - writer - .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) - .unwrap(); + writer.write_batch(batch).unwrap(); writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) } @@ -368,7 +366,7 @@ mod tests { let out = MemOutputFile::new(); let mut writer = MosaicWriter::new( out, - &crate::arrow::legacy::to_legacy_schema(batches[0].schema().as_ref()).unwrap(), + batches[0].schema().as_ref(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -378,9 +376,7 @@ mod tests { ) .unwrap(); for batch in batches { - writer - .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) - .unwrap(); + writer.write_batch(batch).unwrap(); } writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) diff --git a/crates/paimon/src/table/pk_vector_read/residual_row_ranges_tests.rs b/crates/paimon/src/table/pk_vector_read/residual_row_ranges_tests.rs index 482949583..ba533e073 100644 --- a/crates/paimon/src/table/pk_vector_read/residual_row_ranges_tests.rs +++ b/crates/paimon/src/table/pk_vector_read/residual_row_ranges_tests.rs @@ -73,7 +73,7 @@ fn id_batch(ids: Vec) -> RecordBatch { fn write_mosaic(batch: &RecordBatch) -> Bytes { let mut writer = MosaicWriter::new( MemOutputFile { data: Vec::new() }, - &crate::arrow::legacy::to_legacy_schema(batch.schema().as_ref()).unwrap(), + batch.schema().as_ref(), WriterOptions { compression: COMPRESSION_NONE, num_buckets: 2, @@ -82,9 +82,7 @@ fn write_mosaic(batch: &RecordBatch) -> Bytes { }, ) .unwrap(); - writer - .write_batch(&crate::arrow::legacy::to_legacy_batch(batch).unwrap()) - .unwrap(); + writer.write_batch(batch).unwrap(); writer.close().unwrap(); Bytes::from(writer.output().data.to_vec()) } diff --git a/crates/paimon/testdata/vortex_compat/generate.py b/crates/paimon/testdata/vortex_compat/generate.py new file mode 100644 index 000000000..95eb75751 --- /dev/null +++ b/crates/paimon/testdata/vortex_compat/generate.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Generate with Python 3.11, vortex-data==0.75.0 and pyarrow==23.0.1.""" +from pathlib import Path +import pyarrow as pa +import vortex +import vortex.io + +assert vortex.__version__ == "0.75.0" +profile_type = pa.struct([pa.field("detail", pa.struct([pa.field("keep", pa.int32())]))]) +profiles = [{"detail": {"keep": 7}}, {"detail": None}, None, {"detail": {"keep": 9}}] * 128 +table = pa.table({ + "id": pa.array(range(512), type=pa.int32()), + "name": pa.array(["one", None, "three", "four"] * 128), + "profile": pa.array(profiles, type=profile_type), +}) +vortex.io.write(vortex.array(table), str(Path(__file__).with_name("vortex_0_75.vortex"))) diff --git a/crates/paimon/testdata/vortex_compat/vortex_0_75.vortex b/crates/paimon/testdata/vortex_compat/vortex_0_75.vortex new file mode 100644 index 0000000000000000000000000000000000000000..e965ff667f646e70677a94123f723f6f9ee235f9 GIT binary patch literal 5984 zcmeHLO=w$35MKRQR#ewfU1QV`f+!~R5JC^3hv0(=A>EC@yMxRH71=@%F(F1kS?jQbSWJP@ojZe#c1u$cW>MoOUB&<3{Po{stKWUg^CjY zIrL|Mr+{251&RHLehw1D(&wWnbe8E(UcNf+1>!21dan0qEzb3*_`C6iv@ErD}Tm!H!nwDlX)l9J7}K;QZL6b z&ZQmVOkZfD1Boq&$r#iAE)3dR6KW0u&t-k#RC4a#=-;x7{?#+W&UviVv^f0l*F=Wd zM-}hio)P{lJ6X?I(RS;5=QtzjB=25eT|?3*^WNnVY00|gYULg)FsOb_g2z2;wo2OW z@4@@_dDM#%4o%w0T4J{+S_anayRerfgT-ZDwO@ty*WHk+EOl-u`CAN3PlpRxqf zA8xYd?o(&LWG$s`^6v@V=*)t*0oqky6?gT|FnvbjFJft7XsQU3ZidOxwU&UecGw2j^kD@&WZFsOd*+3`&2jy0_tR^id1Xd%>E z9nIDn?NF6hExmUgU0hw^a#^qtO1$>3_t9DowzyqWv@t~ZOUo-L<(pYOyDdpnBKJlW~O}skt z+`CmBd49mV*C;fbyBEkq^n;Z)531ecDKmRVCS!c%H$O*;e2RWf0=e%wC+kHsY}EWf zOTX?dYLKMP`&A=zTWrmk`+-Yj$Vd*Oe*_Lv*ZpneIOGYCaT>uk;vK}?Ch>w6iC;o5 zo40_}8OWKE5*eNsVi;cHb6KfnOF~oRPe7j9-M-_SXZLNHm_8fqGuU}l#%cAF>{dyC!w5MkE zB>kcfybm!aOrB)r4#{=l7zdQ2O=b;Ew$mm?o3cBtkDLGD06)x`H7Yui^NVgy2rxF9 zx@+!XmpPu?iGMW*oCZ=a`=9?A$9(f@ zz0?RJy?nS8h3!g&{$0y^smrVS;5@K&+SD22MJ;sIX8)L4;6xO;3eQ3a2i+! zW`UnRHNGbNS!qR8^@XJ;VzA<0Sn)1k%#W7GT(G__d7KQ3(7b-fDNYnI0xv3@i| zf6it$d^82GJ7qH@6U68gwf zZ}rn_EznO}I)9t8cGdi4U7c92KlR&~W+^n%&> z_ai-vSW7bBwQ`BIw{qtGLLXW0tX)+v(viQY-EdVvo$850rBQE1OrC?5m(X2VXxAlx z$NI<7uK7kIQ0FZ#5kH|XS$bkmdO?&op6J%qSGUN_`%%-Y&`H{2>a;n3vHjI5kg$%u%>EnL z_XO^H_VY9da0AHxt)Bi24_AO2FCQl8-v(0tjyWXM*H+)My^vpa`WaM!<8y5ZBdErv zq86nNBem2S;31%^xg~J;9YwzYnKQ6ufrY`sBg72N%so7=m_cY2pn2=KEA*L3Ay7bm UZRJnF)(v|Yq|}-^hu_x!1~MOVq5uE@ literal 0 HcmV?d00001 diff --git a/deny.toml b/deny.toml index 59bd5bdda..b0e6ce62b 100644 --- a/deny.toml +++ b/deny.toml @@ -18,14 +18,11 @@ [advisories] ignore = [ { id = "RUSTSEC-2023-0071", reason = "No fixed version; reqsign only signs locally generated outbound requests and exposes no remote signing or decryption timing oracle." }, - { id = "RUSTSEC-2026-0176", reason = "The affected PyList and PyTuple iterator APIs are unused." }, - { id = "RUSTSEC-2026-0177", reason = "PyCFunction::new_closure is unused." }, # zstd 0.13 is still required by Arrow, DataFusion, Parquet, and other upstream crates. # Revisit these API-specific exceptions as those dependencies adopt zstd 0.14. { id = "RUSTSEC-2026-0391", reason = "No caller uses the affected raw Encoder/Decoder::with_context constructors." }, { id = "RUSTSEC-2026-0398", reason = "The affected zstd-safe seekable feature is not enabled." }, { id = "RUSTSEC-2026-0403", reason = "No old zstd caller uses write::Decoder or raw streaming decode; compression-codecs now uses patched zstd-safe 8.2." }, - { id = "RUSTSEC-2026-0435", reason = "Parquet 58 pins thrift 0.17 and only uses the compact protocol; the affected TBinaryInputProtocol::read_bytes is unused." }, ] [licenses] From c353c94daecf66114aa994a11ac5fc4d948c292b Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Tue, 3 Nov 2026 15:23:26 +0800 Subject: [PATCH 5/6] [python] Fix writer GC and URI callback lifetimes Allow batch and stream writers to be collected on any Python thread while retaining exclusive method access. Track every Python reference retained by URI factories, readers, and streams, retire released references, and close live streams once outside locks. Suspend the runtime for cancellation cleanup callbacks so they can reenter Native APIs. Cover cycle collection, thread handoff, replacement failures, cleanup errors, and runtime reentry. --- bindings/python/README.md | 6 + bindings/python/src/blob_uri_reader.rs | 388 ++++++++++++++++-- bindings/python/src/write.rs | 108 +++-- bindings/python/tests/test_blob_consumer.py | 101 +++++ bindings/python/tests/test_blob_uri_reader.py | 132 ++++++ 5 files changed, 652 insertions(+), 83 deletions(-) diff --git a/bindings/python/README.md b/bindings/python/README.md index fd549a086..a00c6c534 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -280,6 +280,12 @@ the ending snapshot, including when no selected files remain. The existing ### Custom Blob URI sources +Batch and stream writers can be transferred between Python threads. Operations +remain exclusive: calling another writer method while a write, prepare, or close +is running raises `RuntimeError`. Call `close()` in a `finally` block to discard +unprepared output and release source streams. Callback reference cycles are also +visible to Python's garbage collector when collection runs on another thread. + Before the first Arrow write, call `writer.with_blob_uri_reader_factory(factory)` to copy referenced Blob payloads from custom sources. The factory provides `create(uri)`, returning a reader with diff --git a/bindings/python/src/blob_uri_reader.rs b/bindings/python/src/blob_uri_reader.rs index c5cd2f913..aae49eb8d 100644 --- a/bindings/python/src/blob_uri_reader.rs +++ b/bindings/python/src/blob_uri_reader.rs @@ -20,9 +20,11 @@ use bytes::Bytes; use paimon::io::{UriInputStream, UriReader, UriReaderFactory}; -use pyo3::exceptions::PyTypeError; +use pyo3::exceptions::{PyRuntimeError, PyTypeError}; use pyo3::prelude::*; +use pyo3::{PyTraverseError, PyVisit}; use std::collections::HashMap; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, Weak}; struct PythonCallbackError(PyErr); @@ -81,8 +83,132 @@ pub(crate) fn to_callback_error(error: paimon::Error) -> PyErr { crate::error::to_py_err(error) } +struct PythonUriReference { + value: Mutex>>, + stream: bool, +} + +impl PythonUriReference { + fn new( + value: Py, + stream: bool, + references: Option<&Arc>, + ) -> Arc { + let reference = Arc::new(Self { + value: Mutex::new(Some(value)), + stream, + }); + if let Some(references) = references { + references.register(reference.clone()); + } + reference + } + + fn clone_ref(&self, py: Python<'_>) -> PyResult> { + self.value + .lock() + .unwrap() + .as_ref() + .map(|value| value.clone_ref(py)) + .ok_or_else(|| PyRuntimeError::new_err("URI callback has been cleared or closed")) + } + + fn take(&self) -> Option> { + self.value.lock().unwrap().take() + } + + fn close(&self) -> PyResult<()> { + match self.take() { + Some(stream) => tokio::task::block_in_place(move || { + Python::attach(move |py| { + stream.bind(py).call_method0("close")?; + Ok(()) + }) + }), + None => Ok(()), + } + } +} + +/// Expose every Python reference retained by a writer's URI adapters to GC. +/// The registry owns slots strongly, so traversal never upgrades and drops the +/// last owner of a Python reference while attaching to Python is forbidden. +#[derive(Default)] +pub(crate) struct PythonUriReferences { + references: Mutex>>, + prune_at: AtomicUsize, +} + +impl PythonUriReferences { + fn register(&self, reference: Arc) { + let prune = { + let mut references = self.references.lock().unwrap(); + references.push(reference); + references.len() >= self.prune_at.load(Ordering::Relaxed) + }; + if prune { + self.prune(); + } + } + + pub(crate) fn traverse(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + for reference in self.references.lock().unwrap().iter() { + // Each slot owns one Py reference, including when several slots + // happen to point at the same Python object. + visit.call(&*reference.value.lock().unwrap())?; + } + Ok(()) + } + + pub(crate) fn prune(&self) { + let retired = { + let mut references = self.references.lock().unwrap(); + let (retained, retired): (Vec<_>, Vec<_>) = std::mem::take(&mut *references) + .into_iter() + .partition(|reference| Arc::strong_count(reference) > 1); + // Bound retired readers/streams within one large write without + // rescanning every live partition's slots for each new reference. + self.prune_at.store( + retained.len().saturating_mul(2).saturating_add(16), + Ordering::Relaxed, + ); + *references = retained; + retired + }; + // Python finalizers may re-enter Native or the registry itself. + drop(retired); + } + + pub(crate) fn clear(&self) { + let references = std::mem::take(&mut *self.references.lock().unwrap()); + // Take every slot before any Python finalizer or stream close runs. + // A core close/Drop racing this clear shares the same take operation. + let values = references + .iter() + .filter_map(|reference| reference.take().map(|value| (reference.stream, value))) + .collect::>(); + drop(references); + // Cancellation may run on a runtime thread. Close callbacks and Python + // finalizers can call Native again, just like ordinary source callbacks. + tokio::task::block_in_place(move || { + Python::attach(move |py| { + for (stream, value) in values { + if stream { + // Match the cancellation Drop path: cleanup errors must + // not replace the error which caused the writer to clear. + let _ = value.bind(py).call_method0("close"); + } + // Drop Python refs inside the suspended runtime context. + drop(value); + } + }); + }); + } +} + pub(crate) struct PythonUriReaderFactory { - factory: Py, + factory: Arc, + references: Option>, // Preserve object identity across interleaved physical Blob writers without // retaining readers after core releases them. URI caching belongs to the factory. readers: Mutex>>, @@ -102,13 +228,30 @@ pub(crate) fn optional_uri_reader_factory( impl PythonUriReaderFactory { pub(crate) fn new(factory: Py, py: Python<'_>) -> PyResult { + Self::with_references(factory, py, None) + } + + pub(crate) fn tracked( + factory: Py, + py: Python<'_>, + references: Arc, + ) -> PyResult { + Self::with_references(factory, py, Some(references)) + } + + fn with_references( + factory: Py, + py: Python<'_>, + references: Option>, + ) -> PyResult { if !factory.bind(py).getattr("create")?.is_callable() { return Err(PyTypeError::new_err( "URI reader factory must provide callable create", )); } Ok(Self { - factory, + factory: PythonUriReference::new(factory, false, references.as_ref()), + references, readers: Mutex::new(HashMap::new()), }) } @@ -118,7 +261,8 @@ impl UriReaderFactory for PythonUriReaderFactory { fn supports_uri(&self, uri: &str) -> paimon::Result { tokio::task::block_in_place(|| { Python::attach(|py| { - let factory = self.factory.bind(py); + let factory = self.factory.clone_ref(py)?; + let factory = factory.bind(py); if factory.hasattr("_supports_uri")? { factory.call_method1("_supports_uri", (uri,))?.extract() } else { @@ -132,11 +276,8 @@ impl UriReaderFactory for PythonUriReaderFactory { fn create(&self, uri: &str) -> paimon::Result> { tokio::task::block_in_place(|| { Python::attach(|py| { - let reader = self - .factory - .bind(py) - .call_method1("create", (uri,))? - .unbind(); + let factory = self.factory.clone_ref(py)?; + let reader = factory.bind(py).call_method1("create", (uri,))?.unbind(); if !reader.bind(py).getattr("new_input_stream")?.is_callable() { return Err(PyTypeError::new_err( "URI reader must provide callable new_input_stream", @@ -148,8 +289,17 @@ impl UriReaderFactory for PythonUriReaderFactory { return Ok(previous as Arc); } cached.retain(|_, reader| reader.strong_count() != 0); - let reader = Arc::new(PythonUriReader { reader }); + let reader = Arc::new(PythonUriReader { + reader: PythonUriReference::new(reader, false, None), + references: self.references.clone(), + }); cached.insert(identity, Arc::downgrade(&reader)); + // Registration may prune retired slots and run Python + // finalizers. Release the identity cache before that happens. + drop(cached); + if let Some(references) = &self.references { + references.register(reader.reader.clone()); + } Ok(reader as Arc) }) }) @@ -158,7 +308,8 @@ impl UriReaderFactory for PythonUriReaderFactory { } struct PythonUriReader { - reader: Py, + reader: Arc, + references: Option>, } #[async_trait::async_trait] @@ -166,13 +317,13 @@ impl UriReader for PythonUriReader { async fn new_input_stream(&self, uri: &str) -> paimon::Result> { tokio::task::block_in_place(|| { Python::attach(|py| { - let stream = self - .reader + let reader = self.reader.clone_ref(py)?; + let stream = reader .bind(py) .call_method1("new_input_stream", (uri,))? .unbind(); Ok(Box::new(PythonUriInputStream { - stream: Some(stream), + stream: PythonUriReference::new(stream, true, self.references.as_ref()), }) as Box) }) }) @@ -181,7 +332,7 @@ impl UriReader for PythonUriReader { } struct PythonUriInputStream { - stream: Option>, + stream: Arc, } #[async_trait::async_trait] @@ -189,10 +340,8 @@ impl UriInputStream for PythonUriInputStream { async fn read(&mut self, length: usize) -> paimon::Result { tokio::task::block_in_place(|| { Python::attach(|py| { - let bytes = self - .stream - .as_ref() - .expect("open stream") + let stream = self.stream.clone_ref(py)?; + let bytes = stream .bind(py) .call_method1("read", (length,))? .extract::>()?; @@ -205,11 +354,8 @@ impl UriInputStream for PythonUriInputStream { async fn seek(&mut self, offset: u64) -> paimon::Result<()> { tokio::task::block_in_place(|| { Python::attach(|py| { - self.stream - .as_ref() - .expect("open stream") - .bind(py) - .call_method1("seek", (offset,))?; + let stream = self.stream.clone_ref(py)?; + stream.bind(py).call_method1("seek", (offset,))?; Ok(()) }) }) @@ -217,16 +363,7 @@ impl UriInputStream for PythonUriInputStream { } async fn close(&mut self) -> paimon::Result<()> { - match self.stream.take() { - Some(stream) => tokio::task::block_in_place(|| { - Python::attach(|py| { - stream.bind(py).call_method0("close")?; - Ok(()) - }) - }) - .map_err(callback_error), - None => Ok(()), - } + self.stream.close().map_err(callback_error) } } @@ -234,10 +371,183 @@ impl Drop for PythonUriInputStream { fn drop(&mut self) { // Async close is explicit in core. On cancellation, still release a // Python stream without replacing the triggering error. - if let Some(stream) = self.stream.take() { - Python::attach(|py| { - let _ = stream.bind(py).call_method0("close"); - }); - } + let _ = self.stream.close(); + } +} + +#[cfg(test)] +mod tests { + use pyo3::ffi::c_str; + use pyo3::types::PyDict; + + use super::*; + + #[pyfunction] + fn reenter_native(py: Python<'_>) -> usize { + py.detach(|| paimon_datafusion::runtime::runtime().block_on(async { 1 })) + } + + #[test] + fn cancellation_cleanup_can_reenter_native_from_a_runtime_thread() { + Python::attach(|py| { + for clear in [false, true] { + let locals = PyDict::new(py); + locals + .set_item("native", wrap_pyfunction!(reenter_native, py).unwrap()) + .unwrap(); + py.run( + c_str!( + r#" +calls = [] +class Stream: + def close(self): + calls.append(native()) +stream = Stream() +"# + ), + Some(&locals), + Some(&locals), + ) + .unwrap(); + let references = Arc::new(PythonUriReferences::default()); + let stream = PythonUriInputStream { + stream: PythonUriReference::new( + locals.get_item("stream").unwrap().unwrap().unbind(), + true, + Some(&references), + ), + }; + py.detach(|| { + paimon_datafusion::runtime::runtime().block_on(async { + if clear { + references.clear(); + } + drop(stream); + }) + }); + assert_eq!( + locals + .get_item("calls") + .unwrap() + .unwrap() + .extract::>() + .unwrap(), + vec![1], + "close must complete its Native callback, clear={clear}" + ); + } + }); + } + + #[test] + fn registry_retires_only_slots_released_by_core() { + Python::attach(|py| { + for tracked in [false, true] { + let references = Arc::new(PythonUriReferences::default()); + let object = py + .eval(c_str!("type('Callback', (), {})()"), None, None) + .unwrap(); + let weak = py + .import("weakref") + .unwrap() + .getattr("ref") + .unwrap() + .call1((&object,)) + .unwrap(); + let slot = + PythonUriReference::new(object.unbind(), false, tracked.then_some(&references)); + references.prune(); + assert!(!weak.call0().unwrap().is_none()); + drop(slot); + assert_eq!(weak.call0().unwrap().is_none(), !tracked); + references.prune(); + assert!(weak.call0().unwrap().is_none()); + } + }); + } + + #[test] + fn registration_bounds_retired_readers_within_one_write() { + Python::attach(|py| { + let references = Arc::new(PythonUriReferences::default()); + let weakref = py.import("weakref").unwrap().getattr("ref").unwrap(); + let mut readers = Vec::new(); + for _ in 0..100 { + let object = py + .eval(c_str!("type('Reader', (), {})()"), None, None) + .unwrap(); + readers.push(weakref.call1((&object,)).unwrap()); + let slot = PythonUriReference::new(object.unbind(), false, Some(&references)); + drop(slot); + } + assert!( + readers + .iter() + .filter(|reader| !reader.call0().unwrap().is_none()) + .count() + < 32, + "a large write must not retain every retired Python reader" + ); + references.prune(); + assert!(readers + .iter() + .all(|reader| reader.call0().unwrap().is_none())); + }); + } + + #[test] + fn registry_clear_and_stream_close_drop_share_one_cleanup() { + Python::attach(|py| { + for clear_first in [false, true] { + let locals = PyDict::new(py); + py.run( + c_str!( + r#" +calls = [] +class Stream: + def close(self): + calls.append('close') + raise ValueError('cleanup failed') +stream = Stream() +"# + ), + Some(&locals), + Some(&locals), + ) + .unwrap(); + let references = Arc::new(PythonUriReferences::default()); + let slot = PythonUriReference::new( + locals.get_item("stream").unwrap().unwrap().unbind(), + true, + Some(&references), + ); + let stream = PythonUriInputStream { + stream: slot.clone(), + }; + if clear_first { + references.clear(); + } else { + assert!(slot + .close() + .unwrap_err() + .is_instance_of::(py)); + } + references.clear(); + slot.close().unwrap(); + drop(stream); + references.prune(); + assert_eq!( + locals + .get_item("calls") + .unwrap() + .unwrap() + .extract::>() + .unwrap(), + vec!["close"] + ); + assert!(slot.clone_ref(py).is_err()); + assert!(references.references.lock().unwrap().is_empty()); + } + }); } } diff --git a/bindings/python/src/write.rs b/bindings/python/src/write.rs index a22a15451..f89e9e345 100644 --- a/bindings/python/src/write.rs +++ b/bindings/python/src/write.rs @@ -35,7 +35,10 @@ use pyo3::prelude::*; use pyo3::types::{PyBytes, PyDict, PyString}; use pyo3::{PyTraverseError, PyVisit}; -use crate::blob_uri_reader::{callback_error, optional_uri_reader_factory, to_callback_error}; +use crate::blob_uri_reader::{ + callback_error, optional_uri_reader_factory, to_callback_error, PythonUriReaderFactory, + PythonUriReferences, +}; use crate::commit_callback::PyCommitCallbackContext; use crate::error::to_py_err; use crate::predicate::{dict_to_table_predicate, py_to_datum}; @@ -154,12 +157,13 @@ impl WriteContext { None => builder, }; Ok(WriteState { - inner: Some(WriteTarget::Table(Box::new( + inner: Mutex::new(Some(WriteTarget::Table(Box::new( builder.new_write().map_err(to_py_err)?, - ))), + )))), table_location: self.table.location().to_string(), commit_user: self.commit_user.clone(), blob_consumers: Vec::new(), + uri_references: None, }) } } @@ -559,12 +563,13 @@ impl PyPostponeFixedBucketWriteBuilder { } Ok(PyBatchTableWrite { state: WriteState { - inner: Some(WriteTarget::PostponeFixed(Box::new( + inner: Mutex::new(Some(WriteTarget::PostponeFixed(Box::new( builder.new_write().map_err(to_py_err)?, - ))), + )))), table_location: context.table.location().to_string(), commit_user: context.commit_user.clone(), blob_consumers: Vec::new(), + uri_references: None, }, prepared: false, }) @@ -576,12 +581,15 @@ impl PyPostponeFixedBucketWriteBuilder { } struct WriteState { - inner: Option, + // Encoders are Send but not Sync. PyO3's exclusive borrow protects methods; + // get_mut keeps that access lock-free while allowing GC/drop on any thread. + inner: Mutex>, table_location: String, commit_user: String, // Core keeps prior consumers in captured partition/bucket settings. Expose // every owned Python reference to GC, including replaced consumers. blob_consumers: Vec>, + uri_references: Option>, } /// Descriptor serialization and exception transport only; record boundaries, @@ -974,17 +982,37 @@ impl UpdateContext { } impl WriteState { + fn writer_mut(&mut self) -> PyResult<&mut WriteTarget> { + self.inner + .get_mut() + .unwrap() + .as_mut() + .ok_or_else(|| PyRuntimeError::new_err("TableWrite is closed")) + } + fn with_blob_uri_reader_factory( &mut self, factory: Option>, py: Python<'_>, ) -> PyResult<()> { - let factory = optional_uri_reader_factory(factory, py)?; - self.inner - .as_mut() - .ok_or_else(|| PyRuntimeError::new_err("TableWrite is closed"))? - .with_blob_uri_reader_factory(factory) - .map_err(to_py_err) + let factory = factory + .map(|factory| { + let references = self + .uri_references + .get_or_insert_with(Default::default) + .clone(); + PythonUriReaderFactory::tracked(factory, py, references) + .map(|factory| Arc::new(factory) as Arc) + }) + .transpose()?; + let result = self.writer_mut().and_then(|writer| { + writer + .with_blob_uri_reader_factory(factory) + .map_err(to_py_err) + }); + // A rejected replacement must release its newly registered Python refs too. + self.prune_callbacks(); + result } fn with_blob_consumer(&mut self, callback: Option>, py: Python<'_>) -> PyResult<()> { @@ -998,9 +1026,7 @@ impl WriteState { })) }) .transpose()?; - self.inner - .as_mut() - .ok_or_else(|| PyRuntimeError::new_err("TableWrite is closed"))? + self.writer_mut()? .with_blob_consumer( consumer .clone() @@ -1009,18 +1035,21 @@ impl WriteState { .map_err(to_py_err)?; // Release replaced callbacks which no bucket has captured. A snapshot // retained by core contributes another Arc, without another Py ref. - self.prune_blob_consumers(); + self.prune_callbacks(); if let Some(consumer) = consumer { self.blob_consumers.push(consumer); } Ok(()) } - fn prune_blob_consumers(&mut self) { + fn prune_callbacks(&mut self) { let (retained, retired): (Vec<_>, Vec<_>) = std::mem::take(&mut self.blob_consumers) .into_iter() .partition(|consumer| Arc::strong_count(consumer) > 1); self.blob_consumers = retained; + if let Some(references) = &self.uri_references { + references.prune(); + } // Python destructors can re-enter Native. No callback mutex is held, // and the retained references are restored before any destructor runs. drop(retired); @@ -1030,6 +1059,9 @@ impl WriteState { for consumer in &self.blob_consumers { visit.call(&*consumer.callback.lock().unwrap())?; } + if let Some(references) = &self.uri_references { + references.traverse(visit)?; + } Ok(()) } @@ -1039,6 +1071,9 @@ impl WriteState { .iter() .map(|consumer| consumer.callback.lock().unwrap().take()) .collect::>(); + if let Some(references) = &self.uri_references { + references.clear(); + } // Destructors may re-enter Native. Drop Python references outside locks. drop(references); } @@ -1048,9 +1083,7 @@ impl WriteState { } fn with_write_type(&mut self, columns: Vec) -> PyResult<()> { - self.inner - .as_mut() - .ok_or_else(|| PyRuntimeError::new_err("TableWrite is closed"))? + self.writer_mut()? .with_write_type(columns) .map_err(to_py_err) } @@ -1062,11 +1095,9 @@ impl WriteState { bucket: Option, ) -> PyResult<()> { let batch = crate::arrow_input::write_batch_from_pyarrow(batch)?; - let inner = self - .inner - .as_mut() - .ok_or_else(|| PyRuntimeError::new_err("TableWrite is closed"))?; + let inner = self.writer_mut()?; let result = py.detach(|| runtime().block_on(inner.write_arrow(&batch, bucket))); + self.prune_callbacks(); self.write_result(result) } @@ -1075,12 +1106,9 @@ impl WriteState { py: Python<'_>, commit_identifier: i64, ) -> PyResult> { - let inner = self - .inner - .as_mut() - .ok_or_else(|| PyRuntimeError::new_err("TableWrite is closed"))?; + let inner = self.writer_mut()?; let result = py.detach(|| runtime().block_on(inner.prepare_commit(commit_identifier))); - self.prune_blob_consumers(); + self.prune_callbacks(); let messages = result.map_err(to_callback_error)?; Ok(wrap_messages( messages, @@ -1090,11 +1118,7 @@ impl WriteState { } } -#[pyclass( - name = "BatchTableWrite", - module = "pypaimon_rust.datafusion", - unsendable -)] +#[pyclass(name = "BatchTableWrite", module = "pypaimon_rust.datafusion")] pub struct PyBatchTableWrite { state: WriteState, prepared: bool, @@ -1452,7 +1476,7 @@ impl PyBatchTableWrite { } fn close(&mut self, py: Python<'_>) { - if let Some(mut writer) = self.state.inner.take() { + if let Some(mut writer) = self.state.inner.get_mut().unwrap().take() { py.detach(|| runtime().block_on(writer.close())); } self.state.clear_callbacks(); @@ -1479,11 +1503,7 @@ impl PyBatchTableWrite { } } -#[pyclass( - name = "StreamTableWrite", - module = "pypaimon_rust.datafusion", - unsendable -)] +#[pyclass(name = "StreamTableWrite", module = "pypaimon_rust.datafusion")] pub struct PyStreamTableWrite { state: WriteState, } @@ -1527,7 +1547,7 @@ impl PyStreamTableWrite { } fn close(&mut self, py: Python<'_>) { - if let Some(mut writer) = self.state.inner.take() { + if let Some(mut writer) = self.state.inner.get_mut().unwrap().take() { py.detach(|| runtime().block_on(writer.close())); } self.state.clear_callbacks(); @@ -2428,14 +2448,14 @@ writer.with_blob_consumer(Callback(writer, 'first')) let object = locals.get_item("writer").unwrap().unwrap(); if batch { let mut writer = object.extract::>().unwrap(); - let inner = writer.state.inner.as_mut().unwrap(); + let inner = writer.state.writer_mut().unwrap(); py.detach(|| runtime().block_on(inner.write_arrow(&input, None))) .unwrap(); } else { let mut writer = object .extract::>() .unwrap(); - let inner = writer.state.inner.as_mut().unwrap(); + let inner = writer.state.writer_mut().unwrap(); py.detach(|| runtime().block_on(inner.write_arrow(&input, None))) .unwrap(); } @@ -2522,7 +2542,7 @@ first = install('first') ]) .unwrap(); let mut borrowed = writer.borrow_mut(py); - let inner = borrowed.state.inner.as_mut().unwrap(); + let inner = borrowed.state.writer_mut().unwrap(); py.detach(|| runtime().block_on(inner.write_arrow(&input, None))) .unwrap(); }; diff --git a/bindings/python/tests/test_blob_consumer.py b/bindings/python/tests/test_blob_consumer.py index d86a2ee82..aa4cf5617 100644 --- a/bindings/python/tests/test_blob_consumer.py +++ b/bindings/python/tests/test_blob_consumer.py @@ -15,7 +15,11 @@ # specific language governing permissions and limitations # under the License. +import gc import struct +import sys +import threading +import weakref from urllib.parse import urlparse import pyarrow as pa @@ -131,3 +135,100 @@ def test_blob_consumer_replacement_and_clear(tmp_path): writer.write_arrow(_batch()) finally: writer.close() + + +@pytest.mark.parametrize('stream', [False, True]) +@pytest.mark.parametrize('captured', [False, True]) +def test_writer_callback_cycles_are_collected_on_another_thread( + tmp_path, monkeypatch, stream, captured): + _, table = _table(tmp_path) + finalized = [] + unraisable = [] + monkeypatch.setattr(sys, 'unraisablehook', + lambda event: unraisable.append(str(event.exc_value))) + + class Callback: + def __init__(self, writer, label): + self.writer = writer + self.label = label + + def __call__(self, field, descriptor): + return False + + def __del__(self): + finalized.append((self.label, threading.get_ident())) + + def cycle(): + builder = (table.new_stream_write_builder() if stream + else table.new_batch_write_builder()) + writer = builder.new_write() + callback = Callback(writer, 'first') + writer.with_blob_consumer(callback) + references = [weakref.ref(callback)] + if captured: + writer.write_arrow(_batch()) + replacement = Callback(writer, 'second') + writer.with_blob_consumer(replacement) + references.append(weakref.ref(replacement)) + return references + + gc.collect() + enabled = gc.isenabled() + gc.disable() + try: + references = cycle() + collector = threading.Thread(target=gc.collect) + collector.start() + collector.join() + assert unraisable == [] + assert all(reference() is None for reference in references) + expected = ['first', 'second'] if captured else ['first'] + assert sorted(finalized) == [(label, collector.ident) for label in expected] + finally: + # Also clean up the original-thread-only implementation on failure. + gc.collect() + if enabled: + gc.enable() + + +@pytest.mark.parametrize('stream', [False, True]) +def test_writer_thread_handoff_keeps_exclusive_access(tmp_path, stream): + context, table = _table(tmp_path) + builder = (table.new_stream_write_builder() if stream + else table.new_batch_write_builder()) + writer = builder.new_write() + entered = threading.Event() + release = threading.Event() + failures = [] + + def callback(field, descriptor): + entered.set() + assert release.wait(10) + return False + + def write(): + try: + writer.write_arrow(_batch()) + except BaseException as error: + failures.append(str(error)) + + writer.with_blob_consumer(callback) + worker = threading.Thread(target=write) + try: + worker.start() + assert entered.wait(10), failures + with pytest.raises(RuntimeError, match='Already borrowed'): + writer.close() + finally: + release.set() + worker.join() + try: + assert failures == [] + messages = writer.prepare_commit(True, 1) if stream else writer.prepare_commit() + commit = builder.new_commit() + commit.commit(1, messages) if stream else commit.commit(messages) + assert pa.Table.from_batches(context.sql( + 'SELECT id, payload FROM paimon.consumer.t ORDER BY id')).to_pydict() == { + 'id': [1, 2, 3], 'payload': [b'payload', None, b'']} + finally: + writer.close() diff --git a/bindings/python/tests/test_blob_uri_reader.py b/bindings/python/tests/test_blob_uri_reader.py index aa457e225..9509bab9c 100644 --- a/bindings/python/tests/test_blob_uri_reader.py +++ b/bindings/python/tests/test_blob_uri_reader.py @@ -15,8 +15,12 @@ # specific language governing permissions and limitations # under the License. +import gc import io import struct +import sys +import threading +import weakref import pyarrow as pa import pytest @@ -108,6 +112,134 @@ def test_custom_uri_read_error_preserves_original_exception_and_closes_once(tmp_ assert not list(tmp_path.rglob('*.blob')) +@pytest.mark.parametrize('stream', [False, True]) +@pytest.mark.parametrize('source', ['unused', 'shared_reader', 'separate_reader']) +def test_uri_callback_cycles_release_every_reference_and_close_once( + tmp_path, monkeypatch, stream, source): + context, table = _table(tmp_path) + references = [] + closes = [] + unraisable = [] + monkeypatch.setattr(sys, 'unraisablehook', + lambda event: unraisable.append(str(event.exc_value))) + + class Stream: + def __init__(self): + self.buffer = io.BytesIO(b'0123456789') + + def read(self, length): + return self.buffer.read(length) + + def seek(self, offset): + return self.buffer.seek(offset) + + def close(self): + closes.append(threading.get_ident()) + # Cleanup callbacks may call Native without nesting its runtime. + context.sql('SELECT 1') + self.buffer.close() + + class Reader: + def __init__(self, writer): + self.writer = writer + references.append(weakref.ref(self)) + + def new_input_stream(self, uri): + value = Stream() + value.writer = self.writer + references.append(weakref.ref(value)) + return value + + class Factory(Reader): + def create(self, uri): + return Reader(self.writer) if source == 'separate_reader' else self + + def cycle(): + builder = (table.new_stream_write_builder() if stream + else table.new_batch_write_builder()) + writer = builder.new_write() + factory = Factory(writer) + writer.with_blob_uri_reader_factory(factory) + if source != 'unused': + writer.write_arrow(_batch([1], [descriptor(0, 3)])) + + gc.collect() + enabled = gc.isenabled() + gc.disable() + try: + cycle() + collector = threading.Thread(target=gc.collect) + collector.start() + collector.join() + assert unraisable == [] + assert all(reference() is None for reference in references) + assert closes == ([] if source == 'unused' else [collector.ident]) + finally: + for reference in references: + value = reference() + if value is not None: + value.writer.close() + gc.collect() + if enabled: + gc.enable() + + +@pytest.mark.parametrize('stream', [False, True]) +def test_uri_references_retire_after_prepare_and_rejected_replacement(tmp_path, stream): + _, table = _table(tmp_path) + builder = (table.new_stream_write_builder() if stream + else table.new_batch_write_builder()) + writer = builder.new_write() + readers = [] + streams = [] + + class Reader(ReaderFactory): + def new_input_stream(self, uri): + value = super().new_input_stream(uri) + streams.append(weakref.ref(value)) + return value + + class Factory: + def create(self, uri): + reader = Reader() + readers.append(weakref.ref(reader)) + return reader + + factory = Factory() + writer.with_blob_uri_reader_factory(factory) + try: + replacement = Factory() + retired = weakref.ref(factory) + del factory + writer.with_blob_uri_reader_factory(replacement) + assert retired() is None + retired = weakref.ref(replacement) + del replacement + writer.with_blob_uri_reader_factory(None) + assert retired() is None + writer.with_blob_uri_reader_factory(Factory()) + writer.write_arrow(_batch([1], [descriptor(0, 3)])) + assert readers[0]() is not None + assert streams[0]() is not None + rejected = Factory() + retired = weakref.ref(rejected) + with pytest.raises(ValueError, match='before any write'): + writer.with_blob_uri_reader_factory(rejected) + del rejected + assert retired() is None + writer.prepare_commit(True, 1) if stream else writer.prepare_commit() + gc.collect() + assert all(reference() is None for reference in readers + streams) + finally: + writer.close() + rejected = Factory() + retired = weakref.ref(rejected) + with pytest.raises(RuntimeError, match='closed'): + writer.with_blob_uri_reader_factory(rejected) + del rejected + assert retired() is None + + def _update_table(tmp_path): context = SQLContext() context.register_catalog('paimon', {'warehouse': str(tmp_path)}) From 24e5c0c6e54be0435204c73c82290cd526801a9b Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Tue, 3 Nov 2026 15:55:02 +0800 Subject: [PATCH 6/6] [arrow] Keep MAP and MULTISET keys non-nullable Use the existing Arrow Map representation for both logical types and let Arrow runtime validation reject actual null keys. Remove multiset List metadata, conversions, format branches, and nullable-element fixtures while preserving Java file encodings, ordinary map value nullability, and prior upgrade and Python GC fixes. --- crates/paimon/src/arrow/format/avro.rs | 80 +--- crates/paimon/src/arrow/format/avro_write.rs | 91 +--- crates/paimon/src/arrow/format/mosaic.rs | 3 - crates/paimon/src/arrow/format/parquet.rs | 24 +- crates/paimon/src/arrow/format/row.rs | 160 ++----- crates/paimon/src/arrow/format/vortex.rs | 3 +- crates/paimon/src/arrow/mod.rs | 114 +++-- crates/paimon/src/arrow/multiset.rs | 446 ------------------ crates/paimon/src/arrow/nested_evolution.rs | 66 +-- .../src/table/postpone_bucket_planner.rs | 56 +-- .../paimon/src/table/write_batch_normalize.rs | 70 --- .../tests/incremental_diff_extended_test.rs | 33 +- docs/src/getting-started.md | 20 +- 13 files changed, 189 insertions(+), 977 deletions(-) delete mode 100644 crates/paimon/src/arrow/multiset.rs diff --git a/crates/paimon/src/arrow/format/avro.rs b/crates/paimon/src/arrow/format/avro.rs index 6ada1f59d..c1f1ce37c 100644 --- a/crates/paimon/src/arrow/format/avro.rs +++ b/crates/paimon/src/arrow/format/avro.rs @@ -410,13 +410,15 @@ fn build_column( DataType::Map(map_type) => build_map_column(records, name, map_type, num_rows)?, // Java encodes MULTISET as a map from the element to an INT count, // sharing the MAP path (`AvroSchemaConverter#extractValueTypeToAvroMap` - // returns IntType). A list preserves NULL elements; counts are non-nullable. + // returns IntType). MAP keys and MULTISET elements are non-nullable + // in Arrow; counts are non-nullable too. DataType::Multiset(multiset_type) => build_map_like_column( records, name, multiset_type.element_type(), &DataType::Int(IntType::new()), - true, + false, + false, num_rows, )?, DataType::Row(row_type) => build_row_column(records, name, row_type, num_rows)?, @@ -587,32 +589,36 @@ fn build_map_column( name, map_type.key_type(), map_type.value_type(), + // `paimon_type_to_arrow` declares a MAP key non-nullable and lets the + // value follow its own type. Mirror that exactly or `RecordBatch::try_new` + // rejects the column. false, + map_type.value_type().is_nullable(), num_rows, ) } -/// Build MAP or MULTISET values from either Java Avro map encoding. +/// Build a Map array from either Avro map encoding Paimon can produce. /// /// Avro maps natively support string keys only, so Java writes a map with any /// other key type as an array of `{key, value}` records instead /// (`AvroSchemaConverter#isArrayMap`). Both shapes decode into the same Arrow -/// key/value entries here. +/// `Map` array here. /// /// The key and value types are passed separately rather than taken from a /// `MapType` so `MULTISET` can reuse this with an implicit `INT` count as the -/// value. MULTISET uses a list because its elements can be NULL; ordinary MAP -/// keys remain non-nullable. +/// value. The two nullability flags must match what `paimon_type_to_arrow` +/// declares for the same type or +/// `RecordBatch::try_new` rejects the column. fn build_map_like_column( records: &[Value], name: &str, key_type: &DataType, value_type: &DataType, - multiset: bool, + key_nullable: bool, + value_nullable: bool, num_rows: usize, ) -> crate::Result> { - let key_nullable = multiset && key_type.is_nullable(); - let value_nullable = !multiset && value_type.is_nullable(); let arrow_key_type = crate::arrow::paimon_type_to_arrow(key_type)?; let arrow_value_type = crate::arrow::paimon_type_to_arrow(value_type)?; @@ -696,18 +702,6 @@ fn build_map_like_column( .collect::>(), )); - if multiset { - let field = crate::arrow::multiset::entries_field( - crate::arrow::paimon_type_to_arrow(key_type)?, - key_nullable, - ); - let list = ListArray::try_new(field, offsets_buf, Arc::new(struct_arr), Some(nulls)) - .map_err(|e| Error::UnexpectedError { - message: format!("Failed to build MULTISET: {e}"), - source: Some(Box::new(e)), - })?; - return Ok(Arc::new(list)); - } let map_arr = MapArray::try_new( Arc::new(entries_field), offsets_buf, @@ -1125,7 +1119,6 @@ mod tests { ); let col = build_map_column(&records, "m", &map_type, 2).unwrap(); let arr = col.as_any().downcast_ref::().unwrap(); - assert!(!arr.entries().fields()[0].is_nullable()); assert_eq!(arr.len(), 2); // First row has 2 entries, second row is null. assert!(!arr.is_null(0)); @@ -1250,15 +1243,9 @@ mod tests { VarCharType::new(10).unwrap(), ))); let col = build_column(&records, "ms", &multiset, 1).unwrap(); - let arr = col.as_any().downcast_ref::().unwrap(); - let entries = arr.values().as_any().downcast_ref::().unwrap(); - assert!(entries.fields()[0].is_nullable()); + let arr = col.as_any().downcast_ref::().unwrap(); assert_eq!(arr.value(0).len(), 1); - let values = entries - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); + let values = arr.values().as_any().downcast_ref::().unwrap(); assert_eq!(values.value(0), 2); } @@ -1268,36 +1255,19 @@ mod tests { fn test_build_column_multiset_int_element() { let records = vec![Value::Record(vec![( "ms".to_string(), - Value::Array(vec![ - Value::Record(vec![ - ("key".to_string(), Value::Int(5)), - ("value".to_string(), Value::Int(3)), - ]), - Value::Record(vec![ - ("key".to_string(), av_null()), - ("value".to_string(), Value::Int(2)), - ]), - ]), + Value::Array(vec![Value::Record(vec![ + ("key".to_string(), Value::Int(5)), + ("value".to_string(), Value::Int(3)), + ])]), )])]; let multiset = DataType::Multiset(MultisetType::new(DataType::Int(IntType::new()))); let col = build_column(&records, "ms", &multiset, 1).unwrap(); - let arr = col.as_any().downcast_ref::().unwrap(); - let entries = arr.values().as_any().downcast_ref::().unwrap(); - assert_eq!(arr.value(0).len(), 2); - let keys = entries - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); + let arr = col.as_any().downcast_ref::().unwrap(); + assert_eq!(arr.value(0).len(), 1); + let keys = arr.keys().as_any().downcast_ref::().unwrap(); assert_eq!(keys.value(0), 5); - assert!(keys.is_null(1)); - let values = entries - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); + let values = arr.values().as_any().downcast_ref::().unwrap(); assert_eq!(values.value(0), 3); - assert_eq!(values.value(1), 2); } /// Going through `build_target_arrow_schema` is what catches a field-level diff --git a/crates/paimon/src/arrow/format/avro_write.rs b/crates/paimon/src/arrow/format/avro_write.rs index ba46810f5..d52ef6006 100644 --- a/crates/paimon/src/arrow/format/avro_write.rs +++ b/crates/paimon/src/arrow/format/avro_write.rs @@ -26,7 +26,7 @@ use apache_avro::{Codec, Schema, Writer}; use arrow_array::{ Array, BinaryArray, BooleanArray, Date32Array, Decimal128Array, FixedSizeListArray, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, LargeBinaryArray, - ListArray, RecordBatch, StringArray, StructArray, Time32MillisecondArray, + ListArray, MapArray, RecordBatch, StringArray, StructArray, Time32MillisecondArray, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampSecondArray, }; use arrow_schema::SchemaRef; @@ -447,23 +447,16 @@ fn nonnull_value_at(array: &dyn Array, row: usize, typ: &DataType) -> Result Result { - let structure = crate::arrow::multiset::entries(array, row)?; - let start = 0; - let end = structure.len(); + let map = downcast::(array)?; + let start = map.value_offsets()[row] as usize; + let end = map.value_offsets()[row + 1] as usize; if matches!(key, DataType::Char(_) | DataType::VarChar(_)) { let mut entries = HashMap::with_capacity(end - start); for index in start..end { - if structure.column(0).is_null(index) { - return Err(Error::DataInvalid { - message: "Avro string maps cannot contain NULL keys or multiset elements" - .into(), - source: None, - }); - } - let map_key = downcast::(structure.column(0))?.value(index); + let map_key = downcast::(map.keys())?.value(index); entries.insert( map_key.to_string(), - value_at(structure.column(1).as_ref(), index, value)?, + value_at(map.values().as_ref(), index, value)?, ); } Ok(Value::Map(entries)) @@ -471,13 +464,10 @@ fn map_value(array: &dyn Array, row: usize, key: &DataType, value: &DataType) -> let mut entries = Vec::with_capacity(end - start); for index in start..end { entries.push(Value::Record(vec![ - ( - "key".into(), - value_at(structure.column(0).as_ref(), index, key)?, - ), + ("key".into(), value_at(map.keys(), index, key)?), ( "value".into(), - value_at(structure.column(1).as_ref(), index, value)?, + value_at(map.values().as_ref(), index, value)?, ), ])); } @@ -540,71 +530,6 @@ mod tests { VectorType, }; - #[tokio::test] - async fn multiset_preserves_null_elements_and_counts_in_avro() { - let fields = vec![DataField::new( - 0, - "bag".into(), - DataType::Multiset(crate::spec::MultisetType::new( - DataType::Int(IntType::new()), - )), - )]; - let schema = build_target_arrow_schema(&fields).unwrap(); - let arrow_schema::DataType::List(child) = schema.field(0).data_type() else { - unreachable!() - }; - let arrow_schema::DataType::Struct(entry_fields) = child.data_type() else { - unreachable!() - }; - let entries = StructArray::try_new( - entry_fields.clone(), - vec![ - Arc::new(Int32Array::from(vec![None, Some(5)])), - Arc::new(Int32Array::from(vec![3, 2])), - ], - None, - ) - .unwrap(); - let bag = ListArray::try_new( - child.clone(), - arrow_buffer::OffsetBuffer::new(vec![0, 2, 2, 2].into()), - Arc::new(entries), - Some(NullBuffer::from(vec![true, false, true])), - ) - .unwrap(); - let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(bag)]).unwrap(); - let io = FileIOBuilder::new("memory").build().unwrap(); - let path = "memory:/multiset-null.avro"; - let mut writer = AvroFormatWriter::new( - &io.new_output(path).unwrap(), - schema, - fields.clone(), - "none", - 1, - None, - ) - .await - .unwrap(); - writer.write(&batch).await.unwrap(); - Box::new(writer).close().await.unwrap(); - let bytes = io.new_input(path).unwrap().read().await.unwrap(); - let decoded = AvroFormatReader - .read_batch_stream( - Box::new(BytesFileRead(bytes.clone())), - bytes.len() as u64, - &fields, - None, - None, - None, - ) - .await - .unwrap() - .try_collect::>() - .await - .unwrap(); - assert_eq!(decoded, vec![batch]); - } - fn fields() -> Vec { vec![ DataField::new(0, "id".into(), DataType::BigInt(BigIntType::new())), diff --git a/crates/paimon/src/arrow/format/mosaic.rs b/crates/paimon/src/arrow/format/mosaic.rs index c646447a3..f84ee8730 100644 --- a/crates/paimon/src/arrow/format/mosaic.rs +++ b/crates/paimon/src/arrow/format/mosaic.rs @@ -818,9 +818,6 @@ pub(super) fn validate_mosaic_schema(schema: &SchemaRef) -> crate::Result<()> { } fn validate_mosaic_arrow_type(data_type: &ArrowDataType) -> Result<(), String> { - if crate::arrow::multiset::contains_multiset(data_type) { - return Err("MULTISET is not supported by Mosaic".into()); - } paimon_mosaic_core::types::validate_data_type(data_type) } diff --git a/crates/paimon/src/arrow/format/parquet.rs b/crates/paimon/src/arrow/format/parquet.rs index 6dbadf9be..a2de1a289 100644 --- a/crates/paimon/src/arrow/format/parquet.rs +++ b/crates/paimon/src/arrow/format/parquet.rs @@ -35,6 +35,7 @@ use crate::spec::{ use crate::table::{ArrowRecordBatchStream, RowRange}; use crate::Error; use arrow_array::{BooleanArray, RecordBatch}; +use arrow_cast::cast; use async_trait::async_trait; use bytes::Bytes; use futures::future::BoxFuture; @@ -373,8 +374,7 @@ impl ParquetFormatWriter { let max_row_group_bytes = props.max_row_group_bytes(); let async_write = output.async_writer().await?; let input_schema = schema; - let schema = - crate::arrow::multiset::parquet_schema(×tamp_millis_schema(&input_schema)); + let schema = timestamp_millis_schema(&input_schema); let inner = create_parquet_arrow_writer(async_write, schema.clone(), props)?; let stats_modes = write_fields .map(|fields| core_options.metadata_stats_modes(fields.iter().map(DataField::name))) @@ -792,17 +792,15 @@ fn cast_record_batch_to_schema( if column.data_type() == field.data_type() { Ok(column.clone()) } else { - crate::arrow::multiset::to_parquet_array(column, field.data_type()).map_err( - |error| crate::Error::DataInvalid { - message: format!( - "Failed to convert Parquet column '{}' from {:?} to {:?}: {error}", - field.name(), - column.data_type(), - field.data_type() - ), - source: Some(Box::new(error)), - }, - ) + cast(column, field.data_type()).map_err(|error| crate::Error::DataInvalid { + message: format!( + "Failed to convert Parquet column '{}' from {:?} to {:?}: {error}", + field.name(), + column.data_type(), + field.data_type() + ), + source: Some(Box::new(error)), + }) } }) .collect::>>()?; diff --git a/crates/paimon/src/arrow/format/row.rs b/crates/paimon/src/arrow/format/row.rs index 1d00ef08c..1bbc0c890 100644 --- a/crates/paimon/src/arrow/format/row.rs +++ b/crates/paimon/src/arrow/format/row.rs @@ -550,8 +550,7 @@ fn validate_arrow_type_for_row_field( )?; true } - (ArrowDataType::List(entries), DataType::Multiset(m)) - | (ArrowDataType::Map(entries, _), DataType::Multiset(m)) => { + (ArrowDataType::Map(entries, _), DataType::Multiset(m)) => { validate_arrow_map_entries( field_name, entries.data_type(), @@ -829,15 +828,9 @@ fn write_field_value( write_map_value(out, map, row_idx, m.key_type(), m.value_type())?; } DataType::Multiset(m) => { - let entries = crate::arrow::multiset::entries(array, row_idx)?; - write_array_slice(out, entries.column(0), 0, entries.len(), m.element_type())?; - write_array_slice( - out, - entries.column(1), - 0, - entries.len(), - &DataType::Int(IntType::new()), - )?; + let map = downcast::(array, data_type)?; + let count_type = DataType::Int(IntType::new()); + write_map_value(out, map, row_idx, m.element_type(), &count_type)?; } DataType::Row(r) => { let row = downcast::(array, data_type)?; @@ -1329,7 +1322,6 @@ enum ColumnBuilder { values: Box, }, Map { - multiset: bool, entries_field: Arc, offsets: Vec, validities: Vec, @@ -1399,7 +1391,6 @@ impl ColumnBuilder { values: Box::new(ColumnBuilder::new(a.element_type(), capacity)?), }, DataType::Map(m) => Self::Map { - multiset: false, entries_field: map_entries_field( m.key_type(), m.value_type(), @@ -1412,21 +1403,17 @@ impl ColumnBuilder { values: Box::new(ColumnBuilder::new(m.value_type(), capacity)?), sorted: false, }, - DataType::Multiset(m) => Self::Map { - multiset: true, - entries_field: crate::arrow::multiset::entries_field( - paimon_type_to_arrow(m.element_type())?, - m.element_type().is_nullable(), - ), - offsets: vec![0], - validities: Vec::with_capacity(capacity), - keys: Box::new(ColumnBuilder::new(m.element_type(), capacity)?), - values: Box::new(ColumnBuilder::new( - &DataType::Int(IntType::new()), - capacity, - )?), - sorted: false, - }, + DataType::Multiset(m) => { + let count_type = DataType::Int(IntType::new()); + Self::Map { + entries_field: map_entries_field(m.element_type(), &count_type, false, false)?, + offsets: vec![0], + validities: Vec::with_capacity(capacity), + keys: Box::new(ColumnBuilder::new(m.element_type(), capacity)?), + values: Box::new(ColumnBuilder::new(&count_type, capacity)?), + sorted: false, + } + } DataType::Row(r) => Self::Row { fields: arrow_fields_for_row(r.fields())?, columns: r @@ -1728,7 +1715,6 @@ impl ColumnBuilder { })?, ), Self::Map { - multiset, entries_field, offsets, validities, @@ -1756,34 +1742,19 @@ impl ColumnBuilder { message: format!("Failed to build .row Map entries: {e}"), source: Some(Box::new(e)), })?; - if multiset { - Arc::new( - ListArray::try_new( - entries_field, - OffsetBuffer::new(ScalarBuffer::from(offsets)), - Arc::new(entries), - Some(null_buffer(validities)), - ) - .map_err(|e| Error::UnexpectedError { - message: format!("Failed to build .row MULTISET: {e}"), - source: Some(Box::new(e)), - })?, - ) - } else { - Arc::new( - MapArray::try_new( - entries_field, - OffsetBuffer::new(ScalarBuffer::from(offsets)), - entries, - Some(null_buffer(validities)), - sorted, - ) - .map_err(|e| Error::UnexpectedError { - message: format!("Failed to build .row MapArray: {e}"), - source: Some(Box::new(e)), - })?, + Arc::new( + MapArray::try_new( + entries_field, + OffsetBuffer::new(ScalarBuffer::from(offsets)), + entries, + Some(null_buffer(validities)), + sorted, ) - } + .map_err(|e| Error::UnexpectedError { + message: format!("Failed to build .row MapArray: {e}"), + source: Some(Box::new(e)), + })?, + ) } Self::Row { fields, @@ -3481,25 +3452,6 @@ mod tests { vec![Some("x"), Some("y"), Some("z")], vec![Some(2), Some(1), Some(4)], ); - let ArrowDataType::List(bag_field) = schema.field(2).data_type() else { - unreachable!() - }; - let bag_entries = StructArray::try_new( - match bag_field.data_type() { - ArrowDataType::Struct(fields) => fields.clone(), - _ => unreachable!(), - }, - bag.entries().columns().to_vec(), - None, - ) - .unwrap(); - let bag = ListArray::try_new( - bag_field.clone(), - bag.offsets().clone(), - Arc::new(bag_entries), - bag.nulls().cloned(), - ) - .unwrap(); let nested = StructArray::try_new( vec![ @@ -3585,24 +3537,11 @@ mod tests { assert!(label_values.is_null(1)); assert_eq!(label_values.value(2), 30); - let bag = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); + let bag = batch.column(2).as_any().downcast_ref::().unwrap(); assert_eq!(bag.value_offsets(), &[0, 2, 2, 3]); assert!(bag.is_null(1)); - let entries = bag.values().as_any().downcast_ref::().unwrap(); - let bag_keys = entries - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let bag_counts = entries - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); + let bag_keys = bag.keys().as_any().downcast_ref::().unwrap(); + let bag_counts = bag.values().as_any().downcast_ref::().unwrap(); assert_eq!(bag_keys.value(0), "x"); assert_eq!(bag_keys.value(2), "z"); assert_eq!(bag_counts.value(0), 2); @@ -3632,45 +3571,6 @@ mod tests { assert_eq!(names.value(2), "nine"); } - #[test] - fn decode_java_multiset_with_null_element() { - let fields = vec![DataField::new( - 0, - "bag".into(), - DataType::Multiset(MultisetType::new(DataType::Int(IntType::new()))), - )]; - // Java RowBlockWriter: one non-null bag, key array [NULL], counts [3], - // row offset 0, and row count 1. No Arrow writer participates here. - let block = [0, 1, 1, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0]; - let batch = decode_row_block( - &block, - &fields, - build_target_arrow_schema(&fields).unwrap(), - &[0], - ) - .unwrap(); - let mut writer = RowBlockWriter::new(); - writer.write_row(&batch, 0, &fields).unwrap(); - assert_eq!(writer.finish(), block); - let bag = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let entries = bag.value(0); - let entries = entries.as_any().downcast_ref::().unwrap(); - assert!(entries.column(0).is_null(0)); - assert_eq!( - entries - .column(1) - .as_any() - .downcast_ref::() - .unwrap() - .value(0), - 3 - ); - } - #[test] fn blob_data_block_matches_java_materialized_descriptor() { use crate::spec::BlobType; diff --git a/crates/paimon/src/arrow/format/vortex.rs b/crates/paimon/src/arrow/format/vortex.rs index 7246c1cf7..b4c4a3004 100644 --- a/crates/paimon/src/arrow/format/vortex.rs +++ b/crates/paimon/src/arrow/format/vortex.rs @@ -541,7 +541,6 @@ fn validate_vortex_schema(schema: &SchemaRef) -> crate::Result<()> { fn contains_unconvertible_type(data_type: &ArrowDataType) -> bool { match data_type { ArrowDataType::Map(..) => true, - ArrowDataType::List(field) if crate::arrow::multiset::is_multiset(field) => true, ArrowDataType::List(field) | ArrowDataType::LargeList(field) | ArrowDataType::ListView(field) @@ -1783,7 +1782,7 @@ mod tests { panic!("expected Unsupported, got {err:?}"); }; assert!(message.contains("'nested'"), "message: {message}"); - assert!(message.contains("List"), "message: {message}"); + assert!(message.contains("Map"), "message: {message}"); } /// Positive control: a nested ROW of primitives must still be accepted, so the walk diff --git a/crates/paimon/src/arrow/mod.rs b/crates/paimon/src/arrow/mod.rs index 403301482..e861c591c 100644 --- a/crates/paimon/src/arrow/mod.rs +++ b/crates/paimon/src/arrow/mod.rs @@ -16,7 +16,6 @@ // under the License. mod date_extract; -pub(crate) mod multiset; pub(crate) use date_extract::evaluate_date_extract; mod cast_width; pub(crate) use cast_width::apply_declared_width; @@ -119,10 +118,23 @@ pub fn paimon_type_to_arrow(dt: &PaimonDataType) -> crate::Result false, ) } - PaimonDataType::Multiset(m) => ArrowDataType::List(multiset::entries_field( - paimon_type_to_arrow(m.element_type())?, - m.element_type().is_nullable(), - )), + PaimonDataType::Multiset(m) => { + let element_type = paimon_type_to_arrow(m.element_type())?; + ArrowDataType::Map( + Arc::new(ArrowField::new( + "entries", + ArrowDataType::Struct( + vec![ + ArrowField::new("key", element_type, false), + ArrowField::new("value", ArrowDataType::Int32, false), + ] + .into(), + ), + false, + )), + false, + ) + } PaimonDataType::Row(r) => { let fields: Vec = r .fields() @@ -228,22 +240,6 @@ pub fn arrow_to_paimon_type( ArrowDataType::Decimal128(p, s) => Ok(PaimonDataType::Decimal(DecimalType::with_nullable( nullable, *p as u32, *s as u32, )?)), - ArrowDataType::List(field) if multiset::is_multiset(field) => { - let ArrowDataType::Struct(entries) = field.data_type() else { - return Err(crate::Error::Unsupported { - message: "Invalid MULTISET entries".into(), - }); - }; - if entries.len() != 2 || entries[1].data_type() != &ArrowDataType::Int32 { - return Err(crate::Error::Unsupported { - message: "Invalid MULTISET element/count fields".into(), - }); - } - let element = arrow_to_paimon_type(entries[0].data_type(), entries[0].is_nullable())?; - Ok(PaimonDataType::Multiset( - crate::spec::MultisetType::with_nullable(nullable, element), - )) - } ArrowDataType::List(field) | ArrowDataType::LargeList(field) => { let element = arrow_to_paimon_type(field.data_type(), field.is_nullable())?; Ok(PaimonDataType::Array(ArrayType::with_nullable( @@ -431,10 +427,24 @@ pub(crate) fn paimon_type_to_arrow_for_read(dt: &PaimonDataType) -> crate::Resul )), false, ), - PaimonDataType::Multiset(multiset) => ArrowDataType::List(self::multiset::entries_field( - paimon_type_to_arrow_for_read(multiset.element_type())?, - true, - )), + PaimonDataType::Multiset(multiset) => ArrowDataType::Map( + Arc::new(ArrowField::new( + "entries", + ArrowDataType::Struct( + vec![ + ArrowField::new( + "key", + paimon_type_to_arrow_for_read(multiset.element_type())?, + false, + ), + ArrowField::new("value", ArrowDataType::Int32, false), + ] + .into(), + ), + false, + )), + false, + ), _ => paimon_type_to_arrow(dt)?, }) } @@ -547,30 +557,38 @@ mod tests { } #[test] - fn multiset_schemas_preserve_nullable_elements() { - let fields = vec![DataField::new( - 0, - "bag".into(), - PaimonDataType::Multiset(crate::spec::MultisetType::new(PaimonDataType::VarChar( - VarCharType::string_type(), - ))), - )]; - for schema in [ - build_target_arrow_schema(&fields).unwrap(), - build_read_arrow_schema(&fields).unwrap(), + fn map_and_multiset_keys_are_non_nullable_in_read_and_write_schemas() { + for (data_type, value_nullable) in [ + ( + PaimonDataType::Map(MapType::new( + PaimonDataType::Int(IntType::new()), + PaimonDataType::Int(IntType::new()), + )), + true, + ), + ( + PaimonDataType::Multiset(crate::spec::MultisetType::new(PaimonDataType::Int( + IntType::new(), + ))), + false, + ), ] { - let ArrowDataType::List(entries) = schema.field(0).data_type() else { - panic!("multiset must use Arrow list storage"); - }; - let ArrowDataType::Struct(fields) = entries.data_type() else { - panic!("map entries must be a struct"); - }; - assert!(fields[0].is_nullable()); - assert!(!fields[1].is_nullable()); - assert!(matches!( - arrow_to_paimon_type(schema.field(0).data_type(), true).unwrap(), - PaimonDataType::Multiset(_) - )); + let fields = vec![DataField::new(0, "values".into(), data_type)]; + for schema in [ + build_target_arrow_schema(&fields).unwrap(), + build_read_arrow_schema(&fields).unwrap(), + ] { + assert!(schema.field(0).is_nullable()); + let ArrowDataType::Map(entries, _) = schema.field(0).data_type() else { + panic!("MAP and MULTISET must both use Arrow Map storage"); + }; + assert!(!entries.is_nullable()); + let ArrowDataType::Struct(fields) = entries.data_type() else { + panic!("map entries must be a struct"); + }; + assert!(!fields[0].is_nullable()); + assert_eq!(fields[1].is_nullable(), value_nullable); + } } } diff --git a/crates/paimon/src/arrow/multiset.rs b/crates/paimon/src/arrow/multiset.rs deleted file mode 100644 index c9f810955..000000000 --- a/crates/paimon/src/arrow/multiset.rs +++ /dev/null @@ -1,446 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! A MULTISET permits NULL elements, whereas Arrow Map keys cannot be NULL. -//! Use a list of element/count entries and mark it to distinguish ARRAY. - -use std::sync::Arc; - -use arrow_array::{ - Array, ArrayRef, FixedSizeListArray, LargeListArray, ListArray, MapArray, StructArray, -}; -use arrow_schema::{DataType, Field, FieldRef, Schema, SchemaRef}; - -use crate::{Error, Result}; - -const MULTISET_KEY: &str = "paimon.multiset"; - -pub(crate) fn entries_field(element: DataType, nullable: bool) -> FieldRef { - Arc::new( - Field::new( - "entries", - DataType::Struct( - vec![ - Field::new("key", element, nullable), - Field::new("value", DataType::Int32, false), - ] - .into(), - ), - false, - ) - .with_metadata([(MULTISET_KEY.into(), "true".into())].into()), - ) -} - -pub(crate) fn is_multiset(field: &Field) -> bool { - field - .metadata() - .get(MULTISET_KEY) - .is_some_and(|value| value == "true") -} - -pub(crate) fn contains_multiset(data_type: &DataType) -> bool { - match data_type { - DataType::List(field) if is_multiset(field) => true, - DataType::List(field) - | DataType::LargeList(field) - | DataType::ListView(field) - | DataType::LargeListView(field) - | DataType::FixedSizeList(field, _) - | DataType::Map(field, _) - | DataType::RunEndEncoded(_, field) => contains_multiset(field.data_type()), - DataType::Struct(fields) => fields - .iter() - .any(|field| contains_multiset(field.data_type())), - DataType::Union(fields, _) => fields - .iter() - .any(|(_, field)| contains_multiset(field.data_type())), - DataType::Dictionary(_, values) => contains_multiset(values), - _ => false, - } -} - -pub(crate) fn entries(array: &dyn Array, row: usize) -> Result { - let entries = if let Some(map) = array.as_any().downcast_ref::() { - Some(map.value(row)) - } else if let Some(list) = array.as_any().downcast_ref::() { - list.value(row) - .as_any() - .downcast_ref::() - .cloned() - } else { - None - }; - entries - .filter(|entries| entries.num_columns() == 2 && entries.null_count() == 0) - .ok_or_else(|| Error::DataInvalid { - message: "MULTISET must contain non-null element/count structs with exactly two fields" - .into(), - source: None, - }) -} - -/// Java Parquet represents MULTISET as MAP and rejects NULL elements. Preserve -/// that disk layout while keeping nullable elements representable elsewhere. -pub(crate) fn parquet_schema(schema: &SchemaRef) -> SchemaRef { - Arc::new(Schema::new_with_metadata( - schema - .fields() - .iter() - .map(parquet_field) - .collect::>(), - schema.metadata().clone(), - )) -} - -fn parquet_field(field: &FieldRef) -> FieldRef { - let data_type = match field.data_type() { - DataType::List(child) if is_multiset(child) => { - let DataType::Struct(fields) = child.data_type() else { - unreachable!("MULTISET entries are structs") - }; - let fields = vec![ - Arc::new( - parquet_field(&fields[0]) - .as_ref() - .clone() - .with_nullable(false), - ), - parquet_field(&fields[1]), - ]; - DataType::Map( - Arc::new(Field::new( - "entries", - DataType::Struct(fields.into()), - false, - )), - false, - ) - } - DataType::List(child) => DataType::List(parquet_field(child)), - DataType::LargeList(child) => DataType::LargeList(parquet_field(child)), - DataType::FixedSizeList(child, size) => { - DataType::FixedSizeList(parquet_field(child), *size) - } - DataType::Map(child, sorted) => DataType::Map(parquet_field(child), *sorted), - DataType::Struct(fields) => { - DataType::Struct(fields.iter().map(parquet_field).collect::>().into()) - } - data_type => data_type.clone(), - }; - Arc::new(field.as_ref().clone().with_data_type(data_type)) -} - -/// Convert logical MULTISET lists to the Parquet MAP layout, descending through -/// nested containers. Scalar conversion retains the existing timestamp casts. -pub(crate) fn to_parquet_array(array: &ArrayRef, target: &DataType) -> Result { - // NULL parents and sliced-away entries have no logical keys. Remove their - // hidden children before Parquet's non-null key validation. - let array = crate::table::normalize_write_array(array, array.data_type())?; - convert_parquet_array(&array, target) -} - -fn convert_parquet_array(array: &ArrayRef, target: &DataType) -> Result { - if array.data_type() == target { - return Ok(array.clone()); - } - let converted: ArrayRef = match (array.data_type(), target) { - (DataType::List(child), DataType::Map(entries, sorted)) if is_multiset(child) => { - let list = array.as_any().downcast_ref::().unwrap(); - let values = convert_parquet_array(list.values(), entries.data_type())?; - let values = values.as_any().downcast_ref::().unwrap(); - Arc::new( - MapArray::try_new( - entries.clone(), - list.offsets().clone(), - values.clone(), - list.nulls().cloned(), - *sorted, - ) - .map_err(storage_error)?, - ) - } - (DataType::List(_), DataType::List(child)) => { - let list = array.as_any().downcast_ref::().unwrap(); - Arc::new( - ListArray::try_new( - child.clone(), - list.offsets().clone(), - convert_parquet_array(list.values(), child.data_type())?, - list.nulls().cloned(), - ) - .map_err(storage_error)?, - ) - } - (DataType::LargeList(_), DataType::LargeList(child)) => { - let list = array.as_any().downcast_ref::().unwrap(); - Arc::new( - LargeListArray::try_new( - child.clone(), - list.offsets().clone(), - convert_parquet_array(list.values(), child.data_type())?, - list.nulls().cloned(), - ) - .map_err(storage_error)?, - ) - } - (DataType::FixedSizeList(_, size), DataType::FixedSizeList(child, _)) => { - let list = array.as_any().downcast_ref::().unwrap(); - Arc::new( - FixedSizeListArray::try_new_with_length( - child.clone(), - *size, - convert_parquet_array(list.values(), child.data_type())?, - list.nulls().cloned(), - list.len(), - ) - .map_err(storage_error)?, - ) - } - (DataType::Struct(_), DataType::Struct(fields)) => { - let structure = array.as_any().downcast_ref::().unwrap(); - let columns = structure - .columns() - .iter() - .zip(fields) - .map(|(column, field)| convert_parquet_array(column, field.data_type())) - .collect::>>()?; - Arc::new( - StructArray::try_new_with_length( - fields.clone(), - columns, - structure.nulls().cloned(), - structure.len(), - ) - .map_err(storage_error)?, - ) - } - (DataType::Map(_, _), DataType::Map(entries, sorted)) => { - let map = array.as_any().downcast_ref::().unwrap(); - let values: ArrayRef = Arc::new(map.entries().clone()); - let values = convert_parquet_array(&values, entries.data_type())?; - Arc::new( - MapArray::try_new( - entries.clone(), - map.offsets().clone(), - values - .as_any() - .downcast_ref::() - .unwrap() - .clone(), - map.nulls().cloned(), - *sorted, - ) - .map_err(storage_error)?, - ) - } - _ => arrow_cast::cast(array, target).map_err(storage_error)?, - }; - Ok(converted) -} - -fn storage_error(source: arrow_schema::ArrowError) -> Error { - Error::DataInvalid { message: format!("Cannot convert MULTISET to Parquet storage (Parquet map keys cannot be NULL): {source}"), source: Some(Box::new(source)) } -} - -#[cfg(test)] -mod tests { - use super::*; - use arrow_array::{Int32Array, RecordBatch}; - use arrow_buffer::{NullBuffer, OffsetBuffer}; - use bytes::Bytes; - use parquet::arrow::{arrow_reader::ParquetRecordBatchReaderBuilder, ArrowWriter}; - - fn bag(values: Vec>, nulls: Option) -> ArrayRef { - let field = entries_field(DataType::Int32, true); - let DataType::Struct(fields) = field.data_type() else { - unreachable!() - }; - let len = values.len(); - let entries = StructArray::try_new( - fields.clone(), - vec![ - Arc::new(Int32Array::from(values)), - Arc::new(Int32Array::from(vec![3; len])), - ], - None, - ) - .unwrap(); - Arc::new( - ListArray::try_new( - field, - OffsetBuffer::new((0..=len as i32).collect::>().into()), - Arc::new(entries), - nulls, - ) - .unwrap(), - ) - } - - #[test] - fn parquet_multiset_retains_java_map_layout_and_evolves_back_to_list() { - let logical = bag(vec![Some(2), Some(4)], None); - let schema = Arc::new(Schema::new(vec![Field::new( - "bag", - logical.data_type().clone(), - true, - )])); - let physical = parquet_schema(&schema); - assert!(matches!( - physical.field(0).data_type(), - DataType::Map(_, false) - )); - let array = to_parquet_array(&logical, physical.field(0).data_type()).unwrap(); - let mut bytes = Vec::new(); - let mut writer = ArrowWriter::try_new(&mut bytes, physical.clone(), None).unwrap(); - writer - .write(&RecordBatch::try_new(physical, vec![array]).unwrap()) - .unwrap(); - writer.close().unwrap(); - let mut reader = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(bytes)) - .unwrap() - .build() - .unwrap(); - let batch = reader.next().unwrap().unwrap(); - let typ = crate::spec::DataType::Multiset(crate::spec::MultisetType::new( - crate::spec::DataType::Int(crate::spec::IntType::new()), - )); - let restored = - crate::arrow::nested_evolution::evolve_column(batch.column(0), &typ, &typ).unwrap(); - assert_eq!(&logical.to_data(), &restored.to_data()); - } - - #[test] - fn parquet_drops_hidden_null_elements_but_rejects_visible_ones() { - let source = bag( - vec![None, None, Some(4)], - Some(NullBuffer::from(vec![true, false, true])), - ); - let schema = Arc::new(Schema::new(vec![Field::new( - "bag", - source.data_type().clone(), - true, - )])); - let physical = parquet_schema(&schema); - assert!(to_parquet_array(&source, physical.field(0).data_type()).is_err()); - // A sliced-away first bag and a NULL second bag hide both NULL keys. - let converted = - to_parquet_array(&source.slice(1, 2), physical.field(0).data_type()).unwrap(); - let map = converted.as_any().downcast_ref::().unwrap(); - assert!(map.is_null(0)); - assert_eq!( - map.keys() - .as_any() - .downcast_ref::() - .unwrap() - .values(), - &[4] - ); - } - - #[test] - fn legacy_parquet_optional_keys_remain_readable() { - use parquet::arrow::arrow_writer::ArrowWriterOptions; - use parquet::schema::{parser::parse_message_type, types::SchemaDescriptor}; - for values in [vec![Some(2), Some(4)], vec![None, Some(4)]] { - let logical = bag(values, None); - let schema = Arc::new(Schema::new(vec![Field::new( - "bag", - logical.data_type().clone(), - true, - )])); - // Arrow 58 accepted nullable MULTISET keys and emitted this - // non-compliant Parquet schema even when every actual key was valid. - let descriptor = SchemaDescriptor::new(Arc::new(parse_message_type( - "message schema { optional group bag (MAP) { repeated group key_value { optional int32 key; required int32 value; } } }", - ).unwrap())); - let options = ArrowWriterOptions::new() - .with_parquet_schema(descriptor) - .with_skip_arrow_metadata(true); - let mut bytes = Vec::new(); - let mut writer = - ArrowWriter::try_new_with_options(&mut bytes, schema.clone(), options).unwrap(); - writer - .write(&RecordBatch::try_new(schema, vec![logical.clone()]).unwrap()) - .unwrap(); - writer.close().unwrap(); - let batch = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(bytes)) - .unwrap() - .build() - .unwrap() - .next() - .unwrap() - .unwrap(); - let typ = crate::spec::DataType::Multiset(crate::spec::MultisetType::new( - crate::spec::DataType::Int(crate::spec::IntType::new()), - )); - let restored = - crate::arrow::nested_evolution::evolve_column(batch.column(0), &typ, &typ).unwrap(); - assert_eq!(logical.to_data(), restored.to_data()); - } - } - - #[test] - fn multiset_detection_descends_through_rows_arrays_and_maps() { - let bag = DataType::List(entries_field(DataType::Int32, true)); - let nested = DataType::Struct( - vec![Field::new( - "bags", - DataType::List(Arc::new(Field::new("element", bag, true))), - true, - )] - .into(), - ); - assert!(contains_multiset(&nested)); - let map = DataType::Map( - Arc::new(Field::new( - "entries", - DataType::Struct( - vec![ - Field::new("key", DataType::Utf8, false), - Field::new("value", nested, true), - ] - .into(), - ), - false, - )), - false, - ); - assert!(contains_multiset(&map)); - } - - #[test] - fn ordinary_arrays_of_key_value_rows_keep_list_storage() { - let field = Field::new( - "entries", - DataType::Struct( - vec![ - Field::new("key", DataType::Int32, true), - Field::new("value", DataType::Int32, false), - ] - .into(), - ), - false, - ); - let schema = Arc::new(Schema::new(vec![Field::new( - "array", - DataType::List(Arc::new(field)), - true, - )])); - assert_eq!(parquet_schema(&schema), schema); - } -} diff --git a/crates/paimon/src/arrow/nested_evolution.rs b/crates/paimon/src/arrow/nested_evolution.rs index 5c136f270..015b2fee4 100644 --- a/crates/paimon/src/arrow/nested_evolution.rs +++ b/crates/paimon/src/arrow/nested_evolution.rs @@ -758,9 +758,9 @@ fn evolve_map( ) } -/// Rebuild a MULTISET whose element type evolved. Arrow uses a list of -/// element/count entries; Parquet and historical files can decode as a map. -/// Only the element side evolves and the counts carry over untouched. +/// Rebuild a MULTISET whose element type evolved. Arrow renders a MULTISET as a +/// map of element -> count, so only the key side is reconciled and the counts +/// carry over untouched. /// /// Java's read-side cast dispatcher has no MULTISET branch and raises "Cannot /// cast from type ... to type ..." for such a schema change; going through the @@ -772,46 +772,16 @@ fn evolve_multiset( source_element: &DataType, target_element: &DataType, ) -> crate::Result { - let list = if let Some(map) = source.as_any().downcast_ref::() { - visible_generic_list(&ListArray::from(map.clone()))? - } else { - visible_list(source)? - }; - let entries = list - .values() - .as_any() - .downcast_ref::() - .filter(|entries| entries.num_columns() == 2) - .ok_or_else(|| crate::Error::DataInvalid { - message: "MULTISET entries must contain an element and count".into(), - source: None, - })?; - let elements = evolve_column(entries.column(0), source_element, target_element)?; - let field = - crate::arrow::multiset::entries_field(paimon_type_to_arrow_for_read(target_element)?, true); - let ArrowDataType::Struct(fields) = field.data_type() else { - unreachable!() - }; - let entries = StructArray::try_new( - fields.clone(), - vec![elements, entries.column(1).clone()], - None, - ) - .map_err(|error| crate::Error::DataInvalid { - message: format!("failed to build schema-evolved MULTISET: {error}"), - source: None, - })?; - let result = ListArray::try_new( - field, - list.offsets().clone(), - Arc::new(entries), - list.nulls().cloned(), + let map = visible_map(as_map_array(source, "MULTISET")?)?; + let elements = evolve_column(map.keys(), source_element, target_element)?; + let counts = map.values().clone(); + rebuild_map( + &map, + elements, + counts, + ArrowField::new("key", paimon_type_to_arrow_for_read(target_element)?, false), + ArrowField::new("value", ArrowDataType::Int32, false), ) - .map_err(|error| crate::Error::DataInvalid { - message: format!("failed to build schema-evolved MULTISET: {error}"), - source: None, - })?; - Ok(Arc::new(result)) } fn as_map_array<'a>(source: &'a ArrayRef, kind: &str) -> crate::Result<&'a MapArray> { @@ -1929,8 +1899,8 @@ mod tests { #[test] fn readded_child_inside_a_multiset_does_not_serve_the_dropped_field_values() { // MULTISET> where `x` was dropped and re-added with a new id. - // A historical Parquet MULTISET decodes as Map; the - // current list representation must still reconcile the field ids. + // Arrow renders a MULTISET as a Map of element -> count, so the source + // and target Arrow types are identical and only the field ids differ. let x: ArrayRef = Arc::new(StringArray::from(vec![Some("old")])); let element_fields = Fields::from(vec![ArrowField::new("x", ArrowDataType::Utf8, true)]); let elements: ArrayRef = @@ -1971,8 +1941,12 @@ mod tests { let out = evolve_column(&source, &source_type, &target_type).unwrap(); - let entries = crate::arrow::multiset::entries(out.as_ref(), 0).unwrap(); - let element = as_struct(entries.column(0)); + let map = out + .as_any() + .downcast_ref::() + .unwrap() + .clone(); + let element = as_struct(&map.keys().clone()); assert_eq!( element.column_by_name("x").unwrap().null_count(), 1, diff --git a/crates/paimon/src/table/postpone_bucket_planner.rs b/crates/paimon/src/table/postpone_bucket_planner.rs index fe72233e4..e259ff6b5 100644 --- a/crates/paimon/src/table/postpone_bucket_planner.rs +++ b/crates/paimon/src/table/postpone_bucket_planner.rs @@ -25,8 +25,8 @@ use crate::spec::{ use crate::table::{PostponeBucketPlan, Table}; use crate::Result; use arrow_array::{ - Array, ArrayRef, FixedSizeListArray, Int64Array, LargeListArray, ListArray, RecordBatch, - StructArray, + Array, ArrayRef, FixedSizeListArray, Int64Array, LargeListArray, ListArray, MapArray, + RecordBatch, StructArray, }; use arrow_schema::{DataType as ArrowType, Field, Schema}; use std::collections::{HashMap, HashSet}; @@ -635,7 +635,11 @@ fn variable_size(array: &ArrayRef, row: usize, data_type: &DataType) -> Result Result { - let entries = crate::arrow::multiset::entries(array.as_ref(), row)?; + let entries = array + .as_any() + .downcast_ref::() + .ok_or_else(|| data_invalid("Expected MAP input while sizing postpone data"))? + .value(row); let keys = array_size(entries.column(0), key)?; let values = array_size(entries.column(1), value)?; round_word(checked_size( @@ -820,52 +824,6 @@ mod tests { assert_eq!(planner.input_statistics(&batch).unwrap().num_rows(), 0); } - #[test] - fn input_statistics_rejects_malformed_multiset_entries() { - let schema = TableSchema::new( - 0, - &crate::spec::Schema::builder() - .column("id", DataType::Int(IntType::new())) - .column( - "bag", - DataType::Multiset(crate::spec::MultisetType::new(DataType::Int( - IntType::new(), - ))), - ) - .primary_key(["id"]) - .option("bucket", "-2") - .option("postpone.target-size-per-bucket", "20 b") - .build() - .unwrap(), - ); - let planner = PostponeBucketPlanner::new(&schema, PostponeBucketPlan::default()).unwrap(); - let entries = StructArray::try_new( - vec![Arc::new(Field::new("key", ArrowType::Int32, true))].into(), - vec![Arc::new(Int32Array::from(vec![5]))], - None, - ) - .unwrap(); - let bag = ListArray::try_new( - Arc::new(Field::new("entries", entries.data_type().clone(), false)), - arrow_buffer::OffsetBuffer::new(vec![0, 1].into()), - Arc::new(entries), - None, - ) - .unwrap(); - let batch = RecordBatch::try_new( - Arc::new(Schema::new(vec![Field::new( - "bag", - bag.data_type().clone(), - true, - )])), - vec![Arc::new(bag)], - ) - .unwrap(); - let error = planner.input_statistics(&batch).unwrap_err(); - assert!(matches!(&error, crate::Error::DataInvalid { .. })); - assert!(error.to_string().contains("exactly two fields")); - } - #[test] fn merged_worker_statistics_are_inferred_once_and_keep_known_counts() { let schema = planner_schema(&[("postpone.target-row-num-per-bucket", "2")]); diff --git a/crates/paimon/src/table/write_batch_normalize.rs b/crates/paimon/src/table/write_batch_normalize.rs index f2bd421fc..13337c613 100644 --- a/crates/paimon/src/table/write_batch_normalize.rs +++ b/crates/paimon/src/table/write_batch_normalize.rs @@ -327,30 +327,6 @@ fn normalize_visible_array( |row| (!fixed.is_null(row)).then(|| fixed.value(row)), )))) } - (DataType::Map(_, _), DataType::List(expected_child)) - if crate::arrow::multiset::is_multiset(expected_child) => - { - let map = array.as_any().downcast_ref::().unwrap(); - let nulls = NullBuffer::union(map.nulls(), parent_nulls); - let entries: ArrayRef = Arc::new(map.entries().clone()); - let (entries, offsets) = visible_children(&entries, map.offsets(), nulls.as_ref())?; - let entries = entries.as_any().downcast_ref::().unwrap(); - let DataType::Struct(fields) = expected_child.data_type() else { - unreachable!() - }; - let columns = entries - .columns() - .iter() - .zip(fields) - .map(|(column, field)| normalize_write_array(column, field.data_type())) - .collect::>>()?; - let entries = - StructArray::try_new(fields.clone(), columns, None).map_err(normalization_error)?; - Ok(Arc::new( - ListArray::try_new(expected_child.clone(), offsets, Arc::new(entries), nulls) - .map_err(normalization_error)?, - )) - } (DataType::List(_), DataType::List(expected_child)) => { let list = array.as_any().downcast_ref::().unwrap(); let nulls = NullBuffer::union(list.nulls(), parent_nulls); @@ -613,52 +589,6 @@ fn normalization_error(error: arrow_schema::ArrowError) -> Error { #[cfg(test)] mod tests { use super::*; - - #[test] - fn legacy_map_input_normalizes_to_multiset_list() { - let fields = arrow_schema::Fields::from(vec![ - Field::new("old_key", DataType::Int32, false), - Field::new("old_count", DataType::Int32, false), - ]); - let entries = StructArray::try_new( - fields.clone(), - vec![ - Arc::new(arrow_array::Int32Array::from(vec![5])), - Arc::new(arrow_array::Int32Array::from(vec![3])), - ], - None, - ) - .unwrap(); - let source: ArrayRef = Arc::new(MapArray::new( - Arc::new(Field::new("entries", DataType::Struct(fields), false)), - OffsetBuffer::new(vec![0, 1].into()), - entries, - None, - false, - )); - let target = DataType::List(crate::arrow::multiset::entries_field(DataType::Int32, true)); - let normalized = normalize_write_array(&source, &target).unwrap(); - let entries = crate::arrow::multiset::entries(normalized.as_ref(), 0).unwrap(); - assert_eq!(normalized.data_type(), &target); - assert_eq!( - entries - .column(0) - .as_any() - .downcast_ref::() - .unwrap() - .value(0), - 5 - ); - assert_eq!( - entries - .column(1) - .as_any() - .downcast_ref::() - .unwrap() - .value(0), - 3 - ); - } use arrow_array::{FixedSizeListArray, Int32Array, LargeBinaryArray, StringArray}; use arrow_buffer::{OffsetBuffer, ScalarBuffer}; use arrow_schema::Field; diff --git a/crates/paimon/tests/incremental_diff_extended_test.rs b/crates/paimon/tests/incremental_diff_extended_test.rs index 3f728d092..cf67ce42d 100644 --- a/crates/paimon/tests/incremental_diff_extended_test.rs +++ b/crates/paimon/tests/incremental_diff_extended_test.rs @@ -118,14 +118,12 @@ fn nested_schema() -> TableSchema { TableSchema::new(0, &schema) } -fn map_or_multiset_array(arrow_type: &ArrowDataType, values: &[StringMap]) -> ArrayRef { - let (entries_field, sorted) = match arrow_type { - ArrowDataType::Map(entries_field, sorted) => (entries_field, Some(*sorted)), - ArrowDataType::List(entries_field) => (entries_field, None), - _ => panic!("expected Arrow Map or multiset List"), +fn map_array(arrow_type: &ArrowDataType, values: &[StringMap]) -> MapArray { + let ArrowDataType::Map(entries_field, sorted) = arrow_type else { + panic!("expected Arrow Map") }; let ArrowDataType::Struct(entry_fields) = entries_field.data_type() else { - panic!("expected entries struct") + panic!("expected Map entries struct") }; let mut offsets = vec![0_i32]; let mut keys = Vec::new(); @@ -150,17 +148,14 @@ fn map_or_multiset_array(arrow_type: &ArrowDataType, values: &[StringMap]) -> Ar None, ) .unwrap(); - let offsets = OffsetBuffer::new(ScalarBuffer::from(offsets)); - let nulls = Some(NullBuffer::from(valid)); - match sorted { - Some(sorted) => Arc::new( - MapArray::try_new(Arc::clone(entries_field), offsets, entries, nulls, sorted).unwrap(), - ), - None => Arc::new( - ListArray::try_new(Arc::clone(entries_field), offsets, Arc::new(entries), nulls) - .unwrap(), - ), - } + MapArray::try_new( + Arc::clone(entries_field), + OffsetBuffer::new(ScalarBuffer::from(offsets)), + entries, + Some(NullBuffer::from(valid)), + *sorted, + ) + .unwrap() } fn nested_batch(table: &Table, rows: &[NestedRow]) -> RecordBatch { @@ -227,8 +222,8 @@ fn nested_batch(table: &Table, rows: &[NestedRow]) -> RecordBatch { )), Arc::new(payload), Arc::new(items), - map_or_multiset_array(schema.field(3).data_type(), &attributes), - map_or_multiset_array(schema.field(4).data_type(), &bag), + Arc::new(map_array(schema.field(3).data_type(), &attributes)), + Arc::new(map_array(schema.field(4).data_type(), &bag)), ], ) .unwrap() diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index b6fe051d1..67f564a10 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -148,19 +148,13 @@ of at most a quarter of the limit and closes the row group once its encoded size reaches the limit, so large or skewed batches stay close to it. These options affect new writes only. -## Multiset Arrow Values - -`MULTISET` uses Arrow `List>`, with a -non-null entry struct and count. The element (`key`) follows the logical element -nullability; this preserves NULL multiset elements, which Arrow Map cannot -represent. The list entry field carries `paimon.multiset=true` metadata so that -schema inference distinguishes it from an ordinary array of rows. Existing Arrow -Map write inputs with non-null keys are also accepted for multiset columns. - -Row and Avro retain their Java file encodings. Parquet retains the Java MAP -encoding, whose keys must be non-null; writing a multiset with a visible NULL -element to Parquet returns an error. Mosaic and Vortex do not support multiset -columns. +## Map and Multiset Arrow Values + +`MAP` and `MULTISET` both use Arrow Map with a non-null entry struct +and non-null keys. A multiset's values are non-null `Int32` occurrence counts. +Column and ordinary map value nullability follow the existing schema rules. +NULL keys are rejected by Arrow's runtime validation. +Row, Avro, and Parquet retain their Java-compatible file encodings. ## Parquet Page Pruning