Skip to content

refactor(context): deduplicate register/read option-building logic #1479

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mesejo wants to merge 1 commit into
base: main
Choose a base branch
from
+131 −150
Open

refactor(context): deduplicate register/read option-building logic #1479

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
281 changes: 131 additions & 150 deletions crates/core/src/context.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
// under the License.

use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::ptr::NonNull;
use std::str::FromStr;
use std::sync::Arc;
Expand Down Expand Up @@ -456,19 +455,8 @@ impl PySessionContext {
) -> PyDataFusionResult<()> {
let options = ListingOptions::new(Arc::new(ParquetFormat::new()))
.with_file_extension(file_extension)
.with_table_partition_cols(
table_partition_cols
.into_iter()
.map(|(name, ty)| (name, ty.0))
.collect::<Vec<(String, DataType)>>(),
)
.with_file_sort_order(
file_sort_order
.unwrap_or_default()
.into_iter()
.map(|e| e.into_iter().map(|f| f.into()).collect())
.collect(),
);
.with_table_partition_cols(convert_partition_cols(table_partition_cols))
.with_file_sort_order(convert_file_sort_order(file_sort_order));
let table_path = ListingTableUrl::parse(path)?;
let resolved_schema: SchemaRef = match schema {
Some(s) => Arc::new(s.0),
Expand Down Expand Up @@ -831,25 +819,15 @@ impl PySessionContext {
file_sort_order: Option<Vec<Vec<PySortExpr>>>,
py: Python,
) -> PyDataFusionResult<()> {
let mut options = ParquetReadOptions::default()
.table_partition_cols(
table_partition_cols
.into_iter()
.map(|(name, ty)| (name, ty.0))
.collect::<Vec<(String, DataType)>>(),
)
.parquet_pruning(parquet_pruning)
.skip_metadata(skip_metadata);
options.file_extension = file_extension;
options.schema = schema.as_ref().map(|x| &x.0);
options.file_sort_order = file_sort_order
.unwrap_or_default()
.into_iter()
.map(|e| e.into_iter().map(|f| f.into()).collect())
.collect();

let result = self.ctx.register_parquet(name, path, options);
wait_for_future(py, result)??;
let options = build_parquet_options(
table_partition_cols,
parquet_pruning,
file_extension,
skip_metadata,
&schema,
file_sort_order,
);
wait_for_future(py, self.ctx.register_parquet(name, path, options))??;
Ok(())
}

Expand All @@ -863,19 +841,17 @@ impl PySessionContext {
options: Option<&PyCsvReadOptions>,
py: Python,
) -> PyDataFusionResult<()> {
let options = options
.map(|opts| opts.try_into())
.transpose()?
.unwrap_or_default();
let options = convert_csv_options(options)?;

if path.is_instance_of::<PyList>() {
let paths = path.extract::<Vec<String>>()?;
let result = self.register_csv_from_multiple_paths(name, paths, options);
wait_for_future(py, result)??;
wait_for_future(
py,
self.register_csv_from_multiple_paths(name, paths, options),
)??;
} else {
let path = path.extract::<String>()?;
let result = self.ctx.register_csv(name, &path, options);
wait_for_future(py, result)??;
wait_for_future(py, self.ctx.register_csv(name, &path, options))??;
}

Ok(())
Expand All @@ -892,33 +868,22 @@ impl PySessionContext {
pub fn register_json(
&self,
name: &str,
path: PathBuf,
path: &str,
schema: Option<PyArrowType<Schema>>,
Comment on lines 868 to 872
Copy link

Copilot AI Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changing path from PathBuf to &str in this #[pymethods] API is a breaking change for Python callers that previously passed pathlib.Path / other os.PathLike objects (PyO3 can extract those into PathBuf, but not into &str). Consider keeping PathBuf (and moving the to_str() conversion into a helper), or accept a Bound<'_, PyAny> and explicitly support both str and os.PathLike inputs to preserve the existing Python surface area.

Copilot uses AI. Check for mistakes.
schema_infer_max_records: usize,
file_extension: &str,
table_partition_cols: Vec<(String, PyArrowType<DataType>)>,
file_compression_type: Option<String>,
py: Python,
) -> PyDataFusionResult<()> {
Comment on lines 868 to 878
Copy link

Copilot AI Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The PR description says "No user-facing changes", but the switch from PathBuf to &str for JSON/Avro read/register methods is user-facing for Python (it changes which argument types are accepted, e.g., pathlib.Path). Either restore the previous behavior or update the PR description / changelog notes accordingly.

Copilot uses AI. Check for mistakes.
let path = path
.to_str()
.ok_or_else(|| PyValueError::new_err("Unable to convert path to a string"))?;

let mut options = JsonReadOptions::default()
.file_compression_type(parse_file_compression_type(file_compression_type)?)
.table_partition_cols(
table_partition_cols
.into_iter()
.map(|(name, ty)| (name, ty.0))
.collect::<Vec<(String, DataType)>>(),
);
options.schema_infer_max_records = schema_infer_max_records;
options.file_extension = file_extension;
options.schema = schema.as_ref().map(|x| &x.0);

let result = self.ctx.register_json(name, path, options);
wait_for_future(py, result)??;

let options = build_json_options(
table_partition_cols,
file_compression_type,
schema_infer_max_records,
file_extension,
&schema,
)?;
wait_for_future(py, self.ctx.register_json(name, path, options))??;
Ok(())
}

Expand All @@ -931,28 +896,14 @@ impl PySessionContext {
pub fn register_avro(
&self,
name: &str,
path: PathBuf,
path: &str,
schema: Option<PyArrowType<Schema>>,
file_extension: &str,
table_partition_cols: Vec<(String, PyArrowType<DataType>)>,
py: Python,
) -> PyDataFusionResult<()> {
let path = path
.to_str()
.ok_or_else(|| PyValueError::new_err("Unable to convert path to a string"))?;

let mut options = AvroReadOptions::default().table_partition_cols(
table_partition_cols
.into_iter()
.map(|(name, ty)| (name, ty.0))
.collect::<Vec<(String, DataType)>>(),
);
options.file_extension = file_extension;
options.schema = schema.as_ref().map(|x| &x.0);

let result = self.ctx.register_avro(name, path, options);
wait_for_future(py, result)??;

let options = build_avro_options(table_partition_cols, file_extension, &schema);
wait_for_future(py, self.ctx.register_avro(name, path, options))??;
Ok(())
}

Expand Down Expand Up @@ -1054,35 +1005,22 @@ impl PySessionContext {
#[pyo3(signature = (path, schema=None, schema_infer_max_records=1000, file_extension=".json", table_partition_cols=vec![], file_compression_type=None))]
pub fn read_json(
&self,
path: PathBuf,
path: &str,
schema: Option<PyArrowType<Schema>>,
schema_infer_max_records: usize,
file_extension: &str,
table_partition_cols: Vec<(String, PyArrowType<DataType>)>,
file_compression_type: Option<String>,
py: Python,
) -> PyDataFusionResult<PyDataFrame> {
let path = path
.to_str()
.ok_or_else(|| PyValueError::new_err("Unable to convert path to a string"))?;
let mut options = JsonReadOptions::default()
.table_partition_cols(
table_partition_cols
.into_iter()
.map(|(name, ty)| (name, ty.0))
.collect::<Vec<(String, DataType)>>(),
)
.file_compression_type(parse_file_compression_type(file_compression_type)?);
options.schema_infer_max_records = schema_infer_max_records;
options.file_extension = file_extension;
let df = if let Some(schema) = schema {
options.schema = Some(&schema.0);
let result = self.ctx.read_json(path, options);
wait_for_future(py, result)??
} else {
let result = self.ctx.read_json(path, options);
wait_for_future(py, result)??
};
let options = build_json_options(
table_partition_cols,
file_compression_type,
schema_infer_max_records,
file_extension,
&schema,
)?;
let df = wait_for_future(py, self.ctx.read_json(path, options))??;
Ok(PyDataFrame::new(df))
}

Expand All @@ -1095,23 +1033,15 @@ impl PySessionContext {
options: Option<&PyCsvReadOptions>,
py: Python,
) -> PyDataFusionResult<PyDataFrame> {
let options = options
.map(|opts| opts.try_into())
.transpose()?
.unwrap_or_default();
let options = convert_csv_options(options)?;

if path.is_instance_of::<PyList>() {
let paths = path.extract::<Vec<String>>()?;
let paths = paths.iter().map(|p| p as &str).collect::<Vec<&str>>();
let result = self.ctx.read_csv(paths, options);
let df = PyDataFrame::new(wait_for_future(py, result)??);
Ok(df)
let paths: Vec<String> = if path.is_instance_of::<PyList>() {
path.extract()?
} else {
let path = path.extract::<String>()?;
let result = self.ctx.read_csv(path, options);
let df = PyDataFrame::new(wait_for_future(py, result)??);
Ok(df)
}
vec![path.extract()?]
};
let df = wait_for_future(py, self.ctx.read_csv(paths, options))??;
Ok(PyDataFrame::new(df))
}

#[allow(clippy::too_many_arguments)]
Expand All @@ -1134,25 +1064,15 @@ impl PySessionContext {
file_sort_order: Option<Vec<Vec<PySortExpr>>>,
py: Python,
) -> PyDataFusionResult<PyDataFrame> {
let mut options = ParquetReadOptions::default()
.table_partition_cols(
table_partition_cols
.into_iter()
.map(|(name, ty)| (name, ty.0))
.collect::<Vec<(String, DataType)>>(),
)
.parquet_pruning(parquet_pruning)
.skip_metadata(skip_metadata);
options.file_extension = file_extension;
options.schema = schema.as_ref().map(|x| &x.0);
options.file_sort_order = file_sort_order
.unwrap_or_default()
.into_iter()
.map(|e| e.into_iter().map(|f| f.into()).collect())
.collect();

let result = self.ctx.read_parquet(path, options);
let df = PyDataFrame::new(wait_for_future(py, result)??);
let options = build_parquet_options(
table_partition_cols,
parquet_pruning,
file_extension,
skip_metadata,
&schema,
file_sort_order,
);
let df = PyDataFrame::new(wait_for_future(py, self.ctx.read_parquet(path, options))??);
Ok(df)
}

Expand All @@ -1166,21 +1086,8 @@ impl PySessionContext {
file_extension: &str,
py: Python,
) -> PyDataFusionResult<PyDataFrame> {
let mut options = AvroReadOptions::default().table_partition_cols(
table_partition_cols
.into_iter()
.map(|(name, ty)| (name, ty.0))
.collect::<Vec<(String, DataType)>>(),
);
options.file_extension = file_extension;
let df = if let Some(schema) = schema {
options.schema = Some(&schema.0);
let read_future = self.ctx.read_avro(path, options);
wait_for_future(py, read_future)??
} else {
let read_future = self.ctx.read_avro(path, options);
wait_for_future(py, read_future)??
};
let options = build_avro_options(table_partition_cols, file_extension, &schema);
let df = wait_for_future(py, self.ctx.read_avro(path, options))??;
Ok(PyDataFrame::new(df))
}

Expand Down Expand Up @@ -1280,7 +1187,7 @@ impl PySessionContext {
// check if the file extension matches the expected extension
for path in &table_paths {
let file_path = path.as_str();
if !file_path.ends_with(option_extension.clone().as_str()) && !path.is_collection() {
if !file_path.ends_with(option_extension.as_str()) && !path.is_collection() {
return exec_err!(
"File path '{file_path}' does not match the expected extension '{option_extension}'"
);
Expand Down Expand Up @@ -1321,6 +1228,80 @@ pub fn parse_file_compression_type(
})
}

fn convert_csv_options(
options: Option<&PyCsvReadOptions>,
) -> PyDataFusionResult<CsvReadOptions<'_>> {
Ok(options
.map(|opts| opts.try_into())
.transpose()?
.unwrap_or_default())
}

fn convert_partition_cols(
table_partition_cols: Vec<(String, PyArrowType<DataType>)>,
) -> Vec<(String, DataType)> {
table_partition_cols
.into_iter()
.map(|(name, ty)| (name, ty.0))
.collect()
}

fn convert_file_sort_order(
file_sort_order: Option<Vec<Vec<PySortExpr>>>,
) -> Vec<Vec<datafusion::logical_expr::SortExpr>> {
file_sort_order
.unwrap_or_default()
.into_iter()
.map(|e| e.into_iter().map(|f| f.into()).collect())
.collect()
}

fn build_parquet_options<'a>(
table_partition_cols: Vec<(String, PyArrowType<DataType>)>,
parquet_pruning: bool,
file_extension: &'a str,
skip_metadata: bool,
schema: &'a Option<PyArrowType<Schema>>,
file_sort_order: Option<Vec<Vec<PySortExpr>>>,
) -> ParquetReadOptions<'a> {
let mut options = ParquetReadOptions::default()
.table_partition_cols(convert_partition_cols(table_partition_cols))
.parquet_pruning(parquet_pruning)
.skip_metadata(skip_metadata);
options.file_extension = file_extension;
options.schema = schema.as_ref().map(|x| &x.0);
options.file_sort_order = convert_file_sort_order(file_sort_order);
options
}

fn build_json_options<'a>(
table_partition_cols: Vec<(String, PyArrowType<DataType>)>,
file_compression_type: Option<String>,
schema_infer_max_records: usize,
file_extension: &'a str,
schema: &'a Option<PyArrowType<Schema>>,
) -> Result<JsonReadOptions<'a>, PyErr> {
let mut options = JsonReadOptions::default()
.table_partition_cols(convert_partition_cols(table_partition_cols))
.file_compression_type(parse_file_compression_type(file_compression_type)?);
options.schema_infer_max_records = schema_infer_max_records;
options.file_extension = file_extension;
options.schema = schema.as_ref().map(|x| &x.0);
Ok(options)
}

fn build_avro_options<'a>(
table_partition_cols: Vec<(String, PyArrowType<DataType>)>,
file_extension: &'a str,
schema: &'a Option<PyArrowType<Schema>>,
) -> AvroReadOptions<'a> {
let mut options = AvroReadOptions::default()
.table_partition_cols(convert_partition_cols(table_partition_cols));
options.file_extension = file_extension;
options.schema = schema.as_ref().map(|x| &x.0);
options
}

impl From<PySessionContext> for SessionContext {
fn from(ctx: PySessionContext) -> SessionContext {
ctx.ctx.as_ref().clone()
Expand Down
Loading