From dc1c61065844f1db2bbb14ef9a9ec3a8335697f5 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Tue, 22 Sep 2026 13:19:17 +0200 Subject: [PATCH 1/4] Accept write_table options in dataset --- python/pyarrow/_dataset_parquet.pyx | 10 +++++++++- python/pyarrow/parquet/core.py | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/_dataset_parquet.pyx b/python/pyarrow/_dataset_parquet.pyx index fac8e96df85b..43eca8957be8 100644 --- a/python/pyarrow/_dataset_parquet.pyx +++ b/python/pyarrow/_dataset_parquet.pyx @@ -620,6 +620,8 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): "coerce_timestamps", "allow_truncated_timestamps", "use_compliant_nested_type", + "store_schema", + "write_time_adjusted_to_utc", } setters = set() @@ -661,6 +663,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): sorting_columns=self._properties["sorting_columns"], store_decimal_as_integer=self._properties["store_decimal_as_integer"], use_content_defined_chunking=self._properties["use_content_defined_chunking"], + bloom_filter_options=self._properties["bloom_filter_options"], ) def _set_arrow_properties(self): @@ -677,7 +680,9 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): writer_engine_version="V2", use_compliant_nested_type=( self._properties["use_compliant_nested_type"] - ) + ), + store_schema=self._properties["store_schema"], + write_time_adjusted_to_utc=self._properties["write_time_adjusted_to_utc"], ) def _set_encryption_config(self): @@ -706,6 +711,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): coerce_timestamps=None, allow_truncated_timestamps=False, use_compliant_nested_type=True, + store_schema=True, encryption_properties=None, write_batch_size=None, dictionary_pagesize_limit=None, @@ -715,6 +721,8 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): sorting_columns=None, store_decimal_as_integer=False, use_content_defined_chunking=False, + write_time_adjusted_to_utc=False, + bloom_filter_options=None, ) self._set_properties() diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py index 152c2694e2ac..5449d16f3cfd 100644 --- a/python/pyarrow/parquet/core.py +++ b/python/pyarrow/parquet/core.py @@ -1088,6 +1088,7 @@ def __init__(self, where, schema, filesystem=None, store_decimal_as_integer=False, write_time_adjusted_to_utc=False, max_rows_per_page=None, + bloom_filter_options=None, use_content_defined_chunking=False, **options): if use_deprecated_int96_timestamps is None: @@ -1144,6 +1145,7 @@ def __init__(self, where, schema, filesystem=None, store_decimal_as_integer=store_decimal_as_integer, write_time_adjusted_to_utc=write_time_adjusted_to_utc, max_rows_per_page=max_rows_per_page, + bloom_filter_options=bloom_filter_options, use_content_defined_chunking=use_content_defined_chunking, **options) self.is_open = True From c484ecacc5052fe482db25df46e4c8bc4444ca8a Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Tue, 22 Sep 2026 15:40:46 +0200 Subject: [PATCH 2/4] Add tests --- python/pyarrow/tests/parquet/test_dataset.py | 26 ++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/python/pyarrow/tests/parquet/test_dataset.py b/python/pyarrow/tests/parquet/test_dataset.py index 88603d9b5b8e..a124c7e74e96 100644 --- a/python/pyarrow/tests/parquet/test_dataset.py +++ b/python/pyarrow/tests/parquet/test_dataset.py @@ -1312,6 +1312,32 @@ def file_visitor(written_file): assert paths_written_set == expected_paths +def test_write_table_options_in_make_write_options(): + import pyarrow.dataset as ds + + not_writer_args = {"table", "where", "row_group_size", "filesystem", "flavor"} + options = { + name: parameter.default + for name, parameter in inspect.signature(pq.write_table).parameters.items() + if name not in not_writer_args + and parameter.kind != inspect.Parameter.VAR_KEYWORD + } + ds.ParquetFileFormat().make_write_options(**options) + + +def test_write_to_dataset_options(tempdir): + table = pa.table({"a": [1, 2, 3], + "t": pa.array([1, 2, 3], pa.time32("ms"))}) + pq.write_to_dataset(table, tempdir, store_schema=False, + write_time_adjusted_to_utc=True, + bloom_filter_options={"a": True}, + ) + metadata = pq.read_metadata(next(tempdir.glob("*.parquet"))) + assert b'ARROW:schema' not in (metadata.metadata or {}) + assert 'isAdjustedToUTC=true' in str(metadata.schema.column(1).logical_type) + assert metadata.row_group(0).column(0).bloom_filter_offset is not None + + @pytest.mark.parametrize("write_dataset_kwarg", ( ("create_dir", True), ("create_dir", False), From b951a922358092927d0bd2640f21316dcb28fa4f Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Fri, 25 Sep 2026 14:53:54 +0200 Subject: [PATCH 3/4] Format line --- python/pyarrow/tests/parquet/test_dataset.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/pyarrow/tests/parquet/test_dataset.py b/python/pyarrow/tests/parquet/test_dataset.py index a124c7e74e96..257e666f9ebf 100644 --- a/python/pyarrow/tests/parquet/test_dataset.py +++ b/python/pyarrow/tests/parquet/test_dataset.py @@ -1330,8 +1330,7 @@ def test_write_to_dataset_options(tempdir): "t": pa.array([1, 2, 3], pa.time32("ms"))}) pq.write_to_dataset(table, tempdir, store_schema=False, write_time_adjusted_to_utc=True, - bloom_filter_options={"a": True}, - ) + bloom_filter_options={"a": True}) metadata = pq.read_metadata(next(tempdir.glob("*.parquet"))) assert b'ARROW:schema' not in (metadata.metadata or {}) assert 'isAdjustedToUTC=true' in str(metadata.schema.column(1).logical_type) From 18e4b1e794ce87804207ada096da6e2f7a0c0760 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Fri, 25 Sep 2026 18:43:05 +0200 Subject: [PATCH 4/4] Add test comment --- python/pyarrow/tests/parquet/test_dataset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/pyarrow/tests/parquet/test_dataset.py b/python/pyarrow/tests/parquet/test_dataset.py index 257e666f9ebf..28b636637965 100644 --- a/python/pyarrow/tests/parquet/test_dataset.py +++ b/python/pyarrow/tests/parquet/test_dataset.py @@ -1315,6 +1315,8 @@ def file_visitor(written_file): def test_write_table_options_in_make_write_options(): import pyarrow.dataset as ds + # Test write_table options are in sync with the dataset writer. + # Any option not in ParquetFileWriteOptions raises TypeError in make_write_options not_writer_args = {"table", "where", "row_group_size", "filesystem", "flavor"} options = { name: parameter.default