From cb2ba8574b5cbe7fb9c7bbffe2cc4c1b833c527e Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Tue, 22 Sep 2026 13:19:17 +0200 Subject: [PATCH 1/2] Accept write_table options in dataset --- python/pyarrow/_dataset_parquet.pyx | 10 +++++++++- python/pyarrow/parquet/core.py | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/_dataset_parquet.pyx b/python/pyarrow/_dataset_parquet.pyx index fac8e96df85..43eca8957be 100644 --- a/python/pyarrow/_dataset_parquet.pyx +++ b/python/pyarrow/_dataset_parquet.pyx @@ -620,6 +620,8 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): "coerce_timestamps", "allow_truncated_timestamps", "use_compliant_nested_type", + "store_schema", + "write_time_adjusted_to_utc", } setters = set() @@ -661,6 +663,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): sorting_columns=self._properties["sorting_columns"], store_decimal_as_integer=self._properties["store_decimal_as_integer"], use_content_defined_chunking=self._properties["use_content_defined_chunking"], + bloom_filter_options=self._properties["bloom_filter_options"], ) def _set_arrow_properties(self): @@ -677,7 +680,9 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): writer_engine_version="V2", use_compliant_nested_type=( self._properties["use_compliant_nested_type"] - ) + ), + store_schema=self._properties["store_schema"], + write_time_adjusted_to_utc=self._properties["write_time_adjusted_to_utc"], ) def _set_encryption_config(self): @@ -706,6 +711,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): coerce_timestamps=None, allow_truncated_timestamps=False, use_compliant_nested_type=True, + store_schema=True, encryption_properties=None, write_batch_size=None, dictionary_pagesize_limit=None, @@ -715,6 +721,8 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): sorting_columns=None, store_decimal_as_integer=False, use_content_defined_chunking=False, + write_time_adjusted_to_utc=False, + bloom_filter_options=None, ) self._set_properties() diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py index 152c2694e2a..5449d16f3cf 100644 --- a/python/pyarrow/parquet/core.py +++ b/python/pyarrow/parquet/core.py @@ -1088,6 +1088,7 @@ def __init__(self, where, schema, filesystem=None, store_decimal_as_integer=False, write_time_adjusted_to_utc=False, max_rows_per_page=None, + bloom_filter_options=None, use_content_defined_chunking=False, **options): if use_deprecated_int96_timestamps is None: @@ -1144,6 +1145,7 @@ def __init__(self, where, schema, filesystem=None, store_decimal_as_integer=store_decimal_as_integer, write_time_adjusted_to_utc=write_time_adjusted_to_utc, max_rows_per_page=max_rows_per_page, + bloom_filter_options=bloom_filter_options, use_content_defined_chunking=use_content_defined_chunking, **options) self.is_open = True From 587a8ff4bf326e525f47b35a2a1a5843b151821a Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Tue, 22 Sep 2026 15:40:46 +0200 Subject: [PATCH 2/2] Add tests --- python/pyarrow/tests/parquet/test_dataset.py | 26 ++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/python/pyarrow/tests/parquet/test_dataset.py b/python/pyarrow/tests/parquet/test_dataset.py index 88603d9b5b8..a124c7e74e9 100644 --- a/python/pyarrow/tests/parquet/test_dataset.py +++ b/python/pyarrow/tests/parquet/test_dataset.py @@ -1312,6 +1312,32 @@ def file_visitor(written_file): assert paths_written_set == expected_paths +def test_write_table_options_in_make_write_options(): + import pyarrow.dataset as ds + + not_writer_args = {"table", "where", "row_group_size", "filesystem", "flavor"} + options = { + name: parameter.default + for name, parameter in inspect.signature(pq.write_table).parameters.items() + if name not in not_writer_args + and parameter.kind != inspect.Parameter.VAR_KEYWORD + } + ds.ParquetFileFormat().make_write_options(**options) + + +def test_write_to_dataset_options(tempdir): + table = pa.table({"a": [1, 2, 3], + "t": pa.array([1, 2, 3], pa.time32("ms"))}) + pq.write_to_dataset(table, tempdir, store_schema=False, + write_time_adjusted_to_utc=True, + bloom_filter_options={"a": True}, + ) + metadata = pq.read_metadata(next(tempdir.glob("*.parquet"))) + assert b'ARROW:schema' not in (metadata.metadata or {}) + assert 'isAdjustedToUTC=true' in str(metadata.schema.column(1).logical_type) + assert metadata.row_group(0).column(0).bloom_filter_offset is not None + + @pytest.mark.parametrize("write_dataset_kwarg", ( ("create_dir", True), ("create_dir", False),