From 876597b495e343094f3ef0c4e6bf59ace69a9126 Mon Sep 17 00:00:00 2001 From: MarkCruz911 <85602094+MarkCruz911@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:36:21 +0000 Subject: [PATCH 1/2] GH-51265: [Python] Refactor and centralize duplicated reader docstrings in parquet/core.py --- python/pyarrow/parquet/core.py | 58 +++++++++++----------------------- 1 file changed, 19 insertions(+), 39 deletions(-) diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py index 6f837d21861d..37e7f7566a59 100644 --- a/python/pyarrow/parquet/core.py +++ b/python/pyarrow/parquet/core.py @@ -28,6 +28,19 @@ import pyarrow as pa +_READER_COMMON_DOCS = """ thrift_string_size_limit : int, default None + If not None, override the maximum total string size allocated + when decoding Thrift structures. The default limit should be + sufficient for most Parquet files. + thrift_container_size_limit : int, default None + If not None, override the maximum total size of containers allocated + when decoding Thrift structures. The default limit should be + sufficient for most Parquet files. + schema_depth_limit : int, default None + If not None, override the maximum nesting depth of the Parquet file schema. + This guards against recursion overflow on invalid schemas. + The default limit should be sufficient for most Parquet files.""" + try: import pyarrow._parquet as _parquet except ImportError as exc: @@ -206,7 +219,7 @@ def convert_single_predicate(col, op, val): class ParquetFile: - """ + f""" Reader interface for a single Parquet file. Parameters @@ -250,18 +263,7 @@ class ParquetFile: in nanoseconds. decryption_properties : FileDecryptionProperties, default None File decryption properties for Parquet Modular Encryption. - thrift_string_size_limit : int, default None - If not None, override the maximum total string size allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. - thrift_container_size_limit : int, default None - If not None, override the maximum total size of containers allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. - schema_depth_limit : int, default None - If not None, override the maximum nesting depth of the Parquet file schema. - This guards against recursion overflow on invalid schemas. - The default limit should be sufficient for most Parquet files. + {_READER_COMMON_DOCS} filesystem : FileSystem, default None If nothing passed, will be inferred based on path. Path will try to be found in the local on-disk filesystem otherwise @@ -1372,18 +1374,7 @@ class ParquetDataset: File-level decryption properties. The decryption properties can be created using ``CryptoFactory.file_decryption_properties()``. -thrift_string_size_limit : int, default None - If not None, override the maximum total string size allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. -thrift_container_size_limit : int, default None - If not None, override the maximum total size of containers allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. -schema_depth_limit : int, default None - If not None, override the maximum nesting depth of the Parquet file schema. - This guards against recursion overflow on invalid schemas. - The default limit should be sufficient for most Parquet files. +{_READER_COMMON_DOCS} page_checksum_verification : bool, default False If True, verify the page checksum for each page read from the file. arrow_extensions_enabled : bool, default True @@ -1736,10 +1727,10 @@ def partitioning(self): """ The partitioning of the Dataset source, if discovered. """ - return self._dataset.partitioning + return self._dataset.partitioning -_read_table_docstring = """ +_read_table_docstring = f""" {0} Parameters @@ -1793,18 +1784,7 @@ def partitioning(self): File-level decryption properties. The decryption properties can be created using ``CryptoFactory.file_decryption_properties()``. -thrift_string_size_limit : int, default None - If not None, override the maximum total string size allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. -thrift_container_size_limit : int, default None - If not None, override the maximum total size of containers allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. -schema_depth_limit : int, default None - If not None, override the maximum nesting depth of the Parquet file schema. - This guards against recursion overflow on invalid schemas. - The default limit should be sufficient for most Parquet files. +{_READER_COMMON_DOCS} page_checksum_verification : bool, default False If True, verify the checksum for each page read from the file. arrow_extensions_enabled : bool, default True From 7789a60419b85b435a8b8a346ff1edc22fff7a60 Mon Sep 17 00:00:00 2001 From: MarkCruz911 <85602094+MarkCruz911@users.noreply.github.com> Date: Sat, 17 Oct 2026 16:49:53 -0400 Subject: [PATCH 2/2] Reuse shared Parquet reader docstrings and fix docstring formatting --- python/pyarrow/parquet/core.py | 121 +++++++++++++++++---------------- 1 file changed, 61 insertions(+), 60 deletions(-) diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py index 37e7f7566a59..50e3d095a1e0 100644 --- a/python/pyarrow/parquet/core.py +++ b/python/pyarrow/parquet/core.py @@ -28,18 +28,52 @@ import pyarrow as pa -_READER_COMMON_DOCS = """ thrift_string_size_limit : int, default None - If not None, override the maximum total string size allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. - thrift_container_size_limit : int, default None - If not None, override the maximum total size of containers allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. - schema_depth_limit : int, default None - If not None, override the maximum nesting depth of the Parquet file schema. - This guards against recursion overflow on invalid schemas. - The default limit should be sufficient for most Parquet files.""" +_read_docstring_common = { + "dataset": """\ +read_dictionary : list, default None + List of names or column paths (for nested types) to read directly + as DictionaryArray. Only supported for BYTE_ARRAY storage. To read + a flat column as dictionary-encoded pass the column name. For + nested types, you must pass the full column "path", which could be + something like level1.level2.list.item. Refer to the Parquet + file's schema to obtain the paths. +binary_type : pyarrow.DataType, default None + If given, Parquet binary columns will be read as this datatype. + This setting is ignored if a serialized Arrow schema is found in + the Parquet metadata. +list_type : subclass of pyarrow.DataType, default None + If given, non-MAP repeated columns will be read as an instance of + this datatype (either pyarrow.ListType or pyarrow.LargeListType). + This setting is ignored if a serialized Arrow schema is found in + the Parquet metadata. +memory_map : bool, default False + If the source is a file path, use a memory map to read file, which can + improve performance in some environments. +buffer_size : int, default 0 + If positive, perform read buffering when deserializing individual + column chunks. Otherwise IO calls are unbuffered. +partitioning : pyarrow.dataset.Partitioning or str or list of str or None, \ +default "hive" + The partitioning scheme for a partitioned dataset. The default of "hive" + assumes directory names with key=value pairs like "/year=2009/month=11". + In addition, a scheme like "/2009/11" is also supported, in which case + you need to specify the field names or a full schema. If no partitioning + is used, pass ``None``. + See the ``pyarrow.dataset.partitioning()`` function for more details.""", + "limits": """\ +thrift_string_size_limit : int, default None + If not None, override the maximum total string size allocated + when decoding Thrift structures. The default limit should be + sufficient for most Parquet files. +thrift_container_size_limit : int, default None + If not None, override the maximum total size of containers allocated + when decoding Thrift structures. The default limit should be + sufficient for most Parquet files. +schema_depth_limit : int, default None + If not None, override the maximum nesting depth of the Parquet file schema. + This guards against recursion overflow on invalid schemas. + The default limit should be sufficient for most Parquet files.""", +} try: import pyarrow._parquet as _parquet @@ -219,7 +253,7 @@ def convert_single_predicate(col, op, val): class ParquetFile: - f""" + __doc__ = f""" Reader interface for a single Parquet file. Parameters @@ -263,7 +297,7 @@ class ParquetFile: in nanoseconds. decryption_properties : FileDecryptionProperties, default None File decryption properties for Parquet Modular Encryption. - {_READER_COMMON_DOCS} + {_read_docstring_common["limits"].replace(chr(10), chr(10) + " ")} filesystem : FileSystem, default None If nothing passed, will be inferred based on path. Path will try to be found in the local on-disk filesystem otherwise @@ -281,9 +315,9 @@ class ParquetFile: Generate an example PyArrow Table and write it to Parquet file: >>> import pyarrow as pa - >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], + >>> table = pa.table({{'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", - ... "Brittle stars", "Centipede"]}) + ... "Brittle stars", "Centipede"]}}) >>> import pyarrow.parquet as pq >>> pq.write_table(table, 'example.parquet') @@ -1258,39 +1292,6 @@ def _get_pandas_index_columns(keyvalues): EXCLUDED_PARQUET_PATHS = {'_SUCCESS'} -_read_docstring_common = """\ -read_dictionary : list, default None - List of names or column paths (for nested types) to read directly - as DictionaryArray. Only supported for BYTE_ARRAY storage. To read - a flat column as dictionary-encoded pass the column name. For - nested types, you must pass the full column "path", which could be - something like level1.level2.list.item. Refer to the Parquet - file's schema to obtain the paths. -binary_type : pyarrow.DataType, default None - If given, Parquet binary columns will be read as this datatype. - This setting is ignored if a serialized Arrow schema is found in - the Parquet metadata. -list_type : subclass of pyarrow.DataType, default None - If given, non-MAP repeated columns will be read as an instance of - this datatype (either pyarrow.ListType or pyarrow.LargeListType). - This setting is ignored if a serialized Arrow schema is found in - the Parquet metadata. -memory_map : bool, default False - If the source is a file path, use a memory map to read file, which can - improve performance in some environments. -buffer_size : int, default 0 - If positive, perform read buffering when deserializing individual - column chunks. Otherwise IO calls are unbuffered. -partitioning : pyarrow.dataset.Partitioning or str or list of str or None, \ -default "hive" - The partitioning scheme for a partitioned dataset. The default of "hive" - assumes directory names with key=value pairs like "/year=2009/month=11". - In addition, a scheme like "/2009/11" is also supported, in which case - you need to specify the field names or a full schema. If no partitioning - is used, pass ``None``. - See the ``pyarrow.dataset.partitioning()`` function for more details.""" - - _parquet_dataset_example = """\ Generate an example PyArrow Table and write it to a partitioned dataset: @@ -1352,7 +1353,7 @@ class ParquetDataset: Within-file level filtering and different partitioning schemes are supported. {_DNF_filter_doc} -{_read_docstring_common} +{_read_docstring_common["dataset"]} ignore_prefixes : list, optional Files matching any of these prefixes will be ignored by the discovery process. @@ -1374,7 +1375,7 @@ class ParquetDataset: File-level decryption properties. The decryption properties can be created using ``CryptoFactory.file_decryption_properties()``. -{_READER_COMMON_DOCS} +{_read_docstring_common["limits"]} page_checksum_verification : bool, default False If True, verify the page checksum for each page read from the file. arrow_extensions_enabled : bool, default True @@ -1727,11 +1728,11 @@ def partitioning(self): """ The partitioning of the Dataset source, if discovered. """ - return self._dataset.partitioning + return self._dataset.partitioning _read_table_docstring = f""" -{0} +{{0}} Parameters ---------- @@ -1751,7 +1752,7 @@ def partitioning(self): schema : Schema, optional Optionally provide the Schema for the parquet dataset, in which case it will not be inferred from the source. -{1} +{{1}} filesystem : FileSystem, default None If nothing passed, will be inferred based on path. Path will try to be found in the local on-disk filesystem otherwise @@ -1762,7 +1763,7 @@ def partitioning(self): exploited to avoid loading files at all if they contain no matching rows. Within-file level filtering and different partitioning schemes are supported. - {3} + {{3}} ignore_prefixes : list, optional Files matching any of these prefixes will be ignored by the discovery process. @@ -1784,7 +1785,7 @@ def partitioning(self): File-level decryption properties. The decryption properties can be created using ``CryptoFactory.file_decryption_properties()``. -{_READER_COMMON_DOCS} +{_read_docstring_common["limits"]} page_checksum_verification : bool, default False If True, verify the checksum for each page read from the file. arrow_extensions_enabled : bool, default True @@ -1794,9 +1795,9 @@ def partitioning(self): Returns ------- -{2} +{{2}} -{4} +{{4}} """ _read_table_example = """\ @@ -1968,7 +1969,7 @@ def read_table(source, *, columns=None, use_threads=True, """Read a Table from Parquet format""", "\n".join(("""use_pandas_metadata : bool, default False If True and file has custom pandas schema metadata, ensure that - index columns are also loaded.""", _read_docstring_common)), + index columns are also loaded.""", _read_docstring_common["dataset"])), """pyarrow.Table Content of the file as a table (of columns)""", _DNF_filter_doc, _read_table_example) @@ -1983,7 +1984,7 @@ def read_pandas(source, columns=None, **kwargs): read_pandas.__doc__ = _read_table_docstring.format( 'Read a Table from Parquet format, also reading DataFrame\n' 'index values if known in the file metadata', - "\n".join((_read_docstring_common, + "\n".join((_read_docstring_common["dataset"], """**kwargs additional options for :func:`read_table`""")), """pyarrow.Table