diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py index 6f837d21861d..b75c9d57471e 100644 --- a/python/pyarrow/parquet/core.py +++ b/python/pyarrow/parquet/core.py @@ -205,115 +205,151 @@ def convert_single_predicate(col, op, val): # Reading a single Parquet file +# Parameter descriptions shared by the Parquet reader docstrings +# (ParquetFile, ParquetDataset, read_table and read_pandas) + +_read_docstring_file_options = """\ +read_dictionary : list, default None + List of names or column paths (for nested types) to read directly + as DictionaryArray. Only supported for BYTE_ARRAY storage. To read + a flat column as dictionary-encoded pass the column name. For + nested types, you must pass the full column "path", which could be + something like level1.level2.list.item. Refer to the Parquet + file's schema to obtain the paths. +binary_type : pyarrow.DataType, default None + If given, Parquet binary columns will be read as this datatype. + This setting is ignored if a serialized Arrow schema is found in + the Parquet metadata. +list_type : subclass of pyarrow.DataType, default None + If given, non-MAP repeated columns will be read as an instance of + this datatype (either pyarrow.ListType or pyarrow.LargeListType). + This setting is ignored if a serialized Arrow schema is found in + the Parquet metadata. +memory_map : bool, default False + If the source is a file path, use a memory map to read file, which can + improve performance in some environments. +buffer_size : int, default 0 + If positive, perform read buffering when deserializing individual + column chunks. Otherwise IO calls are unbuffered.""" + +_read_docstring_partitioning = """\ +partitioning : pyarrow.dataset.Partitioning or str or list of str or None, \ +default "hive" + The partitioning scheme for a partitioned dataset. The default of "hive" + assumes directory names with key=value pairs like "/year=2009/month=11". + In addition, a scheme like "/2009/11" is also supported, in which case + you need to specify the field names or a full schema. If no partitioning + is used, pass ``None``. + See the ``pyarrow.dataset.partitioning()`` function for more details.""" + +_read_docstring_common = "\n".join((_read_docstring_file_options, + _read_docstring_partitioning)) + +_read_docstring_filesystem = """\ +filesystem : FileSystem, default None + If nothing passed, will be inferred based on path. + Path will try to be found in the local on-disk filesystem otherwise + it will be parsed as an URI to determine the filesystem.""" + +_read_docstring_reader_options = """\ +pre_buffer : bool, default True + Coalesce and issue file reads in parallel to improve performance on + high-latency filesystems (e.g. S3, GCS). If True, Arrow will use a + background I/O thread pool. If using a filesystem layer that itself + performs readahead (e.g. fsspec's S3FS), disable readahead for best + results. Set to False if you want to prioritize minimal memory usage + over maximum speed. +coerce_int96_timestamp_unit : str, default None + Cast timestamps that are stored in INT96 format to a particular + resolution (e.g. 'ms'). Setting to None is equivalent to 'ns' + and therefore INT96 timestamps will be inferred as timestamps + in nanoseconds. +decryption_properties : FileDecryptionProperties or None + File-level decryption properties for Parquet Modular Encryption. + The decryption properties can be created using + ``CryptoFactory.file_decryption_properties()``. +thrift_string_size_limit : int, default None + If not None, override the maximum total string size allocated + when decoding Thrift structures. The default limit should be + sufficient for most Parquet files. +thrift_container_size_limit : int, default None + If not None, override the maximum total size of containers allocated + when decoding Thrift structures. The default limit should be + sufficient for most Parquet files. +schema_depth_limit : int, default None + If not None, override the maximum nesting depth of the Parquet file schema. + This guards against recursion overflow on invalid schemas. + The default limit should be sufficient for most Parquet files.""" + +_read_docstring_checksum_options = """\ +page_checksum_verification : bool, default False + If True, verify the checksum for each page read from the file. +arrow_extensions_enabled : bool, default True + If True, read Parquet logical types as Arrow extension types where + possible (e.g., read JSON as the canonical `arrow.json` extension type + or UUID as the canonical `arrow.uuid` extension type).""" + + class ParquetFile: - """ - Reader interface for a single Parquet file. + __doc__ = f""" +Reader interface for a single Parquet file. - Parameters - ---------- - source : str, pathlib.Path, pyarrow.NativeFile, or file-like object - Readable source. For passing bytes or buffer-like file containing a - Parquet file, use pyarrow.BufferReader. - metadata : FileMetaData, default None - Use existing metadata object, rather than reading from file. - common_metadata : FileMetaData, default None - Will be used in reads for pandas schema metadata if not found in the - main file's metadata, no other uses at the moment. - read_dictionary : list - List of column names to read directly as DictionaryArray. - binary_type : pyarrow.DataType, default None - If given, Parquet binary columns will be read as this datatype. - This setting is ignored if a serialized Arrow schema is found in - the Parquet metadata. - list_type : subclass of pyarrow.DataType, default None - If given, non-MAP repeated columns will be read as an instance of - this datatype (either pyarrow.ListType or pyarrow.LargeListType). - This setting is ignored if a serialized Arrow schema is found in - the Parquet metadata. - memory_map : bool, default False - If the source is a file path, use a memory map to read file, which can - improve performance in some environments. - buffer_size : int, default 0 - If positive, perform read buffering when deserializing individual - column chunks. Otherwise IO calls are unbuffered. - pre_buffer : bool, default True - Coalesce and issue file reads in parallel to improve performance on - high-latency filesystems (e.g. S3, GCS). If True, Arrow will use a - background I/O thread pool. If using a filesystem layer that itself - performs readahead (e.g. fsspec's S3FS), disable readahead for best - results. Set to False if you want to prioritize minimal memory usage - over maximum speed. - coerce_int96_timestamp_unit : str, default None - Cast timestamps that are stored in INT96 format to a particular - resolution (e.g. 'ms'). Setting to None is equivalent to 'ns' - and therefore INT96 timestamps will be inferred as timestamps - in nanoseconds. - decryption_properties : FileDecryptionProperties, default None - File decryption properties for Parquet Modular Encryption. - thrift_string_size_limit : int, default None - If not None, override the maximum total string size allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. - thrift_container_size_limit : int, default None - If not None, override the maximum total size of containers allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. - schema_depth_limit : int, default None - If not None, override the maximum nesting depth of the Parquet file schema. - This guards against recursion overflow on invalid schemas. - The default limit should be sufficient for most Parquet files. - filesystem : FileSystem, default None - If nothing passed, will be inferred based on path. - Path will try to be found in the local on-disk filesystem otherwise - it will be parsed as an URI to determine the filesystem. - page_checksum_verification : bool, default False - If True, verify the checksum for each page read from the file. - arrow_extensions_enabled : bool, default True - If True, read Parquet logical types as Arrow extension types where - possible (e.g., read JSON as the canonical `arrow.json` extension type - or UUID as the canonical `arrow.uuid` extension type). +Parameters +---------- +source : str, pathlib.Path, pyarrow.NativeFile, or file-like object + Readable source. For passing bytes or buffer-like file containing a + Parquet file, use pyarrow.BufferReader. +metadata : FileMetaData, default None + Use existing metadata object, rather than reading from file. +common_metadata : FileMetaData, default None + Will be used in reads for pandas schema metadata if not found in the + main file's metadata, no other uses at the moment. +{_read_docstring_file_options} +{_read_docstring_reader_options} +{_read_docstring_filesystem} +{_read_docstring_checksum_options} - Examples - -------- +Examples +-------- - Generate an example PyArrow Table and write it to Parquet file: +Generate an example PyArrow Table and write it to Parquet file: - >>> import pyarrow as pa - >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], - ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", - ... "Brittle stars", "Centipede"]}) +>>> import pyarrow as pa +>>> table = pa.table({{'n_legs': [2, 2, 4, 4, 5, 100], +... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", +... "Brittle stars", "Centipede"]}}) - >>> import pyarrow.parquet as pq - >>> pq.write_table(table, 'example.parquet') +>>> import pyarrow.parquet as pq +>>> pq.write_table(table, 'example.parquet') - Create a ``ParquetFile`` object from the Parquet file: +Create a ``ParquetFile`` object from the Parquet file: - >>> parquet_file = pq.ParquetFile('example.parquet') +>>> parquet_file = pq.ParquetFile('example.parquet') - Read the data: +Read the data: - >>> parquet_file.read() - pyarrow.Table - n_legs: int64 - animal: string - ---- - n_legs: [[2,2,4,4,5,100]] - animal: [["Flamingo","Parrot","Dog","Horse","Brittle stars","Centipede"]] +>>> parquet_file.read() +pyarrow.Table +n_legs: int64 +animal: string +---- +n_legs: [[2,2,4,4,5,100]] +animal: [["Flamingo","Parrot","Dog","Horse","Brittle stars","Centipede"]] - Create a ParquetFile object with "animal" column as DictionaryArray: +Create a ParquetFile object with "animal" column as DictionaryArray: - >>> parquet_file = pq.ParquetFile('example.parquet', - ... read_dictionary=["animal"]) - >>> parquet_file.read() - pyarrow.Table - n_legs: int64 - animal: dictionary - ---- - n_legs: [[2,2,4,4,5,100]] - animal: [ -- dictionary: - ["Flamingo","Parrot",...,"Brittle stars","Centipede"] -- indices: - [0,1,2,3,4,5]] - """ +>>> parquet_file = pq.ParquetFile('example.parquet', +... read_dictionary=["animal"]) +>>> parquet_file.read() +pyarrow.Table +n_legs: int64 +animal: dictionary +---- +n_legs: [[2,2,4,4,5,100]] +animal: [ -- dictionary: +["Flamingo","Parrot",...,"Brittle stars","Centipede"] -- indices: +[0,1,2,3,4,5]] +""" def __init__(self, source, *, metadata=None, common_metadata=None, read_dictionary=None, binary_type=None, list_type=None, @@ -1256,39 +1292,6 @@ def _get_pandas_index_columns(keyvalues): EXCLUDED_PARQUET_PATHS = {'_SUCCESS'} -_read_docstring_common = """\ -read_dictionary : list, default None - List of names or column paths (for nested types) to read directly - as DictionaryArray. Only supported for BYTE_ARRAY storage. To read - a flat column as dictionary-encoded pass the column name. For - nested types, you must pass the full column "path", which could be - something like level1.level2.list.item. Refer to the Parquet - file's schema to obtain the paths. -binary_type : pyarrow.DataType, default None - If given, Parquet binary columns will be read as this datatype. - This setting is ignored if a serialized Arrow schema is found in - the Parquet metadata. -list_type : subclass of pyarrow.DataType, default None - If given, non-MAP repeated columns will be read as an instance of - this datatype (either pyarrow.ListType or pyarrow.LargeListType). - This setting is ignored if a serialized Arrow schema is found in - the Parquet metadata. -memory_map : bool, default False - If the source is a file path, use a memory map to read file, which can - improve performance in some environments. -buffer_size : int, default 0 - If positive, perform read buffering when deserializing individual - column chunks. Otherwise IO calls are unbuffered. -partitioning : pyarrow.dataset.Partitioning or str or list of str or None, \ -default "hive" - The partitioning scheme for a partitioned dataset. The default of "hive" - assumes directory names with key=value pairs like "/year=2009/month=11". - In addition, a scheme like "/2009/11" is also supported, in which case - you need to specify the field names or a full schema. If no partitioning - is used, pass ``None``. - See the ``pyarrow.dataset.partitioning()`` function for more details.""" - - _parquet_dataset_example = """\ Generate an example PyArrow Table and write it to a partitioned dataset: @@ -1336,10 +1339,7 @@ class ParquetDataset: ---------- path_or_paths : str or List[str] A directory name, single file name, or list of file names. -filesystem : FileSystem, default None - If nothing passed, will be inferred based on path. - Path will try to be found in the local on-disk filesystem otherwise - it will be parsed as an URI to determine the filesystem. +{_read_docstring_filesystem} schema : pyarrow.parquet.Schema Optionally provide the Schema for the Dataset, in which case it will not be inferred from the source. @@ -1357,39 +1357,8 @@ class ParquetDataset: This is matched to the basename of a path. By default this is ['.', '_']. Note that discovery happens only if a directory is passed as source. -pre_buffer : bool, default True - Coalesce and issue file reads in parallel to improve performance on - high-latency filesystems (e.g. S3, GCS). If True, Arrow will use a - background I/O thread pool. If using a filesystem layer that itself - performs readahead (e.g. fsspec's S3FS), disable readahead for best - results. Set to False if you want to prioritize minimal memory usage - over maximum speed. -coerce_int96_timestamp_unit : str, default None - Cast timestamps that are stored in INT96 format to a particular resolution - (e.g. 'ms'). Setting to None is equivalent to 'ns' and therefore INT96 - timestamps will be inferred as timestamps in nanoseconds. -decryption_properties : FileDecryptionProperties or None - File-level decryption properties. - The decryption properties can be created using - ``CryptoFactory.file_decryption_properties()``. -thrift_string_size_limit : int, default None - If not None, override the maximum total string size allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. -thrift_container_size_limit : int, default None - If not None, override the maximum total size of containers allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. -schema_depth_limit : int, default None - If not None, override the maximum nesting depth of the Parquet file schema. - This guards against recursion overflow on invalid schemas. - The default limit should be sufficient for most Parquet files. -page_checksum_verification : bool, default False - If True, verify the page checksum for each page read from the file. -arrow_extensions_enabled : bool, default True - If True, read Parquet logical types as Arrow extension types where possible, - (e.g., read JSON as the canonical `arrow.json` extension type or UUID as - the canonical `arrow.uuid` extension type). +{_read_docstring_reader_options} +{_read_docstring_checksum_options} Examples -------- @@ -1761,10 +1730,7 @@ def partitioning(self): Optionally provide the Schema for the parquet dataset, in which case it will not be inferred from the source. {1} -filesystem : FileSystem, default None - If nothing passed, will be inferred based on path. - Path will try to be found in the local on-disk filesystem otherwise - it will be parsed as an URI to determine the filesystem. +{7} filters : pyarrow.compute.Expression or List[Tuple] or List[List[Tuple]], default None Rows which do not match the filter predicate will be removed from scanned data. Partition keys embedded in a nested directory structure will be @@ -1778,39 +1744,8 @@ def partitioning(self): This is matched to the basename of a path. By default this is ['.', '_']. Note that discovery happens only if a directory is passed as source. -pre_buffer : bool, default True - Coalesce and issue file reads in parallel to improve performance on - high-latency filesystems (e.g. S3). If True, Arrow will use a - background I/O thread pool. If using a filesystem layer that itself - performs readahead (e.g. fsspec's S3FS), disable readahead for best - results. -coerce_int96_timestamp_unit : str, default None - Cast timestamps that are stored in INT96 format to a particular - resolution (e.g. 'ms'). Setting to None is equivalent to 'ns' - and therefore INT96 timestamps will be inferred as timestamps - in nanoseconds. -decryption_properties : FileDecryptionProperties or None - File-level decryption properties. - The decryption properties can be created using - ``CryptoFactory.file_decryption_properties()``. -thrift_string_size_limit : int, default None - If not None, override the maximum total string size allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. -thrift_container_size_limit : int, default None - If not None, override the maximum total size of containers allocated - when decoding Thrift structures. The default limit should be - sufficient for most Parquet files. -schema_depth_limit : int, default None - If not None, override the maximum nesting depth of the Parquet file schema. - This guards against recursion overflow on invalid schemas. - The default limit should be sufficient for most Parquet files. -page_checksum_verification : bool, default False - If True, verify the checksum for each page read from the file. -arrow_extensions_enabled : bool, default True - If True, read Parquet logical types as Arrow extension types where possible, - (e.g., read JSON as the canonical `arrow.json` extension type or UUID as - the canonical `arrow.uuid` extension type). +{5} +{6} Returns ------- @@ -1991,7 +1926,8 @@ def read_table(source, *, columns=None, use_threads=True, index columns are also loaded.""", _read_docstring_common)), """pyarrow.Table Content of the file as a table (of columns)""", - _DNF_filter_doc, _read_table_example) + _DNF_filter_doc, _read_table_example, _read_docstring_reader_options, + _read_docstring_checksum_options, _read_docstring_filesystem) def read_pandas(source, columns=None, **kwargs): @@ -2009,7 +1945,8 @@ def read_pandas(source, columns=None, **kwargs): """pyarrow.Table Content of the file as a Table of Columns, including DataFrame indexes as columns""", - _DNF_filter_doc, "") + _DNF_filter_doc, "", _read_docstring_reader_options, + _read_docstring_checksum_options, _read_docstring_filesystem) def write_table(table, where, row_group_size=None, version='2.6',