diff --git a/python/pyarrow-stubs/pyarrow/__init__.pyi b/python/pyarrow-stubs/pyarrow/__init__.pyi index ccec8d5abc07..6dca7576eb02 100644 --- a/python/pyarrow-stubs/pyarrow/__init__.pyi +++ b/python/pyarrow-stubs/pyarrow/__init__.pyi @@ -15,15 +15,318 @@ # specific language governing permissions and limitations # under the License. -"""Type stubs for PyArrow. +from pyarrow.ipc import ( + Message as Message, + MessageReader as MessageReader, + MetadataVersion as MetadataVersion, + RecordBatchFileReader as RecordBatchFileReader, + RecordBatchFileWriter as RecordBatchFileWriter, + RecordBatchStreamReader as RecordBatchStreamReader, + RecordBatchStreamWriter as RecordBatchStreamWriter, + deserialize_pandas as deserialize_pandas, + serialize_pandas as serialize_pandas, +) +from pyarrow.lib import ( + Array as Array, + ArrowCancelled as ArrowCancelled, + ArrowCapacityError as ArrowCapacityError, + ArrowException as ArrowException, + ArrowIOError as ArrowIOError, + ArrowIndexError as ArrowIndexError, + ArrowInvalid as ArrowInvalid, + ArrowKeyError as ArrowKeyError, + ArrowMemoryError as ArrowMemoryError, + ArrowNotImplementedError as ArrowNotImplementedError, + ArrowSerializationError as ArrowSerializationError, + ArrowTypeError as ArrowTypeError, + BaseExtensionType as BaseExtensionType, + BinaryArray as BinaryArray, + BinaryScalar as BinaryScalar, + BinaryViewArray as BinaryViewArray, + BinaryViewScalar as BinaryViewScalar, + Bool8Array as Bool8Array, + Bool8Scalar as Bool8Scalar, + Bool8Type as Bool8Type, + BooleanArray as BooleanArray, + BooleanScalar as BooleanScalar, + Buffer as Buffer, + BufferOutputStream as BufferOutputStream, + BufferReader as BufferReader, + BufferedInputStream as BufferedInputStream, + BufferedOutputStream as BufferedOutputStream, + BuildInfo as BuildInfo, + CacheOptions as CacheOptions, + ChunkedArray as ChunkedArray, + Codec as Codec, + CompressedInputStream as CompressedInputStream, + CompressedOutputStream as CompressedOutputStream, + CppBuildInfo as CppBuildInfo, + DataType as DataType, + Date32Array as Date32Array, + Date32Scalar as Date32Scalar, + Date64Array as Date64Array, + Date64Scalar as Date64Scalar, + Decimal128Array as Decimal128Array, + Decimal128Scalar as Decimal128Scalar, + Decimal128Type as Decimal128Type, + Decimal256Array as Decimal256Array, + Decimal256Scalar as Decimal256Scalar, + Decimal256Type as Decimal256Type, + Decimal32Array as Decimal32Array, + Decimal32Scalar as Decimal32Scalar, + Decimal32Type as Decimal32Type, + Decimal64Array as Decimal64Array, + Decimal64Scalar as Decimal64Scalar, + Decimal64Type as Decimal64Type, + DenseUnionType as DenseUnionType, + Device as Device, + DeviceAllocationType as DeviceAllocationType, + DictionaryArray as DictionaryArray, + DictionaryMemo as DictionaryMemo, + DictionaryScalar as DictionaryScalar, + DictionaryType as DictionaryType, + DoubleArray as DoubleArray, + DoubleScalar as DoubleScalar, + DurationArray as DurationArray, + DurationScalar as DurationScalar, + DurationType as DurationType, + ExtensionArray as ExtensionArray, + ExtensionScalar as ExtensionScalar, + ExtensionType as ExtensionType, + Field as Field, + FixedShapeTensorArray as FixedShapeTensorArray, + FixedShapeTensorScalar as FixedShapeTensorScalar, + FixedShapeTensorType as FixedShapeTensorType, + FixedSizeBinaryArray as FixedSizeBinaryArray, + FixedSizeBinaryScalar as FixedSizeBinaryScalar, + FixedSizeBinaryType as FixedSizeBinaryType, + FixedSizeBufferWriter as FixedSizeBufferWriter, + FixedSizeListArray as FixedSizeListArray, + FixedSizeListScalar as FixedSizeListScalar, + FixedSizeListType as FixedSizeListType, + FloatArray as FloatArray, + FloatScalar as FloatScalar, + FloatingPointArray as FloatingPointArray, + HalfFloatArray as HalfFloatArray, + HalfFloatScalar as HalfFloatScalar, + Int16Array as Int16Array, + Int16Scalar as Int16Scalar, + Int32Array as Int32Array, + Int32Scalar as Int32Scalar, + Int64Array as Int64Array, + Int64Scalar as Int64Scalar, + Int8Array as Int8Array, + Int8Scalar as Int8Scalar, + IntegerArray as IntegerArray, + JsonArray as JsonArray, + JsonScalar as JsonScalar, + JsonType as JsonType, + KeyValueMetadata as KeyValueMetadata, + LargeBinaryArray as LargeBinaryArray, + LargeBinaryScalar as LargeBinaryScalar, + LargeListArray as LargeListArray, + LargeListScalar as LargeListScalar, + LargeListType as LargeListType, + LargeListViewArray as LargeListViewArray, + LargeListViewScalar as LargeListViewScalar, + LargeListViewType as LargeListViewType, + LargeStringArray as LargeStringArray, + LargeStringScalar as LargeStringScalar, + ListArray as ListArray, + ListScalar as ListScalar, + ListType as ListType, + ListViewArray as ListViewArray, + ListViewScalar as ListViewScalar, + ListViewType as ListViewType, + LoggingMemoryPool as LoggingMemoryPool, + MapArray as MapArray, + MapScalar as MapScalar, + MapType as MapType, + MemoryManager as MemoryManager, + MemoryMappedFile as MemoryMappedFile, + MemoryPool as MemoryPool, + MockOutputStream as MockOutputStream, + MonthDayNano as MonthDayNano, + MonthDayNanoIntervalArray as MonthDayNanoIntervalArray, + MonthDayNanoIntervalScalar as MonthDayNanoIntervalScalar, + NA as NA, + NativeFile as NativeFile, + NullArray as NullArray, + NullScalar as NullScalar, + NumericArray as NumericArray, + OSFile as OSFile, + OpaqueArray as OpaqueArray, + OpaqueScalar as OpaqueScalar, + OpaqueType as OpaqueType, + ProxyMemoryPool as ProxyMemoryPool, + PythonFile as PythonFile, + RecordBatch as RecordBatch, + RecordBatchReader as RecordBatchReader, + ResizableBuffer as ResizableBuffer, + RunEndEncodedArray as RunEndEncodedArray, + RunEndEncodedScalar as RunEndEncodedScalar, + RunEndEncodedType as RunEndEncodedType, + RuntimeInfo as RuntimeInfo, + Scalar as Scalar, + Schema as Schema, + SparseCOOTensor as SparseCOOTensor, + SparseCSCMatrix as SparseCSCMatrix, + SparseCSFTensor as SparseCSFTensor, + SparseCSRMatrix as SparseCSRMatrix, + SparseUnionType as SparseUnionType, + StringArray as StringArray, + StringScalar as StringScalar, + StringViewArray as StringViewArray, + StringViewScalar as StringViewScalar, + StructArray as StructArray, + StructScalar as StructScalar, + StructType as StructType, + Table as Table, + TableGroupBy as TableGroupBy, + Tensor as Tensor, + Time32Array as Time32Array, + Time32Scalar as Time32Scalar, + Time32Type as Time32Type, + Time64Array as Time64Array, + Time64Scalar as Time64Scalar, + Time64Type as Time64Type, + TimestampArray as TimestampArray, + TimestampScalar as TimestampScalar, + TimestampType as TimestampType, + TransformInputStream as TransformInputStream, + UInt16Array as UInt16Array, + UInt16Scalar as UInt16Scalar, + UInt32Array as UInt32Array, + UInt32Scalar as UInt32Scalar, + UInt64Array as UInt64Array, + UInt64Scalar as UInt64Scalar, + UInt8Array as UInt8Array, + UInt8Scalar as UInt8Scalar, + UnionArray as UnionArray, + UnionScalar as UnionScalar, + UnionType as UnionType, + UnknownExtensionType as UnknownExtensionType, + UuidArray as UuidArray, + UuidScalar as UuidScalar, + UuidType as UuidType, + VersionInfo as VersionInfo, + allocate_buffer as allocate_buffer, + arange as arange, + array as array, + binary as binary, + binary_view as binary_view, + bool8 as bool8, + bool_ as bool_, + build_info as build_info, + chunked_array as chunked_array, + compress as compress, + concat_arrays as concat_arrays, + concat_batches as concat_batches, + concat_tables as concat_tables, + cpp_build_info as cpp_build_info, + cpp_version as cpp_version, + cpp_version_info as cpp_version_info, + cpu_count as cpu_count, + create_memory_map as create_memory_map, + date32 as date32, + date64 as date64, + decimal128 as decimal128, + decimal256 as decimal256, + decimal32 as decimal32, + decimal64 as decimal64, + decompress as decompress, + default_cpu_memory_manager as default_cpu_memory_manager, + default_memory_pool as default_memory_pool, + dense_union as dense_union, + dictionary as dictionary, + duration as duration, + enable_signal_handlers as enable_signal_handlers, + field as field, + fixed_shape_tensor as fixed_shape_tensor, + float16 as float16, + float32 as float32, + float64 as float64, + foreign_buffer as foreign_buffer, + from_numpy_dtype as from_numpy_dtype, + infer_type as infer_type, + input_stream as input_stream, + int16 as int16, + int32 as int32, + int64 as int64, + int8 as int8, + io_thread_count as io_thread_count, + is_opentelemetry_enabled as is_opentelemetry_enabled, + jemalloc_memory_pool as jemalloc_memory_pool, + jemalloc_set_decay_ms as jemalloc_set_decay_ms, + json_ as json_, + large_binary as large_binary, + large_list as large_list, + large_list_view as large_list_view, + large_string as large_string, + large_utf8 as large_utf8, + list_ as list_, + list_view as list_view, + log_memory_allocations as log_memory_allocations, + logging_memory_pool as logging_memory_pool, + map_ as map_, + memory_map as memory_map, + mimalloc_memory_pool as mimalloc_memory_pool, + month_day_nano_interval as month_day_nano_interval, + null as null, + nulls as nulls, + opaque as opaque, + output_stream as output_stream, + proxy_memory_pool as proxy_memory_pool, + py_buffer as py_buffer, + record_batch as record_batch, + register_extension_type as register_extension_type, + repeat as repeat, + run_end_encoded as run_end_encoded, + runtime_info as runtime_info, + scalar as scalar, + schema as schema, + set_cpu_count as set_cpu_count, + set_io_thread_count as set_io_thread_count, + set_memory_pool as set_memory_pool, + set_timezone_db_path as set_timezone_db_path, + sparse_union as sparse_union, + string as string, + string_view as string_view, + struct as struct, + supported_memory_backends as supported_memory_backends, + system_memory_pool as system_memory_pool, + table as table, + time32 as time32, + time64 as time64, + timestamp as timestamp, + total_allocated_bytes as total_allocated_bytes, + transcoding_input_stream as transcoding_input_stream, + type_for_alias as type_for_alias, + uint16 as uint16, + uint32 as uint32, + uint64 as uint64, + uint8 as uint8, + unify_schemas as unify_schemas, + union as union, + unregister_extension_type as unregister_extension_type, + utf8 as utf8, + uuid as uuid, +) +from pyarrow.util import ( + _deprecate_api as _deprecate_api, + _deprecate_class as _deprecate_class, +) -This is a placeholder stub file. -Complete type annotations will be added in subsequent PRs. -""" - -from typing import Any - -# TODO(GH-48970): remove __getattr__ before release as this -# will annotate non-existing attributes as Any. -# https://github.com/apache/arrow/issues/48970 -def __getattr__(name: str) -> Any: ... +def parse_git(root, **kwargs): ... +def show_versions() -> None: ... +def _module_is_available(module): ... +def _filesystem_is_available(fs): ... +def have_libhdfs() -> bool: ... +def show_info() -> None: ... +def get_include() -> str: ... +def _get_pkg_config_executable(): ... +def _has_pkg_config(pkgname): ... +def _read_pkg_config_variable(pkgname, cli_args): ... +def get_libraries(): ... +def create_library_symlinks(): ... +def get_library_dirs(): ... diff --git a/python/pyarrow-stubs/pyarrow/_acero.pyi b/python/pyarrow-stubs/pyarrow/_acero.pyi new file mode 100644 index 000000000000..fb2fdb7f3b2c --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_acero.pyi @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_acero.pyx + +import pyarrow +from pyarrow.lib import _Weakrefable +from pyarrow._compute import Expression +from pyarrow.lib import * +from pyarrow.lib import RecordBatchReader, Table + +class ExecNodeOptions(_Weakrefable): + __slots__ = () + +class _TableSourceNodeOptions(ExecNodeOptions): + def _set_options(self, table: Table) -> None: ... + +class TableSourceNodeOptions(_TableSourceNodeOptions): + def __init__(self, table: Table): ... + +class _RecordBatchReaderSourceNodeOptions(ExecNodeOptions): + def _set_options(self, reader: RecordBatchReader) -> None: ... + +class RecordBatchReaderSourceNodeOptions(_RecordBatchReaderSourceNodeOptions): + def __init__(self, reader: RecordBatchReader): ... + +class _FilterNodeOptions(ExecNodeOptions): + def _set_options(self, filter_expression: Expression) -> None: ... + +class FilterNodeOptions(_FilterNodeOptions): + def __init__(self, filter_expression: Expression): ... + +class _ProjectNodeOptions(ExecNodeOptions): + def _set_options(self, expressions, names=None) -> None: ... + +class ProjectNodeOptions(_ProjectNodeOptions): + def __init__(self, expressions, names=None): ... + +class _AggregateNodeOptions(ExecNodeOptions): + def _set_options(self, aggregates, keys=None) -> None: ... + +class AggregateNodeOptions(_AggregateNodeOptions): + def __init__(self, aggregates, keys=None): ... + +class _OrderByNodeOptions(ExecNodeOptions): + def _set_options(self, sort_keys, null_placement) -> None: ... + +class OrderByNodeOptions(_OrderByNodeOptions): + def __init__(self, sort_keys=(), *, null_placement=None): ... + +class _HashJoinNodeOptions(ExecNodeOptions): + def _set_options( + self, + join_type, + left_keys, + right_keys, + left_output=None, + right_output=None, + output_suffix_for_left: str = "", + output_suffix_for_right: str = "", + filter_expression: Expression | None = None, + ) -> None: ... + +class HashJoinNodeOptions(_HashJoinNodeOptions): + def __init__( + self, + join_type, + left_keys, + right_keys, + left_output=None, + right_output=None, + output_suffix_for_left: str = "", + output_suffix_for_right: str = "", + filter_expression=None, + ): ... + +class _AsofJoinNodeOptions(ExecNodeOptions): + def _set_options(self, left_on, left_by, right_on, right_by, tolerance) -> None: ... + +class AsofJoinNodeOptions(_AsofJoinNodeOptions): + def __init__(self, left_on, left_by, right_on, right_by, tolerance): ... + +class Declaration(_Weakrefable): + def __init__(self, factory_name, options: ExecNodeOptions, inputs=None): ... + @staticmethod + def from_sequence(decls: list[Declaration]) -> Declaration: ... + def to_table(self, use_threads: bool = True) -> pyarrow.Table: ... + def to_reader(self, use_threads: bool = True) -> pyarrow.RecordBatchReader: ... diff --git a/python/pyarrow-stubs/pyarrow/_azurefs.pyi b/python/pyarrow-stubs/pyarrow/_azurefs.pyi new file mode 100644 index 000000000000..b0a8ac096fd2 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_azurefs.pyi @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_azurefs.pyx + +from pyarrow._fs import FileSystem + +class AzureFileSystem(FileSystem): + def __init__( + self, + account_name, + *, + account_key=None, + blob_storage_authority=None, + blob_storage_scheme=None, + client_id=None, + client_secret=None, + dfs_storage_authority=None, + dfs_storage_scheme=None, + sas_token=None, + tenant_id=None, + ): ... + @staticmethod + def _reconstruct(kwargs) -> AzureFileSystem: ... + def __reduce__(self) -> tuple: ... diff --git a/python/pyarrow-stubs/pyarrow/_compute.pyi b/python/pyarrow-stubs/pyarrow/_compute.pyi new file mode 100644 index 000000000000..f74ab5c045a3 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_compute.pyi @@ -0,0 +1,699 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_compute.pyx + +import pyarrow +from typing import NamedTuple +from collections.abc import Iterable +from pyarrow.lib import _Weakrefable +from _typeshed import Incomplete +from typing import Any +from collections.abc import Callable +from pyarrow.lib import * + +__pas: Incomplete +_substrait_msg: str +SUPPORTED_INPUT_ARR_TYPES: tuple[Incomplete, ...] +_deserialize: Incomplete + +def _pas(): ... +def _forbid_instantiation(klass, subclasses_instead: bool = True): ... +def function_registry(): ... +def get_function(name: str): ... +def list_functions(): ... +def call_function( + name: str, + args: list, + options=None, + memory_pool: pyarrow.MemoryPool | None = None, + length: int | None = None, +): ... +def _raise_invalid_function_option(value, description, *, exception_class=...): ... +def _skip_nulls_doc() -> str: ... +def _min_count_doc(*, default) -> str: ... +def _get_udf_context(memory_pool, batch_length): ... +def register_scalar_function( + func: Callable[..., Any], + function_name: str, + function_doc: dict, + in_types, + out_type: pyarrow.DataType, + func_registry: FunctionRegistry | None = None, +): ... +def register_vector_function( + func: Callable[..., Any], + function_name: str, + function_doc: dict, + in_types, + out_type: pyarrow.DataType, + func_registry: FunctionRegistry | None = None, +): ... +def register_aggregate_function( + func: Callable[..., Any], + function_name: str, + function_doc: dict, + in_types, + out_type: pyarrow.DataType, + func_registry: FunctionRegistry | None = None, +): ... +def register_tabular_function( + func: Callable[..., Any], + function_name: str, + function_doc: dict, + in_types, + out_type, + func_registry: FunctionRegistry | None = None, +): ... +def _register_user_defined_function( + register_func, + func: Callable[..., Any], + function_name: str, + function_doc: dict, + in_types, + out_type: pyarrow.DataType, + func_registry: FunctionRegistry | None = None, +) -> None: ... +def call_tabular_function( + function_name: str, + args: Iterable[Any] | None = None, + func_registry: FunctionRegistry | None = None, +) -> pyarrow.RecordBatchReader: ... + +class FunctionDoc(NamedTuple): + summary: Incomplete + description: Incomplete + arg_names: Incomplete + options_class: Incomplete + options_required: Incomplete + +class Kernel(_Weakrefable): + def __init__(self): ... + +class ScalarKernel(Kernel): ... +class VectorKernel(Kernel): ... +class ScalarAggregateKernel(Kernel): ... +class HashAggregateKernel(Kernel): ... + +class Function(_Weakrefable): + _kind_map: Incomplete + + def __init__(self): ... + def __reduce__(self) -> tuple: ... + @property + def name(self) -> str: ... + @property + def arity(self): ... + @property + def kind(self): ... + @property + def _doc(self): ... + @property + def num_kernels(self): ... + def call( + self, + args: Iterable[Any], + options: FunctionOptions | None = None, + memory_pool: MemoryPool | None = None, + length: int | None = None, + ): ... + +class ScalarFunction(Function): + @property + def kernels(self) -> list: ... + +class VectorFunction(Function): + @property + def kernels(self) -> list: ... + +class ScalarAggregateFunction(Function): + @property + def kernels(self) -> list: ... + +class HashAggregateFunction(Function): + @property + def kernels(self) -> list: ... + +class MetaFunction(Function): + @property + def kernels(self) -> list: ... + +class FunctionRegistry(_Weakrefable): + def __init__(self): ... + def list_functions(self) -> list: ... + def get_function(self, name: str): ... + +class FunctionOptions(_Weakrefable): + __slots__ = () + + def serialize(self) -> pyarrow.Buffer: ... + @staticmethod + def deserialize(buf: Buffer): ... + def __eq__(self, other: FunctionOptions): ... + +class _CastOptions(FunctionOptions): + def _set_options( + self, + target_type: DataType, + allow_int_overflow, + allow_time_truncate, + allow_time_overflow, + allow_decimal_truncate, + allow_float_truncate, + allow_invalid_utf8, + ) -> None: ... + def _set_type(self, target_type=None) -> None: ... + def _set_safe(self) -> None: ... + def _set_unsafe(self) -> None: ... + def is_safe(self) -> bool: ... + @property + def allow_int_overflow(self) -> None: ... + @allow_int_overflow.setter + def allow_int_overflow(self, flag: bool) -> None: ... + @property + def allow_time_truncate(self) -> None: ... + @allow_time_truncate.setter + def allow_time_truncate(self, flag: bool) -> None: ... + @property + def allow_time_overflow(self) -> None: ... + @allow_time_overflow.setter + def allow_time_overflow(self, flag: bool) -> None: ... + @property + def allow_decimal_truncate(self) -> None: ... + @allow_decimal_truncate.setter + def allow_decimal_truncate(self, flag: bool) -> None: ... + @property + def allow_float_truncate(self) -> None: ... + @allow_float_truncate.setter + def allow_float_truncate(self, flag: bool) -> None: ... + @property + def allow_invalid_utf8(self) -> None: ... + @allow_invalid_utf8.setter + def allow_invalid_utf8(self, flag: bool) -> None: ... + +class CastOptions(_CastOptions): + def __init__( + self, + target_type=None, + *, + allow_int_overflow=None, + allow_time_truncate=None, + allow_time_overflow=None, + allow_decimal_truncate=None, + allow_float_truncate=None, + allow_invalid_utf8=None, + ): ... + @staticmethod + def safe(target_type=None) -> CastOptions: ... + @staticmethod + def unsafe(target_type=None) -> CastOptions: ... + +class _ElementWiseAggregateOptions(FunctionOptions): + def _set_options(self, skip_nulls) -> None: ... + +class ElementWiseAggregateOptions(_ElementWiseAggregateOptions): + def __init__(self, *, skip_nulls: bool = True): ... + +class _RoundOptions(FunctionOptions): + def _set_options(self, ndigits, round_mode) -> None: ... + +class RoundOptions(_RoundOptions): + def __init__(self, ndigits: int = 0, round_mode: str = "half_to_even"): ... + +class _RoundBinaryOptions(FunctionOptions): + def _set_options(self, round_mode) -> None: ... + +class RoundBinaryOptions(_RoundBinaryOptions): + def __init__(self, round_mode: str = "half_to_even"): ... + +class _RoundTemporalOptions(FunctionOptions): + def _set_options( + self, + multiple, + unit, + week_starts_monday, + ceil_is_strictly_greater, + calendar_based_origin, + ) -> None: ... + +class RoundTemporalOptions(_RoundTemporalOptions): + def __init__( + self, + multiple: int = 1, + unit: str = "day", + *, + week_starts_monday: bool = True, + ceil_is_strictly_greater: bool = False, + calendar_based_origin: bool = False, + ): ... + +class _RoundToMultipleOptions(FunctionOptions): + def _set_options(self, multiple, round_mode) -> None: ... + +class RoundToMultipleOptions(_RoundToMultipleOptions): + def __init__(self, multiple: float = 1.0, round_mode: str = "half_to_even"): ... + +class _JoinOptions(FunctionOptions): + _null_handling_map: Incomplete + + def _set_options(self, null_handling, null_replacement) -> None: ... + +class JoinOptions(_JoinOptions): + def __init__( + self, null_handling: str = "emit_null", null_replacement: str = "" + ): ... + +class _MatchSubstringOptions(FunctionOptions): + def _set_options(self, pattern, ignore_case) -> None: ... + +class MatchSubstringOptions(_MatchSubstringOptions): + def __init__(self, pattern, *, ignore_case: bool = False): ... + +class _PadOptions(FunctionOptions): + def _set_options(self, width, padding, lean_left_on_odd_padding) -> None: ... + +class PadOptions(_PadOptions): + def __init__( + self, width, padding: str = " ", lean_left_on_odd_padding: bool = True + ): ... + +class _ZeroFillOptions(FunctionOptions): + def _set_options(self, width, padding) -> None: ... + +class ZeroFillOptions(_ZeroFillOptions): + def __init__(self, width, padding: str = "0"): ... + +class _TrimOptions(FunctionOptions): + def _set_options(self, characters) -> None: ... + +class TrimOptions(_TrimOptions): + def __init__(self, characters): ... + +class _ReplaceSubstringOptions(FunctionOptions): + def _set_options(self, pattern, replacement, max_replacements) -> None: ... + +class ReplaceSubstringOptions(_ReplaceSubstringOptions): + def __init__(self, pattern, replacement, *, max_replacements=None): ... + +class _ExtractRegexOptions(FunctionOptions): + def _set_options(self, pattern) -> None: ... + +class ExtractRegexOptions(_ExtractRegexOptions): + def __init__(self, pattern): ... + +class _ExtractRegexSpanOptions(FunctionOptions): + def _set_options(self, pattern) -> None: ... + +class ExtractRegexSpanOptions(_ExtractRegexSpanOptions): + def __init__(self, pattern): ... + +class _SliceOptions(FunctionOptions): + def _set_options(self, start, stop, step) -> None: ... + +class SliceOptions(_SliceOptions): + def __init__(self, start, stop=None, step: int = 1): ... + +class _ListSliceOptions(FunctionOptions): + def _set_options( + self, start, stop=None, step: int = 1, return_fixed_size_list=None + ): ... + +class ListSliceOptions(_ListSliceOptions): + def __init__( + self, start, stop=None, step: int = 1, return_fixed_size_list=None + ): ... + +class _ReplaceSliceOptions(FunctionOptions): + def _set_options(self, start, stop, replacement) -> None: ... + +class ReplaceSliceOptions(_ReplaceSliceOptions): + def __init__(self, start, stop, replacement): ... + +class _FilterOptions(FunctionOptions): + _null_selection_map: Incomplete + + def _set_options(self, null_selection_behavior) -> None: ... + +class FilterOptions(_FilterOptions): + def __init__(self, null_selection_behavior: str = "drop"): ... + +class _DictionaryEncodeOptions(FunctionOptions): + _null_encoding_map: Incomplete + + def _set_options(self, null_encoding) -> None: ... + +class DictionaryEncodeOptions(_DictionaryEncodeOptions): + def __init__(self, null_encoding: str = "mask"): ... + +class _RunEndEncodeOptions(FunctionOptions): + def _set_options(self, run_end_type) -> None: ... + +class RunEndEncodeOptions(_RunEndEncodeOptions): + def __init__(self, run_end_type=...): ... + +class _InversePermutationOptions(FunctionOptions): + def _set_options(self, max_index=-1, output_type=None) -> None: ... + +class InversePermutationOptions(_InversePermutationOptions): + def __init__(self, max_index=-1, output_type=None): ... + +class _ScatterOptions(FunctionOptions): + def _set_options(self, max_index) -> None: ... + +class ScatterOptions(_ScatterOptions): + def __init__(self, max_index=-1): ... + +class _TakeOptions(FunctionOptions): + def _set_options(self, boundscheck) -> None: ... + +class TakeOptions(_TakeOptions): + def __init__(self, *, boundscheck: bool = True): ... + +class _MakeStructOptions(FunctionOptions): + def _set_options(self, field_names, field_nullability, field_metadata) -> None: ... + +class MakeStructOptions(_MakeStructOptions): + def __init__( + self, field_names=(), *, field_nullability=None, field_metadata=None + ): ... + +class _StructFieldOptions(FunctionOptions): + def _set_options(self, indices) -> None: ... + +class StructFieldOptions(_StructFieldOptions): + def __init__(self, indices): ... + +class _ScalarAggregateOptions(FunctionOptions): + def _set_options(self, skip_nulls, min_count) -> None: ... + +class ScalarAggregateOptions(_ScalarAggregateOptions): + def __init__(self, *, skip_nulls: bool = True, min_count: int = 1): ... + +class _CountOptions(FunctionOptions): + _mode_map: Incomplete + + def _set_options(self, mode) -> None: ... + +class CountOptions(_CountOptions): + def __init__(self, mode: str = "only_valid"): ... + +class _IndexOptions(FunctionOptions): + def _set_options(self, scalar) -> None: ... + +class IndexOptions(_IndexOptions): + def __init__(self, value): ... + +class _MapLookupOptions(FunctionOptions): + _occurrence_map: Incomplete + + def _set_options(self, query_key, occurrence) -> None: ... + +class MapLookupOptions(_MapLookupOptions): + def __init__(self, query_key, occurrence): ... + +class _ModeOptions(FunctionOptions): + def _set_options(self, n, skip_nulls, min_count) -> None: ... + +class ModeOptions(_ModeOptions): + def __init__(self, n: int = 1, *, skip_nulls: bool = True, min_count: int = 0): ... + +class _SetLookupOptions(FunctionOptions): + def _set_options(self, value_set, skip_nulls: bool) -> None: ... + +class SetLookupOptions(_SetLookupOptions): + def __init__(self, value_set, *, skip_nulls: bool = False): ... + +class _StrptimeOptions(FunctionOptions): + _unit_map: Incomplete + + def _set_options(self, format, unit, error_is_null) -> None: ... + +class StrptimeOptions(_StrptimeOptions): + def __init__(self, format, unit, error_is_null: bool = False): ... + +class _StrftimeOptions(FunctionOptions): + def _set_options(self, format, locale) -> None: ... + +class StrftimeOptions(_StrftimeOptions): + def __init__(self, format: str = "%Y-%m-%dT%H:%M:%S", locale: str = "C"): ... + +class _DayOfWeekOptions(FunctionOptions): + def _set_options(self, count_from_zero, week_start) -> None: ... + +class DayOfWeekOptions(_DayOfWeekOptions): + def __init__(self, *, count_from_zero: bool = True, week_start: int = 1): ... + +class _WeekOptions(FunctionOptions): + def _set_options( + self, week_starts_monday, count_from_zero, first_week_is_fully_in_year + ) -> None: ... + +class WeekOptions(_WeekOptions): + def __init__( + self, + *, + week_starts_monday: bool = True, + count_from_zero: bool = False, + first_week_is_fully_in_year: bool = False, + ): ... + +class _AssumeTimezoneOptions(FunctionOptions): + _ambiguous_map: Incomplete + _nonexistent_map: Incomplete + + def _set_options(self, timezone, ambiguous, nonexistent) -> None: ... + +class AssumeTimezoneOptions(_AssumeTimezoneOptions): + def __init__( + self, timezone, *, ambiguous: str = "raise", nonexistent: str = "raise" + ): ... + +class _NullOptions(FunctionOptions): + def _set_options(self, nan_is_null) -> None: ... + +class NullOptions(_NullOptions): + def __init__(self, *, nan_is_null: bool = False): ... + +class _VarianceOptions(FunctionOptions): + def _set_options(self, ddof, skip_nulls, min_count) -> None: ... + +class VarianceOptions(_VarianceOptions): + def __init__( + self, *, ddof: int = 0, skip_nulls: bool = True, min_count: int = 0 + ): ... + +class _SkewOptions(FunctionOptions): + def _set_options(self, skip_nulls, biased, min_count) -> None: ... + +class SkewOptions(_SkewOptions): + def __init__( + self, *, skip_nulls: bool = True, biased: bool = True, min_count: int = 0 + ): ... + +class _SplitOptions(FunctionOptions): + def _set_options(self, max_splits, reverse) -> None: ... + +class SplitOptions(_SplitOptions): + def __init__(self, *, max_splits=None, reverse: bool = False): ... + +class _SplitPatternOptions(FunctionOptions): + def _set_options(self, pattern, max_splits, reverse) -> None: ... + +class SplitPatternOptions(_SplitPatternOptions): + def __init__(self, pattern, *, max_splits=None, reverse: bool = False): ... + +class _PartitionNthOptions(FunctionOptions): + def _set_options(self, pivot, null_placement) -> None: ... + +class PartitionNthOptions(_PartitionNthOptions): + def __init__(self, pivot, *, null_placement: str = "at_end"): ... + +class _WinsorizeOptions(FunctionOptions): + def _set_options(self, lower_limit, upper_limit) -> None: ... + +class WinsorizeOptions(_WinsorizeOptions): + def __init__(self, lower_limit, upper_limit): ... + +class _CumulativeOptions(FunctionOptions): + def _set_options(self, start, skip_nulls) -> None: ... + +class CumulativeOptions(_CumulativeOptions): + def __init__(self, start=None, *, skip_nulls: bool = False): ... + +class CumulativeSumOptions(_CumulativeOptions): + def __init__(self, start=None, *, skip_nulls: bool = False): ... + +class _PairwiseOptions(FunctionOptions): + def _set_options(self, period) -> None: ... + +class PairwiseOptions(_PairwiseOptions): + def __init__(self, period: int = 1): ... + +class _ListFlattenOptions(FunctionOptions): + def _set_options(self, recursive) -> None: ... + +class ListFlattenOptions(_ListFlattenOptions): + def __init__(self, recursive: bool = False): ... + +class _ArraySortOptions(FunctionOptions): + def _set_options(self, order, null_placement) -> None: ... + +class ArraySortOptions(_ArraySortOptions): + def __init__(self, order: str = "ascending", *, null_placement: str = "at_end"): ... + +class _SearchSortedOptions(FunctionOptions): + def _set_options(self, side) -> None: ... + +class SearchSortedOptions(_SearchSortedOptions): + def __init__(self, side: str = "left"): ... + +class _SortOptions(FunctionOptions): + def _set_options(self, sort_keys, null_placement) -> None: ... + +class SortOptions(_SortOptions): + def __init__(self, sort_keys=(), *, null_placement=None): ... + +class _SelectKOptions(FunctionOptions): + def _set_options(self, k, sort_keys) -> None: ... + +class SelectKOptions(_SelectKOptions): + def __init__(self, k, sort_keys): ... + +class _QuantileOptions(FunctionOptions): + _interp_map: Incomplete + + def _set_options(self, quantiles, interp, skip_nulls, min_count) -> None: ... + +class QuantileOptions(_QuantileOptions): + def __init__( + self, + q: float = 0.5, + *, + interpolation: str = "linear", + skip_nulls: bool = True, + min_count: int = 0, + ): ... + +class _TDigestOptions(FunctionOptions): + def _set_options( + self, quantiles, delta, buffer_size, skip_nulls, min_count + ) -> None: ... + +class TDigestOptions(_TDigestOptions): + def __init__( + self, + q: float = 0.5, + *, + delta: int = 100, + buffer_size: int = 500, + skip_nulls: bool = True, + min_count: int = 0, + ): ... + +class _Utf8NormalizeOptions(FunctionOptions): + _form_map: Incomplete + + def _set_options(self, form) -> None: ... + +class Utf8NormalizeOptions(_Utf8NormalizeOptions): + def __init__(self, form): ... + +class _RandomOptions(FunctionOptions): + def _set_options(self, initializer) -> None: ... + +class RandomOptions(_RandomOptions): + def __init__(self, *, initializer: str = "system"): ... + +class _RankOptions(FunctionOptions): + _tiebreaker_map: Incomplete + + def _set_options(self, sort_keys, null_placement, tiebreaker) -> None: ... + +class RankOptions(_RankOptions): + def __init__( + self, + sort_keys: str = "ascending", + *, + null_placement=None, + tiebreaker: str = "first", + ): ... + +class _RankQuantileOptions(FunctionOptions): + def _set_options(self, sort_keys, null_placement) -> None: ... + +class RankQuantileOptions(_RankQuantileOptions): + def __init__(self, sort_keys: str = "ascending", *, null_placement=None): ... + +class _PivotWiderOptions(FunctionOptions): + def _set_options(self, key_names, unexpected_key_behavior) -> None: ... + +class PivotWiderOptions(_PivotWiderOptions): + def __init__(self, key_names, *, unexpected_key_behavior: str = "ignore"): ... + +class Expression(_Weakrefable): + def __init__(self): ... + def equals(self, other: Expression) -> bool: ... + @staticmethod + def from_substrait(message: object) -> Expression: ... + def to_substrait( + self, schema: Schema, allow_arrow_extensions: bool = False + ) -> pyarrow.Buffer: ... + @staticmethod + def _deserialize(buffer: Buffer) -> Expression: ... + def __reduce__(self) -> tuple: ... + @staticmethod + def _call( + function_name: str, arguments: list[Any], options: FunctionOptions | None = None + ) -> Expression: ... + def __bool__(self): ... + def __invert__(self): ... + def __and__(self: Expression, other): ... + def __rand__(self: Expression, other): ... + def __or__(self: Expression, other): ... + def __ror__(self: Expression, other): ... + def __add__(self: Expression, other): ... + def __radd__(self: Expression, other): ... + def __mul__(self: Expression, other): ... + def __rmul__(self: Expression, other): ... + def __sub__(self: Expression, other): ... + def __rsub__(self: Expression, other): ... + def __truediv__(self: Expression, other): ... + def __rtruediv__(self: Expression, other): ... + def is_valid(self) -> Expression: ... + def is_null(self, nan_is_null: bool = False) -> Expression: ... + def is_nan(self) -> Expression: ... + def cast( + self, + type: pyarrow.DataType | None = None, + safe: bool | None = None, + options: CastOptions | None = None, + ) -> Expression: ... + def isin(self, values: pyarrow.Array | Iterable[Any]) -> Expression: ... + @staticmethod + def _field(name_or_idx) -> Expression: ... + @staticmethod + def _nested_field(names: tuple[Any, ...]) -> Expression: ... + @staticmethod + def _scalar(value) -> Expression: ... + +class UdfContext(_Weakrefable): + def __init__(self): ... + @property + def batch_length(self): ... + @property + def memory_pool(self): ... + +class RegisterUdf(_Weakrefable): ... diff --git a/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi b/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi new file mode 100644 index 000000000000..a83cc5dc1e77 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from _typeshed import Incomplete + +function_doc_additions: Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_csv.pyi b/python/pyarrow-stubs/pyarrow/_csv.pyi new file mode 100644 index 000000000000..84b3fd8bf37d --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_csv.pyi @@ -0,0 +1,307 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_csv.pyx + +import pyarrow +from typing import NamedTuple +import pyarrow.csv +from pyarrow.lib import __name__ as __name__ +from _typeshed import Incomplete +from pyarrow.lib import ( + MemoryPool, + RecordBatchReader, + Schema, + _CRecordBatchWriter, + _Weakrefable, +) + +ISO8601: _ISO8601 + +def read_csv( + input_file, + read_options: pyarrow.csv.ReadOptions | None = None, + parse_options: pyarrow.csv.ParseOptions | None = None, + convert_options: pyarrow.csv.ConvertOptions | None = None, + memory_pool: MemoryPool | None = None, +) -> pyarrow.Table: ... +def open_csv( + input_file, + read_options: pyarrow.csv.ReadOptions | None = None, + parse_options: pyarrow.csv.ParseOptions | None = None, + convert_options: pyarrow.csv.ConvertOptions | None = None, + memory_pool: MemoryPool | None = None, +) -> CSVStreamingReader: ... +def _raise_invalid_function_option(value, description, *, exception_class=...): ... +def write_csv( + data: pyarrow.RecordBatch | pyarrow.Table, + output_file, + write_options: pyarrow.csv.WriteOptions | None = None, + memory_pool: MemoryPool | None = None, +) -> None: ... + +class _InvalidRow(NamedTuple): + expected_columns: Incomplete + actual_columns: Incomplete + number: Incomplete + text: Incomplete + +class InvalidRow(_InvalidRow): + __slots__ = () + +class ReadOptions(_Weakrefable): + __slots__ = () + encoding: object + + def __init__( + self, + *, + use_threads=None, + block_size=None, + skip_rows=None, + skip_rows_after_names=None, + column_names=None, + autogenerate_column_names=None, + encoding: str = "utf8", + ): ... + @property + def use_threads(self) -> None: ... + @use_threads.setter + def use_threads(self, value) -> None: ... + @property + def block_size(self) -> None: ... + @block_size.setter + def block_size(self, value) -> None: ... + @property + def skip_rows(self) -> None: ... + @skip_rows.setter + def skip_rows(self, value) -> None: ... + @property + def skip_rows_after_names(self) -> None: ... + @skip_rows_after_names.setter + def skip_rows_after_names(self, value) -> None: ... + @property + def column_names(self) -> None: ... + @column_names.setter + def column_names(self, value) -> None: ... + @property + def autogenerate_column_names(self) -> None: ... + @autogenerate_column_names.setter + def autogenerate_column_names(self, value) -> None: ... + def validate(self) -> None: ... + def equals(self, other: ReadOptions) -> bool: ... + def __getstate__(self) -> tuple: ... + def __setstate__(self, state) -> None: ... + def __eq__(self, other): ... + def _repr_base(self) -> str: ... + +class ParseOptions(_Weakrefable): + __slots__ = () + + def __init__( + self, + *, + delimiter=None, + quote_char=None, + double_quote=None, + escape_char=None, + newlines_in_values=None, + ignore_empty_lines=None, + invalid_row_handler=None, + ): ... + @property + def delimiter(self) -> None: ... + @delimiter.setter + def delimiter(self, value) -> None: ... + @property + def quote_char(self) -> None: ... + @quote_char.setter + def quote_char(self, value) -> None: ... + @property + def double_quote(self) -> None: ... + @double_quote.setter + def double_quote(self, value) -> None: ... + @property + def escape_char(self) -> None: ... + @escape_char.setter + def escape_char(self, value) -> None: ... + @property + def newlines_in_values(self) -> None: ... + @newlines_in_values.setter + def newlines_in_values(self, value) -> None: ... + @property + def ignore_empty_lines(self) -> None: ... + @property + def invalid_row_handler(self) -> None: ... + @invalid_row_handler.setter + def invalid_row_handler(self, value) -> None: ... + @ignore_empty_lines.setter + def ignore_empty_lines(self, value) -> None: ... + def validate(self) -> None: ... + def equals(self, other: ParseOptions) -> bool: ... + def __getstate__(self) -> tuple: ... + def __setstate__(self, state) -> None: ... + def __eq__(self, other): ... + def _repr_base(self) -> str: ... + +class _ISO8601(_Weakrefable): + __slots__ = () + + def __eq__(self, other) -> bool: ... + +class ConvertOptions(_Weakrefable): + __slots__ = () + + def __init__( + self, + *, + check_utf8=None, + column_types=None, + default_column_type=None, + null_values=None, + true_values=None, + false_values=None, + decimal_point=None, + strings_can_be_null=None, + quoted_strings_can_be_null=None, + include_columns=None, + include_missing_columns=None, + auto_dict_encode=None, + auto_dict_max_cardinality=None, + timestamp_parsers=None, + ): ... + @property + def check_utf8(self) -> None: ... + @check_utf8.setter + def check_utf8(self, value) -> None: ... + @property + def strings_can_be_null(self) -> None: ... + @strings_can_be_null.setter + def strings_can_be_null(self, value) -> None: ... + @property + def quoted_strings_can_be_null(self) -> None: ... + @quoted_strings_can_be_null.setter + def quoted_strings_can_be_null(self, value) -> None: ... + @property + def column_types(self) -> None: ... + @column_types.setter + def column_types(self, value) -> None: ... + @property + def default_column_type(self) -> None: ... + @default_column_type.setter + def default_column_type(self, value) -> None: ... + @property + def null_values(self) -> None: ... + @null_values.setter + def null_values(self, value) -> None: ... + @property + def true_values(self) -> None: ... + @true_values.setter + def true_values(self, value) -> None: ... + @property + def false_values(self) -> None: ... + @false_values.setter + def false_values(self, value) -> None: ... + @property + def decimal_point(self) -> None: ... + @decimal_point.setter + def decimal_point(self, value) -> None: ... + @property + def auto_dict_encode(self) -> None: ... + @auto_dict_encode.setter + def auto_dict_encode(self, value) -> None: ... + @property + def auto_dict_max_cardinality(self) -> None: ... + @auto_dict_max_cardinality.setter + def auto_dict_max_cardinality(self, value) -> None: ... + @property + def include_columns(self) -> None: ... + @include_columns.setter + def include_columns(self, value) -> None: ... + @property + def include_missing_columns(self) -> None: ... + @include_missing_columns.setter + def include_missing_columns(self, value) -> None: ... + @property + def timestamp_parsers(self) -> None: ... + @timestamp_parsers.setter + def timestamp_parsers(self, value) -> None: ... + def validate(self) -> None: ... + def equals(self, other: ConvertOptions) -> bool: ... + def __getstate__(self) -> tuple: ... + def __setstate__(self, state) -> None: ... + def __eq__(self, other): ... + def _repr_base(self) -> str: ... + +class CSVStreamingReader(RecordBatchReader): + schema: Schema + + def __init__(self): ... + +class WriteOptions(_Weakrefable): + __slots__ = () + + def __init__( + self, + *, + include_header=None, + batch_size=None, + delimiter=None, + eol=None, + null_string=None, + quoting_style=None, + quoting_header=None, + ): ... + @property + def include_header(self) -> None: ... + @include_header.setter + def include_header(self, value) -> None: ... + @property + def batch_size(self) -> None: ... + @batch_size.setter + def batch_size(self, value) -> None: ... + @property + def delimiter(self) -> None: ... + @delimiter.setter + def delimiter(self, value) -> None: ... + @property + def eol(self) -> None: ... + @eol.setter + def eol(self, value) -> None: ... + @property + def null_string(self) -> None: ... + @null_string.setter + def null_string(self, value) -> None: ... + @property + def quoting_style(self) -> None: ... + @quoting_style.setter + def quoting_style(self, value) -> None: ... + @property + def quoting_header(self) -> None: ... + @quoting_header.setter + def quoting_header(self, value) -> None: ... + def validate(self) -> None: ... + def _repr_base(self) -> str: ... + +class CSVWriter(_CRecordBatchWriter): + def __init__( + self, + sink, + schema: Schema, + *, + write_options: WriteOptions | None = None, + memory_pool: MemoryPool | None = None, + ): ... diff --git a/python/pyarrow-stubs/pyarrow/_cuda.pyi b/python/pyarrow-stubs/pyarrow/_cuda.pyi new file mode 100644 index 000000000000..a0b978c0084e --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_cuda.pyi @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated by stubgen-pyx from python/pyarrow/_cuda.pyx + +from pyarrow.includes.common import * +from pyarrow.includes.libarrow import * +from pyarrow.includes.libarrow_cuda import * +from pyarrow.lib import * + +class Context(_Weakrefable): + def __init__(self, *args, **kwargs): ... + @staticmethod + def from_numba(context=None): ... + def to_numba(self): ... + @staticmethod + def get_num_devices(): ... + @property + def device_number(self): ... + @property + def handle(self): ... + def synchronize(self): ... + @property + def bytes_allocated(self): ... + def get_device_address(self, address: int): ... + def new_buffer(self, nbytes: int): ... + @property + def memory_manager(self): ... + @property + def device(self): ... + def foreign_buffer(self, address, size, base=None): ... + def open_ipc_buffer(self, ipc_handle): ... + def buffer_from_data(self, data: object, offset: int = 0, size: int = -1): ... + def buffer_from_object(self, obj): ... + +class IpcMemHandle(_Weakrefable): + @staticmethod + def from_buffer(opaque_handle: Buffer): ... + def serialize(self, pool=None): ... + +class CudaBuffer(Buffer): + def __init__(self): ... + @staticmethod + def from_buffer(buf): ... + @staticmethod + def from_numba(mem): ... + def to_numba(self): ... + def copy_to_host( + self, + position: int = 0, + nbytes: int = -1, + buf: Buffer | None = None, + memory_pool: MemoryPool | None = None, + resizable: c_bool = False, + ): ... + def copy_from_host(self, data, position: int = 0, nbytes: int = -1): ... + def copy_from_device(self, buf, position: int = 0, nbytes: int = -1): ... + def export_for_ipc(self): ... + @property + def context(self): ... + def slice(self, offset=0, length=None): ... + def to_pybytes(self): ... + +class HostBuffer(Buffer): + def __init__(self): ... + @property + def size(self): ... + +class BufferReader(NativeFile): + def __init__(self, obj: CudaBuffer): ... + def read_buffer(self, nbytes=None): ... + +class BufferWriter(NativeFile): + def __init__(self, buffer: CudaBuffer): ... + def writeat(self, position: int, data: object): ... + def flush(self): ... + def seek(self, position: int, whence: int = 0): ... + @property + def buffer_size(self): ... + @buffer_size.setter + def buffer_size(self, buffer_size: int): ... + @property + def num_bytes_buffered(self): ... + +def new_host_buffer(size: int, device: int = 0): ... +def serialize_record_batch(batch: object, ctx: object): ... +def read_message(source: object, pool=None): ... +def read_record_batch( + buffer: object, + schema: object, + *, + dictionary_memo: DictionaryMemo | None = None, + pool=None, +): ... diff --git a/python/pyarrow-stubs/pyarrow/_dataset.pyi b/python/pyarrow-stubs/pyarrow/_dataset.pyi new file mode 100644 index 000000000000..86100b465553 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_dataset.pyi @@ -0,0 +1,639 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_dataset.pyx + +import os +import pyarrow +from typing import IO, NamedTuple, Self +from collections.abc import Iterator +import pyarrow.compute +import pyarrow.dataset +import pyarrow.fs +from pyarrow.lib import _Weakrefable +from _typeshed import Incomplete +from typing import Any +from pyarrow._acero import ExecNodeOptions +from pyarrow._compute import Expression +from pyarrow._csv import ConvertOptions, ParseOptions, ReadOptions, WriteOptions +from pyarrow._fs import FileSystem +from pyarrow._json import ParseOptions as JsonParseOptions +from pyarrow._json import ReadOptions as JsonReadOptions +from pyarrow.lib import * + +_DEFAULT_BATCH_SIZE: int +_DEFAULT_BATCH_READAHEAD: int +_DEFAULT_FRAGMENT_READAHEAD: int +_orc_fileformat: Incomplete +_orc_imported: bool +_dataset_pq: bool + +def _get_orc_fileformat() -> None: ... +def _get_parquet_classes() -> None: ... +def _get_parquet_symbol(name): ... +def _constructor_directory_partitioning_factory(*args): ... +def _constructor_hive_partitioning_factory(*args): ... +def _constructor_filename_partitioning_factory(*args): ... +def get_partition_keys(partition_expression: Expression) -> dict: ... +def _filesystemdataset_write( + data: Scanner, + base_dir: object, + basename_template: str, + filesystem: FileSystem, + partitioning: Partitioning, + preserve_order: bool, + file_options: FileWriteOptions, + max_partitions: int, + file_visitor: object, + existing_data_behavior: str, + max_open_files: int, + max_rows_per_file: int, + min_rows_per_group: int, + max_rows_per_group: int, + create_dir: bool, +) -> None: ... + +class Dataset(_Weakrefable): + _scan_options: dict[Any, Any] + + def __init__(self): ... + @property + def partition_expression(self) -> pyarrow.compute.Expression: ... + def replace_schema(self, schema: Schema) -> Dataset: ... + def get_fragments(self, filter: Expression | None = None) -> Iterator[Fragment]: ... + def _get_fragments(self, filter: Expression) -> Iterator[Fragment]: ... + def _scanner_options(self, options): ... + def scanner( + self, + columns: object | None = None, + filter: object | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Scanner: ... + def to_batches( + self, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Iterator[pyarrow.RecordBatch]: ... + def to_table( + self, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> pyarrow.Table: ... + def take( + self, + indices: object, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> pyarrow.Table: ... + def head( + self, + num_rows: int, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> pyarrow.Table: ... + def count_rows( + self, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> int: ... + @property + def schema(self) -> pyarrow.Schema: ... + def filter(self, expression: pyarrow.compute.Expression) -> Dataset: ... + def sort_by(self, sorting, **kwargs) -> InMemoryDataset: ... + def join( + self, + right_dataset, + keys: str | list[str], + right_keys: str | list[str] | None = None, + join_type: str = "left outer", + left_suffix: str | None = None, + right_suffix: str | None = None, + coalesce_keys: bool = True, + use_threads: bool = True, + ) -> InMemoryDataset: ... + def join_asof( + self, + right_dataset, + on: str, + by: str | list[str], + tolerance: int, + right_on: str | list[str] | None = None, + right_by: str | list[str] | None = None, + ) -> InMemoryDataset: ... + +class InMemoryDataset(Dataset): + def __init__(self, source, schema: Schema | None = None): ... + +class UnionDataset(Dataset): + def __init__(self, schema: Schema, children): ... + def __reduce__(self) -> tuple: ... + @property + def children(self) -> list: ... + +class FileSystemDataset(Dataset): + def __init__( + self, + fragments, + schema: Schema, + format: FileFormat, + filesystem: FileSystem | None = None, + root_partition=None, + ): ... + @property + def filesystem(self) -> pyarrow.fs.FileSystem: ... + @property + def partitioning(self) -> None | Partitioning: ... + def __reduce__(self) -> tuple: ... + @classmethod + def from_paths( + cls, + paths: list[str], + schema: pyarrow.Schema | None = None, + format: FileFormat | None = None, + filesystem: pyarrow.fs.FileSystem | None = None, + partitions: list[pyarrow.compute.Expression] | None = None, + root_partition: pyarrow.compute.Expression | None = None, + ) -> FileSystemDataset: ... + @property + def files(self) -> list: ... + @property + def format(self) -> FileFormat: ... + +class FileWriteOptions(_Weakrefable): + def __init__(self): ... + @property + def format(self) -> FileFormat: ... + +class FileFormat(_Weakrefable): + def __init__(self): ... + def inspect( + self, file: IO[Any] | os.PathLike[str] | str, filesystem=None + ) -> pyarrow.Schema: ... + def make_fragment( + self, + file: IO[Any] | os.PathLike[str] | str, + filesystem=None, + partition_expression: Expression | None = None, + *, + file_size: int | None = None, + ) -> Fragment: ... + def make_write_options(self) -> FileWriteOptions: ... + @property + def default_extname(self) -> str: ... + @property + def default_fragment_scan_options(self) -> None: ... + @default_fragment_scan_options.setter + def default_fragment_scan_options(self, options: FragmentScanOptions) -> None: ... + def __eq__(self, other): ... + +class Fragment(_Weakrefable): + def __init__(self): ... + @property + def physical_schema(self) -> pyarrow.Schema: ... + @property + def partition_expression(self) -> pyarrow.compute.Expression: ... + def scanner( + self, + schema: Schema | None = None, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Scanner: ... + def to_batches( + self, + schema: Schema | None = None, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Iterator[pyarrow.RecordBatch]: ... + def to_table( + self, + schema: Schema | None = None, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> pyarrow.Table: ... + def take( + self, + indices: object, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> pyarrow.Table: ... + def head( + self, + num_rows: int, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> pyarrow.Table: ... + def count_rows( + self, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> int: ... + +class FileFragment(Fragment): + def __reduce__(self) -> tuple: ... + def open(self): ... + @property + def path(self) -> str: ... + @property + def filesystem(self) -> None | pyarrow.fs.FileSystem: ... + @property + def buffer(self) -> None | pyarrow.Buffer: ... + @property + def format(self) -> FileFormat: ... + +class FragmentScanOptions(_Weakrefable): + def __init__(self): ... + @property + def type_name(self) -> str: ... + def __eq__(self, other): ... + +class IpcFileWriteOptions(FileWriteOptions): + def __init__(self): ... + @property + def write_options(self) -> None: ... + @write_options.setter + def write_options(self, write_options: IpcWriteOptions) -> None: ... + +class IpcFileFormat(FileFormat): + def __init__(self): ... + def equals(self, other: IpcFileFormat) -> bool: ... + def make_write_options(self, **kwargs) -> IpcFileWriteOptions: ... + @property + def default_extname(self) -> str: ... + def __reduce__(self) -> tuple: ... + +class FeatherFileFormat(IpcFileFormat): + @property + def default_extname(self) -> str: ... + +class CsvFileFormat(FileFormat): + __slots__ = () + _read_options_py: ReadOptions + + def __init__( + self, + parse_options: ParseOptions | None = None, + default_fragment_scan_options=None, + convert_options: ConvertOptions | None = None, + read_options: ReadOptions | None = None, + ): ... + def make_write_options(self, **kwargs) -> CsvFileWriteOptions: ... + @property + def parse_options(self) -> None: ... + @parse_options.setter + def parse_options(self, parse_options: ParseOptions) -> None: ... + def equals(self, other: CsvFileFormat) -> bool: ... + def __reduce__(self) -> tuple: ... + +class CsvFragmentScanOptions(FragmentScanOptions): + __slots__ = () + + def __init__( + self, + convert_options: ConvertOptions | None = None, + read_options: ReadOptions | None = None, + ): ... + @property + def convert_options(self) -> None: ... + @convert_options.setter + def convert_options(self, convert_options: ConvertOptions) -> None: ... + @property + def read_options(self) -> None: ... + @read_options.setter + def read_options(self, read_options: ReadOptions) -> None: ... + def equals(self, other: CsvFragmentScanOptions) -> bool: ... + def __reduce__(self) -> tuple: ... + +class CsvFileWriteOptions(FileWriteOptions): + def __init__(self): ... + @property + def write_options(self) -> None: ... + @write_options.setter + def write_options(self, write_options: WriteOptions) -> None: ... + +class JsonFileFormat(FileFormat): + __slots__ = () + + def __init__( + self, + default_fragment_scan_options=None, + parse_options: JsonParseOptions | None = None, + read_options: JsonReadOptions | None = None, + ): ... + def equals(self, other: JsonFileFormat) -> bool: ... + def __reduce__(self) -> tuple: ... + +class JsonFragmentScanOptions(FragmentScanOptions): + __slots__ = () + + def __init__( + self, + parse_options: JsonParseOptions | None = None, + read_options: JsonReadOptions | None = None, + ): ... + @property + def parse_options(self) -> None: ... + @parse_options.setter + def parse_options(self, parse_options: JsonParseOptions) -> None: ... + @property + def read_options(self) -> None: ... + @read_options.setter + def read_options(self, read_options: JsonReadOptions) -> None: ... + def equals(self, other: JsonFragmentScanOptions) -> bool: ... + def __reduce__(self) -> tuple: ... + +class Partitioning(_Weakrefable): + def __init__(self): ... + def __eq__(self, other): ... + def parse(self, path: str) -> pyarrow.compute.Expression: ... + def format(self, expr: pyarrow.dataset.Expression) -> tuple: ... + @property + def schema(self) -> pyarrow.Schema: ... + +class PartitioningFactory(_Weakrefable): + def __init__(self): ... + def __reduce__(self) -> tuple: ... + @property + def type_name(self) -> str: ... + +class KeyValuePartitioning(Partitioning): + def __init__(self): ... + def __reduce__(self) -> tuple: ... + @property + def dictionaries(self) -> list: ... + +class DirectoryPartitioning(KeyValuePartitioning): + def __init__( + self, schema: Schema, dictionaries=None, segment_encoding: str = "uri" + ): ... + @staticmethod + def discover( + field_names: list[str] | None = None, + infer_dictionary: bool = False, + max_partition_dictionary_size: int = 0, + schema: pyarrow.Schema | None = None, + segment_encoding: str = "uri", + ) -> PartitioningFactory: ... + +class HivePartitioning(KeyValuePartitioning): + def __init__( + self, + schema: Schema, + dictionaries=None, + null_fallback: str = "__HIVE_DEFAULT_PARTITION__", + segment_encoding: str = "uri", + ): ... + def __reduce__(self) -> tuple: ... + @staticmethod + def discover( + infer_dictionary: bool = False, + max_partition_dictionary_size: int = 0, + null_fallback: str = "__HIVE_DEFAULT_PARTITION__", + schema: pyarrow.Schema | None = None, + segment_encoding: str = "uri", + ) -> PartitioningFactory: ... + +class FilenamePartitioning(KeyValuePartitioning): + def __init__( + self, schema: Schema, dictionaries=None, segment_encoding: str = "uri" + ): ... + @staticmethod + def discover( + field_names: list[str] | None = None, + infer_dictionary: bool = False, + schema: pyarrow.Schema | None = None, + segment_encoding: str = "uri", + ) -> PartitioningFactory: ... + +class DatasetFactory(_Weakrefable): + def __init__(self): ... + @property + def root_partition(self) -> None: ... + @root_partition.setter + def root_partition(self, expr: Expression) -> None: ... + def inspect_schemas(self) -> list: ... + def inspect( + self, *, promote_options: str = "default", fragments: int | None = None + ) -> pyarrow.Schema: ... + def finish(self, schema: Schema | None = None) -> Dataset: ... + +class FileSystemFactoryOptions(_Weakrefable): + __slots__ = () + + def __init__( + self, + partition_base_dir=None, + partitioning=None, + exclude_invalid_files=None, + selector_ignore_prefixes: list[Any] | None = None, + ): ... + @property + def partitioning(self) -> None: ... + @partitioning.setter + def partitioning(self, value: Partitioning) -> None: ... + @property + def partitioning_factory(self) -> None: ... + @partitioning_factory.setter + def partitioning_factory(self, value: PartitioningFactory) -> None: ... + @property + def partition_base_dir(self) -> None: ... + @partition_base_dir.setter + def partition_base_dir(self, value) -> None: ... + @property + def exclude_invalid_files(self) -> None: ... + @exclude_invalid_files.setter + def exclude_invalid_files(self, value: bool) -> None: ... + @property + def selector_ignore_prefixes(self) -> None: ... + @selector_ignore_prefixes.setter + def selector_ignore_prefixes(self, values) -> None: ... + +class FileSystemDatasetFactory(DatasetFactory): + def __init__( + self, + filesystem: FileSystem, + paths_or_selector, + format: FileFormat, + options: FileSystemFactoryOptions | None = None, + ): ... + +class UnionDatasetFactory(DatasetFactory): + def __init__(self, factories: list[Any]): ... + +class RecordBatchIterator(_Weakrefable): + def __init__(self): ... + def __iter__(self) -> Self: ... + def __next__(self) -> pyarrow.RecordBatch: ... + +class TaggedRecordBatch(NamedTuple): + record_batch: Incomplete + fragment: Incomplete + +class TaggedRecordBatchIterator(_Weakrefable): + def __init__(self): ... + def __iter__(self) -> Self: ... + def __next__(self) -> TaggedRecordBatch: ... + +class Scanner(_Weakrefable): + def __init__(self): ... + @staticmethod + def from_dataset( + dataset: Dataset, + *, + columns: object | None = None, + filter: object | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Scanner: ... + @staticmethod + def from_fragment( + fragment: Fragment, + *, + schema: Schema | None = None, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Scanner: ... + @staticmethod + def from_batches( + source, + *, + schema: Schema | None = None, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Scanner: ... + @property + def dataset_schema(self) -> pyarrow.Schema: ... + @property + def projected_schema(self) -> pyarrow.Schema: ... + def to_batches(self) -> Iterator[pyarrow.RecordBatch]: ... + def scan_batches(self) -> TaggedRecordBatchIterator: ... + def to_table(self) -> pyarrow.Table: ... + def take(self, indices: object) -> pyarrow.Table: ... + def head(self, num_rows: int) -> pyarrow.Table: ... + def count_rows(self) -> int: ... + def to_reader(self) -> pyarrow.RecordBatchReader: ... + +class WrittenFile(_Weakrefable): + path: str + metadata: object + size: int + + def __init__(self, path, metadata, size): ... + +class _ScanNodeOptions(ExecNodeOptions): + def _set_options(self, dataset: Dataset, scan_options: dict[Any, Any]) -> None: ... + +class ScanNodeOptions(_ScanNodeOptions): + def __init__(self, dataset: Dataset, **kwargs): ... diff --git a/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi b/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi new file mode 100644 index 000000000000..72242b17ec99 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_dataset_orc.pyx + +from pyarrow._dataset import FileFormat +from pyarrow.lib import * + +class OrcFileFormat(FileFormat): + def __init__(self): ... + def equals(self, other: OrcFileFormat) -> bool: ... + @property + def default_extname(self) -> str: ... + def __reduce__(self) -> tuple: ... diff --git a/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi b/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi new file mode 100644 index 000000000000..ec62a32f7643 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi @@ -0,0 +1,227 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_dataset_parquet.pyx + +import os +import pyarrow +from typing import IO +from collections.abc import Iterable +import pyarrow.dataset +import pyarrow.parquet +from pyarrow.lib import _Weakrefable +from typing import Any +from pyarrow._compute import Expression +from pyarrow._dataset import ( + CacheOptions, + DatasetFactory, + FileFormat, + FileFragment, + FileWriteOptions, + FragmentScanOptions, + Partitioning, + PartitioningFactory, +) +from pyarrow._fs import FileSystem +from pyarrow.lib import * + +parquet_encryption_enabled: bool + +class ParquetFileFormat(FileFormat): + def __init__( + self, read_options=None, default_fragment_scan_options=None, **kwargs + ): ... + @property + def read_options(self) -> ParquetReadOptions: ... + def make_write_options(self, **kwargs) -> pyarrow.dataset.FileWriteOptions: ... + def equals(self, other: ParquetFileFormat) -> bool: ... + @property + def default_extname(self) -> str: ... + def __reduce__(self) -> tuple: ... + def make_fragment( + self, + file: IO[Any] | os.PathLike[str] | str, + filesystem=None, + partition_expression: Expression | None = None, + row_groups: Iterable[Any] | None = None, + *, + file_size: int | None = None, + ) -> pyarrow.dataset.Fragment: ... + +class RowGroupInfo: + def __init__(self, id, metadata, schema): ... + @property + def num_rows(self): ... + @property + def total_byte_size(self): ... + @property + def statistics(self) -> dict: ... + def __eq__(self, other) -> bool: ... + +class ParquetFileFragment(FileFragment): + def __reduce__(self) -> tuple: ... + def ensure_complete_metadata(self) -> None: ... + @property + def row_groups(self) -> list: ... + @property + def metadata(self) -> pyarrow.parquet.FileMetaData: ... + @property + def num_row_groups(self): ... + def split_by_row_group( + self, filter: Expression | None = None, schema: Schema | None = None + ) -> list: ... + def subset( + self, + filter: Expression | None = None, + schema: Schema | None = None, + row_group_ids: object | None = None, + ) -> pyarrow.dataset.Fragment: ... + +class ParquetReadOptions(_Weakrefable): + dictionary_columns: set[Any] + _coerce_int96_timestamp_unit: int + _binary_type: int + _list_type: int + + def __init__( + self, + dictionary_columns=None, + coerce_int96_timestamp_unit=None, + binary_type=None, + list_type=None, + ): ... + @property + def binary_type(self) -> None: ... + @binary_type.setter + def binary_type(self, ty) -> None: ... + @property + def list_type(self) -> None: ... + @list_type.setter + def list_type(self, ty) -> None: ... + @property + def coerce_int96_timestamp_unit(self) -> None: ... + @coerce_int96_timestamp_unit.setter + def coerce_int96_timestamp_unit(self, unit) -> None: ... + def equals(self, other: ParquetReadOptions) -> bool: ... + def __eq__(self, other): ... + +class ParquetFileWriteOptions(FileWriteOptions): + def update(self, **kwargs) -> None: ... + def _set_properties(self) -> None: ... + def _set_arrow_properties(self) -> None: ... + def _set_encryption_config(self) -> None: ... + +class ParquetFragmentScanOptions(FragmentScanOptions): + __slots__ = () + + def __init__( + self, + *, + use_buffered_stream: bool = False, + buffer_size: int = 8192, + pre_buffer: bool = True, + cache_options=None, + thrift_string_size_limit=None, + thrift_container_size_limit=None, + schema_depth_limit=None, + decryption_config=None, + decryption_properties=None, + page_checksum_verification: bool = False, + arrow_extensions_enabled: bool = False, + ): ... + @property + def use_buffered_stream(self) -> None: ... + @use_buffered_stream.setter + def use_buffered_stream(self, use_buffered_stream: bool) -> None: ... + @property + def buffer_size(self) -> None: ... + @buffer_size.setter + def buffer_size(self, buffer_size) -> None: ... + @property + def pre_buffer(self) -> None: ... + @pre_buffer.setter + def pre_buffer(self, pre_buffer: bool) -> None: ... + @property + def cache_options(self) -> None: ... + @cache_options.setter + def cache_options(self, options: CacheOptions) -> None: ... + @property + def thrift_string_size_limit(self) -> None: ... + @thrift_string_size_limit.setter + def thrift_string_size_limit(self, size) -> None: ... + @property + def thrift_container_size_limit(self) -> None: ... + @thrift_container_size_limit.setter + def thrift_container_size_limit(self, size) -> None: ... + @property + def schema_depth_limit(self) -> None: ... + @schema_depth_limit.setter + def schema_depth_limit(self, limit) -> None: ... + @property + def decryption_properties(self) -> None: ... + @decryption_properties.setter + def decryption_properties(self, config) -> None: ... + @property + def parquet_decryption_config(self) -> None: ... + @parquet_decryption_config.setter + def parquet_decryption_config(self, config) -> None: ... + @property + def page_checksum_verification(self) -> None: ... + @page_checksum_verification.setter + def page_checksum_verification(self, page_checksum_verification: bool) -> None: ... + @property + def arrow_extensions_enabled(self) -> None: ... + @arrow_extensions_enabled.setter + def arrow_extensions_enabled(self, arrow_extensions_enabled: bool) -> None: ... + def equals(self, other: ParquetFragmentScanOptions) -> bool: ... + @staticmethod + def _reconstruct(kwargs) -> ParquetFragmentScanOptions: ... + def __reduce__(self) -> tuple: ... + +class ParquetFactoryOptions(_Weakrefable): + __slots__ = () + + def __init__( + self, + partition_base_dir=None, + partitioning=None, + validate_column_chunk_paths: bool = False, + ): ... + @property + def partitioning(self) -> None: ... + @partitioning.setter + def partitioning(self, value: Partitioning) -> None: ... + @property + def partitioning_factory(self) -> None: ... + @partitioning_factory.setter + def partitioning_factory(self, value: PartitioningFactory) -> None: ... + @property + def partition_base_dir(self) -> None: ... + @partition_base_dir.setter + def partition_base_dir(self, value) -> None: ... + @property + def validate_column_chunk_paths(self) -> None: ... + @validate_column_chunk_paths.setter + def validate_column_chunk_paths(self, value) -> None: ... + +class ParquetDatasetFactory(DatasetFactory): + def __init__( + self, + metadata_path, + filesystem: FileSystem, + format: FileFormat, + options: ParquetFactoryOptions | None = None, + ): ... diff --git a/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi b/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi new file mode 100644 index 000000000000..315a86b82c15 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_dataset_parquet_encryption.pyx + +import pyarrow.parquet +import pyarrow +from pyarrow.lib import _Weakrefable +from pyarrow._dataset_parquet import ParquetFileWriteOptions, ParquetFragmentScanOptions +from pyarrow._parquet_encryption import * + +def set_encryption_config( + opts: ParquetFileWriteOptions, config: ParquetEncryptionConfig +) -> None: ... +def set_decryption_properties( + opts: ParquetFragmentScanOptions, config: pyarrow.parquet.FileDecryptionProperties +) -> None: ... +def set_decryption_config( + opts: ParquetFragmentScanOptions, config: ParquetDecryptionConfig +) -> None: ... + +class ParquetEncryptionConfig(_Weakrefable): + __slots__ = () + + def __init__( + self, + crypto_factory: CryptoFactory, + kms_connection_config: KmsConnectionConfig, + encryption_config: EncryptionConfiguration, + ): ... + +class ParquetDecryptionConfig(_Weakrefable): + __slots__ = () + + def __init__( + self, + crypto_factory: CryptoFactory, + kms_connection_config: KmsConnectionConfig, + decryption_config: DecryptionConfiguration, + ): ... diff --git a/python/pyarrow-stubs/pyarrow/_feather.pyi b/python/pyarrow-stubs/pyarrow/_feather.pyi new file mode 100644 index 000000000000..8b2785320ae6 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_feather.pyi @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_feather.pyx + +import pyarrow +from pyarrow.lib import Table, _Weakrefable + +def write_feather( + table: Table, + dest: object, + compression=None, + compression_level=None, + chunksize=None, + version: int = 2, +) -> None: ... + +class FeatherError(Exception): ... + +class FeatherReader(_Weakrefable): + def __init__(self, source, use_memory_map: bool, use_threads: bool): ... + @property + def version(self): ... + def read(self) -> pyarrow.Table: ... + def read_indices(self, indices) -> pyarrow.Table: ... + def read_names(self, names) -> pyarrow.Table: ... diff --git a/python/pyarrow-stubs/pyarrow/_flight.pyi b/python/pyarrow-stubs/pyarrow/_flight.pyi new file mode 100644 index 000000000000..a1cc899886de --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_flight.pyi @@ -0,0 +1,484 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_flight.pyx + +import pyarrow +from typing import NamedTuple, Self +from collections.abc import Iterable, Iterator +import pyarrow.ipc +from pyarrow.lib import _CRecordBatchWriter, _Weakrefable +from _typeshed import Incomplete +import enum +import re +from typing import Any +from pyarrow.ipc import _ReadPandasMixin as _ReadPandasMixin +from pyarrow.lib import * +from pyarrow.lib import ( + ArrowCancelled as ArrowCancelled, + ArrowException as ArrowException, + ArrowInvalid as ArrowInvalid, +) + +_FLIGHT_SERVER_ERROR_REGEX: re.Pattern[str] + +def _munge_grpc_python_error(message): ... +def connect(location: str | tuple | Location, **kwargs) -> FlightClient: ... + +class _CertKeyPair(NamedTuple): + cert: Incomplete + key: Incomplete + +class _ActionType(NamedTuple): + type: Incomplete + description: Incomplete + +class _CallInfo(NamedTuple): + method: Incomplete + +class FlightCallOptions(_Weakrefable): + def __init__( + self, + timeout: float | None = None, + write_options: pyarrow.ipc.IpcWriteOptions | None = None, + headers=None, + read_options: IpcReadOptions | None = None, + ): ... + @property + def timeout(self): ... + @property + def headers(self): ... + @property + def read_options(self): ... + @property + def write_options(self): ... + +class CertKeyPair(_CertKeyPair): ... + +class FlightError(Exception): + def __init__(self, message: str = "", extra_info: bytes = b""): ... + +class FlightInternalError(FlightError, ArrowException): ... +class FlightTimedOutError(FlightError, ArrowException): ... +class FlightCancelledError(FlightError, ArrowCancelled): ... +class FlightServerError(FlightError, ArrowException): ... +class FlightUnauthenticatedError(FlightError, ArrowException): ... +class FlightUnauthorizedError(FlightError, ArrowException): ... +class FlightUnavailableError(FlightError, ArrowException): ... + +class FlightWriteSizeExceededError(ArrowInvalid): + def __init__(self, message, limit, actual): ... + +class Action(_Weakrefable): + def __init__(self, action_type: bytes | str, buf): ... + @property + def type(self) -> str: ... + @property + def body(self) -> pyarrow.Buffer: ... + def serialize(self): ... + @classmethod + def deserialize(cls, serialized) -> Action: ... + def __eq__(self, other: Action) -> bool: ... + +class ActionType(_ActionType): + def make_action(self, buf) -> Action: ... + +class Result(_Weakrefable): + def __init__(self, buf): ... + @property + def body(self) -> pyarrow.Buffer: ... + def serialize(self): ... + @classmethod + def deserialize(cls, serialized) -> Result: ... + def __eq__(self, other: Result) -> bool: ... + +class BasicAuth(_Weakrefable): + def __init__(self, username: str | None = None, password: str | None = None): ... + @property + def username(self): ... + @property + def password(self): ... + @staticmethod + def deserialize(serialized) -> BasicAuth: ... + def serialize(self): ... + def __eq__(self, other: BasicAuth) -> bool: ... + +class DescriptorType(enum.Enum): + UNKNOWN = 0 + PATH = 1 + CMD = 2 + +class FlightMethod(enum.Enum): + INVALID = 0 + HANDSHAKE = 1 + LIST_FLIGHTS = 2 + GET_FLIGHT_INFO = 3 + GET_SCHEMA = 4 + DO_GET = 5 + DO_PUT = 6 + DO_ACTION = 7 + LIST_ACTIONS = 8 + DO_EXCHANGE = 9 + +class FlightDescriptor(_Weakrefable): + def __init__(self): ... + @staticmethod + def for_path(*path) -> FlightDescriptor: ... + @staticmethod + def for_command(command) -> FlightDescriptor: ... + @property + def descriptor_type(self): ... + @property + def command(self): ... + @property + def path(self): ... + def serialize(self): ... + @classmethod + def deserialize(cls, serialized) -> FlightDescriptor: ... + def __eq__(self, other: FlightDescriptor) -> bool: ... + +class Ticket(_Weakrefable): + def __init__(self, ticket): ... + @property + def ticket(self): ... + def serialize(self): ... + @classmethod + def deserialize(cls, serialized) -> Ticket: ... + def __eq__(self, other: Ticket) -> bool: ... + +class Location(_Weakrefable): + def __init__(self, uri): ... + @property + def uri(self): ... + def equals(self, other: Location) -> bool: ... + def __eq__(self, other): ... + @staticmethod + def for_grpc_tcp(host, port) -> Location: ... + @staticmethod + def for_grpc_tls(host, port) -> Location: ... + @staticmethod + def for_grpc_unix(path) -> Location: ... + +class FlightEndpoint(_Weakrefable): + def __init__( + self, + ticket: Ticket | bytes, + locations, + expiration_time: pyarrow.TimestampScalar | None = None, + app_metadata: bytes | str = "", + ): ... + @property + def ticket(self) -> Ticket: ... + @property + def locations(self) -> list: ... + @property + def expiration_time(self): ... + @property + def app_metadata(self): ... + def serialize(self): ... + @classmethod + def deserialize(cls, serialized) -> FlightEndpoint: ... + def __eq__(self, other: FlightEndpoint) -> bool: ... + +class SchemaResult(_Weakrefable): + def __init__(self, schema: Schema): ... + @property + def schema(self) -> pyarrow.Schema: ... + def serialize(self): ... + @classmethod + def deserialize(cls, serialized) -> SchemaResult: ... + def __eq__(self, other: SchemaResult) -> bool: ... + +class FlightInfo(_Weakrefable): + def __init__( + self, + schema: Schema, + descriptor: FlightDescriptor, + endpoints: list[FlightEndpoint], + total_records: int | None = None, + total_bytes: int | None = None, + ordered: bool = False, + app_metadata: bytes | str = "", + ): ... + @property + def total_records(self): ... + @property + def total_bytes(self): ... + @property + def ordered(self): ... + @property + def app_metadata(self): ... + @property + def schema(self) -> None | pyarrow.Schema: ... + @property + def descriptor(self) -> FlightDescriptor: ... + @property + def endpoints(self) -> list: ... + def serialize(self): ... + @classmethod + def deserialize(cls, serialized) -> FlightInfo: ... + def __eq__(self, other: FlightInfo) -> bool: ... + +class FlightStreamChunk(_Weakrefable): + @property + def data(self) -> None | pyarrow.RecordBatch: ... + @property + def app_metadata(self) -> None | pyarrow.Buffer: ... + def __iter__(self): ... + +class _MetadataRecordBatchReader(_Weakrefable, _ReadPandasMixin): + def __iter__(self) -> Self: ... + def __next__(self): ... + @property + def schema(self) -> pyarrow.Schema: ... + def read_all(self) -> pyarrow.Table: ... + def read_chunk(self) -> FlightStreamChunk: ... + def to_reader(self) -> pyarrow.RecordBatchReader: ... + @property + def stats(self) -> pyarrow.ipc.ReadStats: ... + +class MetadataRecordBatchReader(_MetadataRecordBatchReader): ... + +class FlightStreamReader(MetadataRecordBatchReader): + def cancel(self) -> None: ... + def read_all(self) -> pyarrow.Table: ... + +class MetadataRecordBatchWriter(_CRecordBatchWriter): + def begin(self, schema: Schema, options=None) -> None: ... + def write_metadata(self, buf) -> None: ... + def write_batch(self, batch: RecordBatch) -> None: ... + def write_table( + self, table: Table, max_chunksize: int | None = None, **kwargs + ) -> None: ... + def close(self) -> None: ... + def write_with_metadata(self, batch: RecordBatch, buf: pyarrow.Buffer) -> None: ... + +class FlightStreamWriter(MetadataRecordBatchWriter): + def done_writing(self) -> None: ... + +class FlightMetadataReader(_Weakrefable): + def read(self) -> None | pyarrow.Buffer: ... + +class FlightMetadataWriter(_Weakrefable): + def write(self, message: pyarrow.Buffer) -> None: ... + +class AsyncioCall: + def __init__(self) -> None: ... + def as_awaitable(self) -> object: ... + def wakeup(self, result_or_exception) -> None: ... + +class AsyncioFlightClient: + def __init__(self, client: FlightClient) -> None: ... + async def get_flight_info( + self, descriptor: FlightDescriptor, *, options: FlightCallOptions | None = None + ): ... + +class FlightClient(_Weakrefable): + def __init__( + self, + location, + *, + tls_root_certs=None, + cert_chain=None, + private_key=None, + override_hostname=None, + middleware=None, + write_size_limit_bytes=None, + disable_server_verification=None, + generic_options=None, + ): ... + @property + def supports_async(self): ... + def as_async(self) -> None: ... + def wait_for_available(self, timeout: int = 5) -> None: ... + @classmethod + def connect( + cls, + location, + tls_root_certs=None, + cert_chain=None, + private_key=None, + override_hostname=None, + disable_server_verification=None, + ) -> FlightClient: ... + def authenticate( + self, auth_handler: ClientAuthHandler, options: FlightCallOptions | None = None + ) -> None: ... + def authenticate_basic_token( + self, username: str, password: str, options: FlightCallOptions | None = None + ): ... + def list_actions(self, options: FlightCallOptions | None = None) -> list: ... + def do_action( + self, action: str | tuple | Action, options: FlightCallOptions | None = None + ): ... + def list_flights( + self, criteria: bytes | None = None, options: FlightCallOptions | None = None + ) -> Iterator[FlightInfo]: ... + def get_flight_info( + self, descriptor: FlightDescriptor, options: FlightCallOptions | None = None + ) -> FlightInfo: ... + def get_schema( + self, descriptor: FlightDescriptor, options: FlightCallOptions | None = None + ) -> SchemaResult: ... + def do_get( + self, ticket: Ticket, options: FlightCallOptions | None = None + ) -> FlightStreamReader: ... + def do_put( + self, + descriptor: FlightDescriptor, + schema: Schema, + options: FlightCallOptions | None = None, + ) -> tuple: ... + def do_exchange( + self, descriptor: FlightDescriptor, options: FlightCallOptions | None = None + ) -> tuple: ... + def close(self) -> None: ... + def __del__(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type, exc_value, traceback) -> None: ... + +class FlightDataStream(_Weakrefable): ... + +class RecordBatchStream(FlightDataStream): + def __init__( + self, + data_source: pyarrow.RecordBatchReader | pyarrow.Table, + options: pyarrow.ipc.IpcWriteOptions | None = None, + ): ... + +class GeneratorStream(FlightDataStream): + def __init__( + self, + schema: pyarrow.Schema, + generator: Iterator[Any] | Iterable[Any], + options: pyarrow.ipc.IpcWriteOptions | None = None, + ): ... + +class ServerCallContext(_Weakrefable): + def peer_identity(self) -> bytes: ... + def peer(self) -> str: ... + def is_cancelled(self): ... + def add_header(self, key, value) -> None: ... + def add_trailer(self, key, value) -> None: ... + def get_middleware(self, key): ... + +class ServerAuthReader(_Weakrefable): + def read(self) -> bytes: ... + +class ServerAuthSender(_Weakrefable): + def write(self, message) -> None: ... + +class ClientAuthReader(_Weakrefable): + def read(self) -> bytes: ... + +class ClientAuthSender(_Weakrefable): + def write(self, message) -> None: ... + +class ServerAuthHandler(_Weakrefable): + def authenticate(self, outgoing: ServerAuthSender, incoming: ServerAuthReader): ... + def is_valid(self, token: bytes): ... + +class ClientAuthHandler(_Weakrefable): + def authenticate(self, outgoing: ClientAuthSender, incoming: ClientAuthReader): ... + def get_token(self): ... + +class CallInfo(_CallInfo): ... + +class ClientMiddlewareFactory(_Weakrefable): + def start_call(self, info: CallInfo) -> None: ... + +class ClientMiddleware(_Weakrefable): + def sending_headers(self) -> None: ... + def received_headers(self, headers: dict) -> None: ... + def call_completed(self, exception: ArrowException) -> None: ... + +class ServerMiddlewareFactory(_Weakrefable): + def start_call(self, info: CallInfo, headers: dict) -> None: ... + +class TracingServerMiddlewareFactory(ServerMiddlewareFactory): ... + +class ServerMiddleware(_Weakrefable): + def sending_headers(self) -> None: ... + def call_completed(self, exception: ArrowException) -> None: ... + +class TracingServerMiddleware(ServerMiddleware): + trace_context: Incomplete + __slots__ = ["trace_context"] + + def __init__(self, trace_context): ... + +class _ServerMiddlewareFactoryWrapper(ServerMiddlewareFactory): + def __init__(self, factories: dict[Any, Any]): ... + def start_call(self, info, headers) -> _ServerMiddlewareWrapper | None: ... + +class _ServerMiddlewareWrapper(ServerMiddleware): + def __init__(self, middleware: dict[Any, Any]): ... + def sending_headers(self): ... + def call_completed(self, exception) -> None: ... + +class _FlightServerFinalizer(_Weakrefable): + def finalize(self) -> None: ... + +class FlightServerBase(_Weakrefable): + def __init__( + self, + location=None, + auth_handler=None, + tls_certificates=None, + verify_client=None, + root_certificates=None, + middleware=None, + ): ... + @property + def port(self): ... + def list_flights( + self, context: ServerCallContext, criteria: bytes + ) -> Iterator[FlightInfo]: ... + def get_flight_info( + self, context: ServerCallContext, descriptor: FlightDescriptor + ) -> FlightInfo: ... + def get_schema( + self, context: ServerCallContext, descriptor: FlightDescriptor + ) -> pyarrow.Schema: ... + def do_put( + self, + context: ServerCallContext, + descriptor: FlightDescriptor, + reader: MetadataRecordBatchReader, + writer: FlightMetadataWriter, + ): ... + def do_get( + self, context: ServerCallContext, ticket: Ticket + ) -> FlightDataStream: ... + def do_exchange( + self, + context: ServerCallContext, + descriptor: FlightDescriptor, + reader: MetadataRecordBatchReader, + writer: MetadataRecordBatchWriter, + ): ... + def list_actions( + self, context: ServerCallContext + ) -> Iterator[ActionType] | tuple: ... + def do_action( + self, context: ServerCallContext, action: Action + ) -> Iterator[bytes]: ... + def serve(self) -> None: ... + def run(self) -> None: ... + def shutdown(self) -> None: ... + def wait(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type, exc_value, traceback) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/_fs.pyi b/python/pyarrow-stubs/pyarrow/_fs.pyi new file mode 100644 index 000000000000..be7557e82fdf --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_fs.pyi @@ -0,0 +1,193 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_fs.pyx + +import pyarrow +from typing import Any +from collections.abc import Mapping +from pyarrow.lib import _Weakrefable +from abc import ABC, abstractmethod +from datetime import datetime +from enum import IntEnum +from pyarrow.lib import * + +def _file_type_to_string(ty) -> str: ... +def _copy_files( + source_fs: FileSystem, + source_path: str, + destination_fs: FileSystem, + destination_path: str, + chunk_size: int, + use_threads: bool, +) -> None: ... +def _copy_files_selector( + source_fs: FileSystem, + source_sel: FileSelector, + destination_fs: FileSystem, + destination_base_dir: str, + chunk_size: int, + use_threads: bool, +) -> None: ... + +class FileType(IntEnum): + NotFound = ... + Unknown = ... + File = ... + Directory = ... + +class FileInfo(_Weakrefable): + def __init__( + self, path, type: FileType = ..., *, mtime=None, mtime_ns=None, size=None + ): ... + @property + def type(self) -> FileType: ... + @property + def is_file(self) -> bool: ... + @property + def path(self) -> str: ... + @property + def base_name(self) -> str: ... + @property + def size(self) -> int | None: ... + @property + def extension(self) -> str: ... + @property + def mtime(self): ... + @property + def mtime_ns(self) -> int | None: ... + +class FileSelector(_Weakrefable): + def __init__( + self, base_dir, allow_not_found: bool = False, recursive: bool = False + ): ... + @property + def base_dir(self) -> None: ... + @base_dir.setter + def base_dir(self, base_dir) -> None: ... + @property + def allow_not_found(self) -> None: ... + @allow_not_found.setter + def allow_not_found(self, allow_not_found: bool) -> None: ... + @property + def recursive(self) -> None: ... + @recursive.setter + def recursive(self, recursive: bool) -> None: ... + +class FileSystem(_Weakrefable): + def __init__(self): ... + @staticmethod + def _from_uri(uri): ... + @staticmethod + def _fsspec_from_uri(uri) -> tuple: ... + @staticmethod + def _native_from_uri(uri) -> tuple: ... + @staticmethod + def from_uri(uri: str): ... + def equals(self, other: FileSystem) -> bool: ... + def __eq__(self, other): ... + @property + def type_name(self) -> str: ... + def get_file_info(self, paths_or_selector) -> FileInfo | list: ... + def create_dir(self, path: str, *, recursive: bool = True) -> None: ... + def delete_dir(self, path: str) -> None: ... + def delete_dir_contents( + self, path: str, *, accept_root_dir: bool = False, missing_dir_ok: bool = False + ) -> None: ... + def move(self, src: str, dest: str) -> None: ... + def copy_file(self, src: str, dest: str) -> None: ... + def delete_file(self, path: str) -> None: ... + def _wrap_input_stream( + self, stream, path, compression, buffer_size + ) -> pyarrow.BufferedInputStream: ... + def _wrap_output_stream( + self, stream, path, compression, buffer_size + ) -> pyarrow.BufferedOutputStream: ... + def open_input_file(self, path: str) -> pyarrow.NativeFile: ... + def open_input_stream( + self, + path: str, + compression: str | None = "detect", + buffer_size: int | None = None, + ) -> pyarrow.NativeFile: ... + def open_output_stream( + self, + path: str, + compression: str | None = "detect", + buffer_size: int | None = None, + metadata: dict | None = None, + ) -> pyarrow.NativeFile: ... + def open_append_stream( + self, + path: str, + compression: str | None = "detect", + buffer_size: int | None = None, + metadata: dict | None = None, + ) -> pyarrow.NativeFile: ... + def normalize_path(self, path: str) -> str: ... + +class LocalFileSystem(FileSystem): + def __init__(self, *, use_mmap: bool = False): ... + def __reduce__(self) -> tuple: ... + +class SubTreeFileSystem(FileSystem): + def __init__(self, base_path, base_fs: FileSystem): ... + def __reduce__(self) -> tuple: ... + @property + def base_path(self) -> str: ... + @property + def base_fs(self) -> FileSystem: ... + +class _MockFileSystem(FileSystem): + def __init__(self, current_time: datetime | None = None): ... + +class PyFileSystem(FileSystem): + def __init__(self, handler): ... + @property + def handler(self) -> FileSystemHandler: ... + def __reduce__(self) -> tuple: ... + +class FileSystemHandler(ABC): + @abstractmethod + def get_type_name(self) -> None: ... + @abstractmethod + def get_file_info(self, paths: list[str]) -> None: ... + @abstractmethod + def get_file_info_selector(self, selector: FileSelector) -> None: ... + @abstractmethod + def create_dir(self, path: str, recursive: bool) -> None: ... + @abstractmethod + def delete_dir(self, path: str) -> None: ... + @abstractmethod + def delete_dir_contents(self, path: str, missing_dir_ok: bool = False) -> None: ... + @abstractmethod + def delete_root_dir_contents(self) -> None: ... + @abstractmethod + def delete_file(self, path: str) -> None: ... + @abstractmethod + def move(self, src: str, dest: str) -> None: ... + @abstractmethod + def copy_file(self, src: str, dest: str) -> None: ... + @abstractmethod + def open_input_stream(self, path: str) -> None: ... + @abstractmethod + def open_input_file(self, path: str) -> None: ... + @abstractmethod + def open_output_stream(self, path: str, metadata: Mapping[Any, Any]) -> None: ... + @abstractmethod + def open_append_stream(self, path: str, metadata: Mapping[Any, Any]) -> None: ... + @abstractmethod + def normalize_path(self, path: str) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/_gcsfs.pyi b/python/pyarrow-stubs/pyarrow/_gcsfs.pyi new file mode 100644 index 000000000000..aa9c52ac5e80 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_gcsfs.pyi @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_gcsfs.pyx + +from pyarrow._fs import FileSystem + +class GcsFileSystem(FileSystem): + def __init__( + self, + *, + anonymous: bool = False, + access_token=None, + target_service_account=None, + credential_token_expiration=None, + default_bucket_location: str = "US", + scheme=None, + endpoint_override=None, + default_metadata=None, + retry_time_limit=None, + project_id=None, + ): ... + def _expiration_datetime_from_options(self): ... + @staticmethod + def _reconstruct(kwargs) -> GcsFileSystem: ... + def __reduce__(self) -> tuple: ... + @property + def default_bucket_location(self) -> str: ... + @property + def project_id(self) -> str: ... diff --git a/python/pyarrow-stubs/pyarrow/_hdfs.pyi b/python/pyarrow-stubs/pyarrow/_hdfs.pyi new file mode 100644 index 000000000000..0c6b9ee65bb8 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_hdfs.pyi @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_hdfs.pyx + +from pyarrow._fs import FileSystem + +def _have_libhdfs() -> bool: ... + +class HadoopFileSystem(FileSystem): + def __init__( + self, + host: str, + port: int = 8020, + *, + user: str | None = None, + replication: int = 3, + buffer_size: int = 0, + default_block_size=None, + kerb_ticket=None, + extra_conf=None, + ): ... + @staticmethod + def from_uri(uri: str) -> HadoopFileSystem: ... + @staticmethod + def _reconstruct(kwargs) -> HadoopFileSystem: ... + def __reduce__(self) -> tuple: ... diff --git a/python/pyarrow-stubs/pyarrow/_json.pyi b/python/pyarrow-stubs/pyarrow/_json.pyi new file mode 100644 index 000000000000..cb4f9db27f16 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_json.pyi @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_json.pyx + +import pyarrow +import pyarrow.json +from pyarrow.lib import MemoryPool, RecordBatchReader, Schema, _Weakrefable + +def read_json( + input_file, + read_options: pyarrow.json.ReadOptions | None = None, + parse_options: pyarrow.json.ParseOptions | None = None, + memory_pool: MemoryPool | None = None, +) -> pyarrow.Table: ... +def open_json( + input_file, + read_options: pyarrow.json.ReadOptions | None = None, + parse_options: pyarrow.json.ParseOptions | None = None, + memory_pool: MemoryPool | None = None, +) -> JSONStreamingReader: ... + +class ReadOptions(_Weakrefable): + __slots__ = () + + def __init__(self, use_threads=None, block_size=None): ... + @property + def use_threads(self) -> None: ... + @use_threads.setter + def use_threads(self, value) -> None: ... + @property + def block_size(self) -> None: ... + @block_size.setter + def block_size(self, value) -> None: ... + def __reduce__(self) -> tuple: ... + def equals(self, other: ReadOptions) -> bool: ... + def __eq__(self, other): ... + +class ParseOptions(_Weakrefable): + __slots__ = () + + def __init__( + self, + explicit_schema=None, + newlines_in_values=None, + unexpected_field_behavior=None, + ): ... + def __reduce__(self) -> tuple: ... + @property + def explicit_schema(self) -> None: ... + @explicit_schema.setter + def explicit_schema(self, value) -> None: ... + @property + def newlines_in_values(self) -> None: ... + @newlines_in_values.setter + def newlines_in_values(self, value) -> None: ... + @property + def unexpected_field_behavior(self) -> None: ... + @unexpected_field_behavior.setter + def unexpected_field_behavior(self, value) -> None: ... + def equals(self, other: ParseOptions) -> bool: ... + def __eq__(self, other): ... + def _repr_base(self) -> str: ... + +class JSONStreamingReader(RecordBatchReader): + schema: Schema + + def __init__(self): ... diff --git a/python/pyarrow-stubs/pyarrow/_orc.pyi b/python/pyarrow-stubs/pyarrow/_orc.pyi new file mode 100644 index 000000000000..151d4c041641 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_orc.pyi @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_orc.pyx + +import pyarrow +from typing import TypedDict +from _typeshed import Incomplete +from pyarrow.lib import MemoryPool, Table, _Weakrefable + +class ORCReader(_Weakrefable): + def __init__(self, memory_pool: MemoryPool | None = None): ... + def open(self, source: object, use_memory_map: bool = True) -> None: ... + def metadata(self) -> pyarrow.KeyValueMetadata: ... + def schema(self) -> pyarrow.Schema: ... + def nrows(self): ... + def nstripes(self): ... + def file_version(self): ... + def software_version(self) -> str: ... + def compression(self): ... + def compression_size(self): ... + def row_index_stride(self): ... + def writer(self): ... + def writer_version(self): ... + def nstripe_statistics(self): ... + def content_length(self): ... + def stripe_statistics_length(self): ... + def file_footer_length(self): ... + def file_postscript_length(self): ... + def file_length(self): ... + def serialized_file_tail(self): ... + def read_stripe(self, n, columns=None) -> pyarrow.RecordBatch: ... + def read(self, columns=None) -> pyarrow.Table: ... + +class ORCWriter(_Weakrefable): + def open( + self, + where: object, + *, + file_version=None, + batch_size=None, + stripe_size=None, + compression=None, + compression_block_size=None, + compression_strategy=None, + row_index_stride=None, + padding_tolerance=None, + dictionary_key_size_threshold=None, + bloom_filter_columns=None, + bloom_filter_fpp=None, + ) -> None: ... + def write(self, table: Table) -> None: ... + def close(self) -> None: ... + +class WriteOptions(TypedDict): + batch_size: int + file_version: Incomplete + stripe_size: int + compression: Incomplete + compression_block_size: int + compression_strategy: Incomplete + row_index_stride: int + padding_tolerance: float + dictionary_key_size_threshold: float + bloom_filter_columns: list[int] + bloom_filter_fpp: float diff --git a/python/pyarrow-stubs/pyarrow/_parquet.pyi b/python/pyarrow-stubs/pyarrow/_parquet.pyi new file mode 100644 index 000000000000..4c4b78ece3d1 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_parquet.pyi @@ -0,0 +1,351 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_parquet.pyx + +import pyarrow +from collections.abc import Iterator +from typing import Any +from pyarrow.lib import Buffer, MemoryPool, Schema, Table, _Weakrefable + +_DEFAULT_ROW_GROUP_SIZE: int +_MAX_ROW_GROUP_SIZE: int + +def _reconstruct_filemetadata(serialized: Buffer) -> FileMetaData: ... + +class Statistics(_Weakrefable): + def __init__(self): ... + def to_dict(self) -> dict: ... + def __eq__(self, other): ... + def equals(self, other: Statistics) -> bool: ... + @property + def has_min_max(self): ... + @property + def has_null_count(self): ... + @property + def has_distinct_count(self): ... + @property + def min_raw(self): ... + @property + def max_raw(self): ... + @property + def min(self): ... + @property + def max(self): ... + @property + def null_count(self): ... + @property + def distinct_count(self): ... + @property + def num_values(self): ... + @property + def physical_type(self): ... + @property + def logical_type(self): ... + @property + def converted_type(self): ... + +class ParquetLogicalType(_Weakrefable): + def __init__(self): ... + def to_json(self) -> str: ... + @property + def type(self): ... + +class GeoStatistics(_Weakrefable): + def __init__(self): ... + def to_dict(self) -> dict: ... + @property + def geospatial_types(self) -> None | list: ... + @property + def xmin(self): ... + @property + def xmax(self): ... + @property + def ymin(self): ... + @property + def ymax(self): ... + @property + def zmin(self): ... + @property + def zmax(self): ... + @property + def mmin(self): ... + @property + def mmax(self): ... + def _x_valid(self): ... + def _y_valid(self): ... + def _z_valid(self): ... + def _m_valid(self): ... + +class ColumnChunkMetaData(_Weakrefable): + def __init__(self): ... + def to_dict(self) -> dict: ... + def __eq__(self, other): ... + def equals(self, other: ColumnChunkMetaData) -> bool: ... + @property + def file_offset(self): ... + @property + def file_path(self) -> str: ... + @property + def physical_type(self): ... + @property + def num_values(self): ... + @property + def path_in_schema(self) -> str: ... + @property + def is_stats_set(self): ... + @property + def statistics(self) -> None | Statistics: ... + @property + def is_geo_stats_set(self): ... + @property + def geo_statistics(self) -> None | GeoStatistics: ... + @property + def compression(self): ... + @property + def encodings(self) -> tuple: ... + @property + def has_dictionary_page(self) -> bool: ... + @property + def dictionary_page_offset(self): ... + @property + def data_page_offset(self): ... + @property + def has_index_page(self): ... + @property + def index_page_offset(self): ... + @property + def total_compressed_size(self): ... + @property + def total_uncompressed_size(self): ... + @property + def bloom_filter_offset(self): ... + @property + def bloom_filter_length(self): ... + @property + def has_offset_index(self): ... + @property + def has_column_index(self): ... + @property + def metadata(self): ... + +class SortingColumn: + def __init__( + self, column_index: int, descending: bool = False, nulls_first: bool = False + ): ... + @classmethod + def from_ordering( + cls, schema: Schema, sort_keys, null_placement: str = "at_end" + ) -> tuple: ... + @staticmethod + def to_ordering( + schema: Schema, sorting_columns: tuple[SortingColumn, ...] + ) -> tuple: ... + def __eq__(self, other: SortingColumn): ... + def __hash__(self): ... + @property + def column_index(self): ... + @property + def descending(self): ... + @property + def nulls_first(self): ... + def to_dict(self) -> dict: ... + +class RowGroupMetaData(_Weakrefable): + def __init__(self): ... + def __reduce__(self) -> tuple: ... + def __eq__(self, other): ... + def equals(self, other: RowGroupMetaData) -> bool: ... + def column(self, i: int) -> ColumnChunkMetaData: ... + def to_dict(self) -> dict: ... + @property + def num_columns(self): ... + @property + def num_rows(self): ... + @property + def total_byte_size(self): ... + @property + def sorting_columns(self) -> tuple: ... + +class FileMetaData(_Weakrefable): + def __init__(self): ... + def __reduce__(self) -> tuple: ... + def __hash__(self): ... + def to_dict(self) -> dict: ... + def __eq__(self, other): ... + def equals(self, other: FileMetaData) -> bool: ... + @property + def schema(self): ... + @property + def serialized_size(self): ... + @property + def num_columns(self): ... + @property + def num_rows(self): ... + @property + def num_row_groups(self): ... + @property + def format_version(self) -> str: ... + @property + def created_by(self) -> str: ... + @property + def metadata(self): ... + def row_group(self, i: int) -> RowGroupMetaData: ... + def set_file_path(self, path: str) -> None: ... + def append_row_groups(self, other: FileMetaData) -> None: ... + def write_metadata_file(self, where) -> None: ... + +class ParquetSchema(_Weakrefable): + def __init__(self, container: FileMetaData): ... + def __reduce__(self) -> tuple: ... + def __len__(self): ... + def __getitem__(self, i): ... + def __hash__(self): ... + @property + def names(self) -> list: ... + def to_arrow_schema(self) -> pyarrow.Schema: ... + def __eq__(self, other): ... + def equals(self, other: ParquetSchema) -> bool: ... + def column(self, i: int) -> ColumnSchema: ... + +class ColumnSchema(_Weakrefable): + def __init__(self, schema: ParquetSchema, index: int): ... + def __eq__(self, other): ... + def __reduce__(self) -> tuple: ... + def equals(self, other: ColumnSchema) -> bool: ... + @property + def name(self) -> str: ... + @property + def path(self) -> str: ... + @property + def max_definition_level(self): ... + @property + def max_repetition_level(self): ... + @property + def physical_type(self): ... + @property + def logical_type(self): ... + @property + def converted_type(self): ... + @property + def length(self): ... + @property + def precision(self): ... + @property + def scale(self): ... + +class ParquetReader(_Weakrefable): + _column_idx_map: Any + + def __init__(self, memory_pool: MemoryPool | None = None): ... + def open( + self, + source: object, + *, + use_memory_map: bool = False, + read_dictionary=None, + binary_type: pyarrow.DataType | None = None, + list_type=None, + metadata: FileMetaData | None = None, + buffer_size: int = 0, + pre_buffer: bool = True, + coerce_int96_timestamp_unit: str | None = None, + decryption_properties: FileDecryptionProperties | None = None, + thrift_string_size_limit: int | None = None, + thrift_container_size_limit: int | None = None, + schema_depth_limit: int | None = None, + page_checksum_verification: bool = False, + arrow_extensions_enabled: bool = False, + ) -> None: ... + @property + def column_paths(self) -> list: ... + @property + def metadata(self): ... + @property + def schema_arrow(self) -> pyarrow.Schema: ... + @property + def num_row_groups(self): ... + def set_use_threads(self, use_threads: bool) -> None: ... + def set_batch_size(self, batch_size: int) -> None: ... + def iter_batches( + self, + batch_size: int, + row_groups: list[int], + column_indices: list[int] | None = None, + use_threads: bool = True, + ) -> Iterator[pyarrow.RecordBatch]: ... + def read_row_group( + self, i: int, column_indices: list[int] | None = None, use_threads: bool = True + ) -> pyarrow.Table: ... + def read_row_groups( + self, + row_groups: list[int], + column_indices: list[int] | None = None, + use_threads: bool = True, + ) -> pyarrow.Table: ... + def read_all( + self, column_indices: list[int] | None = None, use_threads: bool = True + ) -> pyarrow.Table: ... + def scan_contents( + self, column_indices: list[int] | None = None, batch_size: int = 65536 + ) -> int: ... + def column_name_idx(self, column_name: str) -> int: ... + def read_column(self, column_index: int) -> pyarrow.ChunkedArray: ... + def close(self) -> None: ... + @property + def closed(self): ... + +class ParquetWriter(_Weakrefable): + def __init__( + self, + where, + schema: Schema, + use_dictionary=None, + compression=None, + version=None, + write_statistics=None, + memory_pool: MemoryPool | None = None, + use_deprecated_int96_timestamps: bool = False, + coerce_timestamps=None, + data_page_size=None, + max_rows_per_page=None, + allow_truncated_timestamps: bool = False, + compression_level=None, + use_byte_stream_split: bool = False, + column_encoding=None, + writer_engine_version=None, + data_page_version=None, + use_compliant_nested_type: bool = True, + encryption_properties=None, + write_batch_size=None, + dictionary_pagesize_limit=None, + store_schema: bool = True, + write_page_index: bool = False, + write_page_checksum: bool = False, + sorting_columns=None, + store_decimal_as_integer: bool = False, + use_content_defined_chunking: bool = False, + write_time_adjusted_to_utc: bool = False, + bloom_filter_options=None, + ): ... + def close(self) -> None: ... + def write_table(self, table: Table, row_group_size=None) -> None: ... + def add_key_value_metadata(self, key_value_metadata) -> None: ... + @property + def metadata(self) -> FileMetaData: ... + +class FileEncryptionProperties: ... +class FileDecryptionProperties: ... diff --git a/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi b/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi new file mode 100644 index 000000000000..ee2e916b360d --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi @@ -0,0 +1,195 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_parquet_encryption.pyx + +import pyarrow +import pyarrow.parquet +from typing import Any +from pyarrow._fs import FileSystem +from pyarrow.lib import _Weakrefable + +def create_decryption_properties( + footer_key: bytes, + *, + aad_prefix: bytes | None = None, + check_footer_integrity: bool = True, + allow_plaintext_files: bool = False, +) -> pyarrow.parquet.FileDecryptionProperties: ... +def create_encryption_properties( + footer_key: bytes, + *, + aad_prefix: bytes | None = None, + store_aad_prefix: bool = True, + encryption_algorithm: str = "AES_GCM_V1", + plaintext_footer: bool = False, +) -> pyarrow.parquet.FileEncryptionProperties: ... + +class EncryptionConfiguration(_Weakrefable): + __slots__ = () + + def __init__( + self, + footer_key, + *, + column_keys=None, + uniform_encryption=None, + encryption_algorithm=None, + plaintext_footer=None, + double_wrapping=None, + cache_lifetime=None, + internal_key_material=None, + data_key_length_bits=None, + ): ... + @property + def footer_key(self) -> str: ... + @property + def column_keys(self) -> None: ... + @column_keys.setter + def column_keys(self, value: dict[Any, Any]) -> None: ... + @property + def uniform_encryption(self) -> None: ... + @uniform_encryption.setter + def uniform_encryption(self, value) -> None: ... + @property + def encryption_algorithm(self) -> None: ... + @encryption_algorithm.setter + def encryption_algorithm(self, value) -> None: ... + @property + def plaintext_footer(self) -> None: ... + @plaintext_footer.setter + def plaintext_footer(self, value) -> None: ... + @property + def double_wrapping(self) -> None: ... + @double_wrapping.setter + def double_wrapping(self, value) -> None: ... + @property + def cache_lifetime(self) -> None: ... + @cache_lifetime.setter + def cache_lifetime(self, value) -> None: ... + @property + def internal_key_material(self) -> None: ... + @internal_key_material.setter + def internal_key_material(self, value) -> None: ... + @property + def data_key_length_bits(self) -> None: ... + @data_key_length_bits.setter + def data_key_length_bits(self, value) -> None: ... + +class DecryptionConfiguration(_Weakrefable): + __slots__ = () + + def __init__(self, *, cache_lifetime=None, read_kms_url=None): ... + @property + def cache_lifetime(self) -> None: ... + @cache_lifetime.setter + def cache_lifetime(self, value) -> None: ... + @property + def read_kms_url(self) -> None: ... + @read_kms_url.setter + def read_kms_url(self, value) -> None: ... + +class KmsConnectionConfig(_Weakrefable): + __slots__ = () + + def __init__( + self, + *, + kms_instance_id=None, + kms_instance_url=None, + key_access_token=None, + custom_kms_conf=None, + ): ... + @property + def kms_instance_id(self) -> None: ... + @kms_instance_id.setter + def kms_instance_id(self, value) -> None: ... + @property + def kms_instance_url(self) -> None: ... + @kms_instance_url.setter + def kms_instance_url(self, value) -> None: ... + @property + def key_access_token(self) -> None: ... + @key_access_token.setter + def key_access_token(self, value) -> None: ... + @property + def custom_kms_conf(self) -> None: ... + @custom_kms_conf.setter + def custom_kms_conf(self, value: dict[Any, Any]) -> None: ... + def refresh_key_access_token(self, value) -> None: ... + +class KmsClient(_Weakrefable): + def __init__(self): ... + def wrap_key(self, key_bytes, master_key_identifier): ... + def unwrap_key(self, wrapped_key, master_key_identifier): ... + +class CryptoFactory(_Weakrefable): + __slots__ = () + + def __init__(self, kms_client_factory): ... + def file_encryption_properties( + self, + kms_connection_config: KmsConnectionConfig, + encryption_config: EncryptionConfiguration, + parquet_file_path=None, + filesystem: FileSystem | None = None, + ) -> pyarrow.parquet.FileEncryptionProperties: ... + def file_decryption_properties( + self, + kms_connection_config: KmsConnectionConfig, + decryption_config: DecryptionConfiguration | None = None, + parquet_file_path=None, + filesystem: FileSystem | None = None, + ) -> pyarrow.parquet.FileDecryptionProperties: ... + def remove_cache_entries_for_token(self, access_token) -> None: ... + def remove_cache_entries_for_all_tokens(self) -> None: ... + def rotate_master_keys( + self, + kms_connection_config: KmsConnectionConfig, + parquet_file_path, + filesystem: FileSystem | None = None, + double_wrapping: bool = True, + cache_lifetime_seconds: float = 600, + read_kms_url: bool = False, + ) -> None: ... + +class KeyMaterial(_Weakrefable): + @property + def is_footer_key(self): ... + @property + def is_double_wrapped(self): ... + @property + def master_key_id(self) -> str: ... + @property + def wrapped_dek(self) -> str: ... + @property + def kek_id(self) -> str: ... + @property + def wrapped_kek(self) -> str: ... + @property + def kms_instance_id(self) -> str: ... + @property + def kms_instance_url(self) -> str: ... + @staticmethod + def parse(key_material_string: bytes) -> KeyMaterial: ... + +class FileSystemKeyMaterialStore(_Weakrefable): + def get_key_material(self, key_id): ... + def get_key_id_set(self): ... + @classmethod + def for_file( + cls, parquet_file_path, filesystem: FileSystem | None = None + ) -> FileSystemKeyMaterialStore: ... diff --git a/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi b/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi new file mode 100644 index 000000000000..7766baa44734 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_pyarrow_cpp_tests.pyx + +def get_cpp_tests() -> list: ... + +class CppTestCase: + @property + def name(self) -> str: ... + def __call__(self) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/_s3fs.pyi b/python/pyarrow-stubs/pyarrow/_s3fs.pyi new file mode 100644 index 000000000000..199d368b2376 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_s3fs.pyi @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_s3fs.pyx + +from enum import IntEnum +from pyarrow._fs import FileSystem + +def initialize_s3( + log_level: S3LogLevel = ..., num_event_loop_threads: int = 1 +) -> None: ... +def ensure_s3_initialized() -> None: ... +def finalize_s3() -> None: ... +def ensure_s3_finalized() -> None: ... +def resolve_s3_region(bucket: str) -> str: ... + +class S3LogLevel(IntEnum): + Off = ... + Fatal = ... + Error = ... + Warn = ... + Info = ... + Debug = ... + Trace = ... + +class S3RetryStrategy: + def __init__(self, max_attempts: int = 3): ... + +class AwsStandardS3RetryStrategy(S3RetryStrategy): ... +class AwsDefaultS3RetryStrategy(S3RetryStrategy): ... + +class S3FileSystem(FileSystem): + def __init__( + self, + *, + access_key=None, + secret_key=None, + session_token=None, + anonymous: bool = False, + region=None, + request_timeout=None, + connect_timeout=None, + scheme=None, + endpoint_override=None, + background_writes: bool = True, + default_metadata=None, + role_arn=None, + session_name=None, + external_id=None, + load_frequency: int = 900, + proxy_options=None, + allow_delayed_open: bool = False, + allow_bucket_creation: bool = False, + allow_bucket_deletion: bool = False, + check_directory_existence_before_creation: bool = False, + retry_strategy: S3RetryStrategy = ..., + force_virtual_addressing: bool = False, + tls_ca_file_path=None, + ): ... + @staticmethod + def _reconstruct(kwargs) -> S3FileSystem: ... + def __reduce__(self) -> tuple: ... + @property + def region(self) -> str: ... diff --git a/python/pyarrow-stubs/pyarrow/_substrait.pyi b/python/pyarrow-stubs/pyarrow/_substrait.pyi new file mode 100644 index 000000000000..ea3c8cd58d80 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_substrait.pyi @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/_substrait.pyx + +import pyarrow +import pyarrow.compute +from pyarrow.lib import _Weakrefable +from pyarrow.lib import * +from types import ModuleType + +def run_query( + plan, *, table_provider: object | None = None, use_threads: bool = True +) -> pyarrow.RecordBatchReader: ... +def _parse_json_plan(plan: bytes) -> pyarrow.Buffer: ... +def serialize_schema(schema: pyarrow.Schema) -> SubstraitSchema: ... +def _serialize_namedstruct_schema(schema): ... +def deserialize_schema( + buf: pyarrow.Buffer | bytes | SubstraitSchema, +) -> pyarrow.Schema: ... +def serialize_expressions( + exprs: list[pyarrow.compute.Expression], + names: list[str], + schema: pyarrow.Schema, + *, + allow_arrow_extensions: bool = False, +) -> pyarrow.Buffer: ... +def deserialize_expressions(buf: pyarrow.Buffer | bytes) -> BoundExpressions: ... +def get_supported_functions() -> list: ... + +py_substrait: ModuleType | None + +class SubstraitSchema: + def __init__(self, schema, expression): ... + def to_pysubstrait(self): ... + +class BoundExpressions(_Weakrefable): + def __init__(self): ... + @property + def schema(self) -> pyarrow.Schema: ... + @property + def expressions(self) -> dict: ... + @classmethod + def from_substrait(cls, message) -> BoundExpressions: ... diff --git a/python/pyarrow-stubs/pyarrow/acero.pyi b/python/pyarrow-stubs/pyarrow/acero.pyi new file mode 100644 index 000000000000..6a94d7377622 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/acero.pyi @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyarrow._compute import CountOptions, PivotWiderOptions +from pyarrow._dataset import FileSystemDataset, InMemoryDataset, UnionDataset +import pyarrow +from typing import Any +from _typeshed import Incomplete +import pyarrow.compute +import pyarrow.dataset +from pyarrow._acero import ( + AggregateNodeOptions as AggregateNodeOptions, + AsofJoinNodeOptions as AsofJoinNodeOptions, + Declaration as Declaration, + ExecNodeOptions as ExecNodeOptions, + FilterNodeOptions as FilterNodeOptions, + HashJoinNodeOptions as HashJoinNodeOptions, + OrderByNodeOptions as OrderByNodeOptions, + ProjectNodeOptions as ProjectNodeOptions, + RecordBatchReaderSourceNodeOptions as RecordBatchReaderSourceNodeOptions, + TableSourceNodeOptions as TableSourceNodeOptions, +) +from pyarrow._dataset import ScanNodeOptions as ScanNodeOptions +from pyarrow.compute import Expression as Expression, field as field +from pyarrow.lib import RecordBatch as RecordBatch, Table as Table, array as array + +ds: Incomplete + +def _dataset_to_decl( + dataset: FileSystemDataset | UnionDataset | InMemoryDataset, + use_threads: bool = True, + implicit_ordering: bool = False, +) -> Declaration: ... +def _perform_join( + join_type: str, + left_operand: Table | pyarrow.dataset.Dataset, + left_keys: str | list[str], + right_operand: Table | pyarrow.dataset.Dataset, + right_keys: str | list[str], + left_suffix: str | None = None, + right_suffix: str | None = None, + use_threads: bool = True, + coalesce_keys: bool = False, + output_type=..., + filter_expression: pyarrow.compute.Expression | None = None, +) -> Table | pyarrow.dataset.InMemoryDataset: ... +def _perform_join_asof( + left_operand: Table | pyarrow.dataset.Dataset, + left_on: str, + left_by: list[Any] | str | list[str], + right_operand: Table | pyarrow.dataset.Dataset, + right_on: str | list[str], + right_by: list[Any] | str | list[str], + tolerance: int, + use_threads: bool = True, + output_type=..., +) -> Table | pyarrow.dataset.InMemoryDataset: ... +def _filter_table( + table: Table | RecordBatch, expression: Expression +) -> Table | RecordBatch: ... +def _sort_source( + table_or_dataset: FileSystemDataset | InMemoryDataset, + sort_keys: list[tuple[str, str, str]], + output_type=..., + **kwargs, +) -> InMemoryDataset: ... +def _group_by( + table: Table, + aggregates: list[ + tuple[tuple[str, str], str, None, str] + | tuple[list[Any], str, None, str] + | tuple[list[str], str, CountOptions, str] + | Any + | tuple[list[str], str, None, str] + | tuple[tuple[str, str], str, PivotWiderOptions, str] + ], + keys: list[str], + use_threads: bool = True, +) -> Table: ... + +class DatasetModuleStub: + class Dataset: ... + class InMemoryDataset: ... diff --git a/python/pyarrow-stubs/pyarrow/benchmark.pyi b/python/pyarrow-stubs/pyarrow/benchmark.pyi new file mode 100644 index 000000000000..68aa3ab3db99 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/benchmark.pyi @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyarrow.lib import benchmark_PandasObjectIsNull as benchmark_PandasObjectIsNull diff --git a/python/pyarrow-stubs/pyarrow/cffi.pyi b/python/pyarrow-stubs/pyarrow/cffi.pyi new file mode 100644 index 000000000000..69456bd3fc53 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/cffi.pyi @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from _typeshed import Incomplete + +c_source: str +ffi: Incomplete diff --git a/python/pyarrow-stubs/pyarrow/compute.pyi b/python/pyarrow-stubs/pyarrow/compute.pyi new file mode 100644 index 000000000000..e2f51437889b --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/compute.pyi @@ -0,0 +1,1379 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyarrow.lib import ( + BinaryArray, + BinaryScalar, + Bool8Array, + BooleanArray, + ChunkedArray, + Date32Array, + Date64Array, + Decimal128Array, + DictionaryArray, + DoubleArray, + DoubleScalar, + DurationArray, + ExtensionArray, + FixedShapeTensorArray, + FixedSizeBinaryArray, + FixedSizeListArray, + FloatArray, + FloatScalar, + HalfFloatArray, + Int16Array, + Int16Scalar, + Int32Array, + Int64Array, + Int64Scalar, + Int8Array, + Int8Scalar, + JsonArray, + LargeBinaryArray, + LargeListArray, + LargeStringArray, + ListArray, + MapArray, + NullArray, + NullScalar, + OpaqueArray, + RecordBatch, + StringArray, + StringScalar, + StringViewArray, + StructArray, + Table, + Time32Array, + Time64Array, + TimestampArray, + TimestampScalar, + UInt16Array, + UInt16Scalar, + UInt32Array, + UInt64Array, + UInt64Scalar, + UInt8Array, + UInt8Scalar, +) +from collections.abc import Sequence +import pyarrow.lib +from _typeshed import Incomplete +from pyarrow._compute import ( + ArraySortOptions as ArraySortOptions, + AssumeTimezoneOptions as AssumeTimezoneOptions, + CastOptions as CastOptions, + CountOptions as CountOptions, + CumulativeOptions as CumulativeOptions, + CumulativeSumOptions as CumulativeSumOptions, + DayOfWeekOptions as DayOfWeekOptions, + DictionaryEncodeOptions as DictionaryEncodeOptions, + ElementWiseAggregateOptions as ElementWiseAggregateOptions, + Expression as Expression, + ExtractRegexOptions as ExtractRegexOptions, + ExtractRegexSpanOptions as ExtractRegexSpanOptions, + FilterOptions as FilterOptions, + Function as Function, + FunctionOptions as FunctionOptions, + FunctionRegistry as FunctionRegistry, + HashAggregateFunction as HashAggregateFunction, + HashAggregateKernel as HashAggregateKernel, + IndexOptions as IndexOptions, + InversePermutationOptions as InversePermutationOptions, + JoinOptions as JoinOptions, + Kernel as Kernel, + ListFlattenOptions as ListFlattenOptions, + ListSliceOptions as ListSliceOptions, + MakeStructOptions as MakeStructOptions, + MapLookupOptions as MapLookupOptions, + MatchSubstringOptions as MatchSubstringOptions, + ModeOptions as ModeOptions, + NullOptions as NullOptions, + PadOptions as PadOptions, + PairwiseOptions as PairwiseOptions, + PartitionNthOptions as PartitionNthOptions, + PivotWiderOptions as PivotWiderOptions, + QuantileOptions as QuantileOptions, + RandomOptions as RandomOptions, + RankOptions as RankOptions, + RankQuantileOptions as RankQuantileOptions, + ReplaceSliceOptions as ReplaceSliceOptions, + ReplaceSubstringOptions as ReplaceSubstringOptions, + RoundBinaryOptions as RoundBinaryOptions, + RoundOptions as RoundOptions, + RoundTemporalOptions as RoundTemporalOptions, + RoundToMultipleOptions as RoundToMultipleOptions, + RunEndEncodeOptions as RunEndEncodeOptions, + ScalarAggregateFunction as ScalarAggregateFunction, + ScalarAggregateKernel as ScalarAggregateKernel, + ScalarAggregateOptions as ScalarAggregateOptions, + ScalarFunction as ScalarFunction, + ScalarKernel as ScalarKernel, + ScatterOptions as ScatterOptions, + SearchSortedOptions as SearchSortedOptions, + SelectKOptions as SelectKOptions, + SetLookupOptions as SetLookupOptions, + SkewOptions as SkewOptions, + SliceOptions as SliceOptions, + SortOptions as SortOptions, + SplitOptions as SplitOptions, + SplitPatternOptions as SplitPatternOptions, + StrftimeOptions as StrftimeOptions, + StrptimeOptions as StrptimeOptions, + StructFieldOptions as StructFieldOptions, + TDigestOptions as TDigestOptions, + TakeOptions as TakeOptions, + TrimOptions as TrimOptions, + UdfContext as UdfContext, + Utf8NormalizeOptions as Utf8NormalizeOptions, + VarianceOptions as VarianceOptions, + VectorFunction as VectorFunction, + VectorKernel as VectorKernel, + WeekOptions as WeekOptions, + WinsorizeOptions as WinsorizeOptions, + ZeroFillOptions as ZeroFillOptions, + call_function as call_function, + call_tabular_function as call_tabular_function, + function_registry as function_registry, + get_function as get_function, + list_functions as list_functions, + register_aggregate_function as register_aggregate_function, + register_scalar_function as register_scalar_function, + register_tabular_function as register_tabular_function, + register_vector_function as register_vector_function, +) +from typing import ClassVar + +def abs(x, *, memory_pool: Incomplete | None = ...): ... +def abs_checked(x, *, memory_pool: Incomplete | None = ...): ... +def acos(x, *, memory_pool: Incomplete | None = ...): ... +def acos_checked(x, *, memory_pool: Incomplete | None = ...): ... +def acosh(x, *, memory_pool: Incomplete | None = ...): ... +def acosh_checked(x, *, memory_pool: Incomplete | None = ...): ... +def add(x, y, *, memory_pool: Incomplete | None = ...): ... +def add_checked(x, y, *, memory_pool: Incomplete | None = ...): ... +def all( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def and_(x, y, *, memory_pool: Incomplete | None = ...): ... +def and_kleene(x, y, *, memory_pool: Incomplete | None = ...): ... +def and_not(x, y, *, memory_pool: Incomplete | None = ...): ... +def and_not_kleene(x, y, *, memory_pool: Incomplete | None = ...): ... +def any( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def approximate_median( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def array_filter( + array, + selection_filter, + null_selection_behavior: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def array_sort_indices( + array, + order: str = ..., + *, + null_placement: str = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def array_take( + array, + indices, + *, + boundscheck: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def ascii_capitalize(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_center( + strings, + width: int, + padding: str = ..., + lean_left_on_odd_padding: bool = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def ascii_is_alnum(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_alpha(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_decimal(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_lower(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_printable(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_space(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_title(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_upper(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_lower(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_lpad( + strings, + width: int, + padding: str = ..., + lean_left_on_odd_padding: bool = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def ascii_ltrim( + strings, + characters: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def ascii_ltrim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_reverse(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_rpad( + strings, + width: int, + padding: str = ..., + lean_left_on_odd_padding: bool = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def ascii_rtrim( + strings, + characters: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def ascii_rtrim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_split_whitespace( + strings, + *, + max_splits: Incomplete | None = ..., + reverse: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def ascii_swapcase(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_title(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_trim( + strings, + characters: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def ascii_trim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_upper(strings, *, memory_pool: Incomplete | None = ...): ... +def asin(x, *, memory_pool: Incomplete | None = ...): ... +def asin_checked(x, *, memory_pool: Incomplete | None = ...): ... +def asinh(x, *, memory_pool: Incomplete | None = ...): ... +def assume_timezone( + timestamps, + timezone: str, + *, + ambiguous: str = ..., + nonexistent: str = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def atan(x, *, memory_pool: Incomplete | None = ...): ... +def atan2(y, x, *, memory_pool: Incomplete | None = ...): ... +def atanh(x, *, memory_pool: Incomplete | None = ...): ... +def atanh_checked(x, *, memory_pool: Incomplete | None = ...): ... +def binary_join(strings, separator, *, memory_pool: Incomplete | None = ...): ... +def binary_join_element_wise( + *strings, + null_handling: str = ..., + null_replacement: str = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def binary_length(strings, *, memory_pool: Incomplete | None = ...): ... +def binary_repeat(strings, num_repeats, *, memory_pool: Incomplete | None = ...): ... +def binary_replace_slice( + strings, + start: int, + stop: int, + replacement: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def binary_reverse(strings, *, memory_pool: Incomplete | None = ...): ... +def binary_slice( + strings, + start: int, + stop: Incomplete | None = ..., + step: int = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def bit_wise_and(x, y, *, memory_pool: Incomplete | None = ...): ... +def bit_wise_not(x, *, memory_pool: Incomplete | None = ...): ... +def bit_wise_or(x, y, *, memory_pool: Incomplete | None = ...): ... +def bit_wise_xor(x, y, *, memory_pool: Incomplete | None = ...): ... +def case_when(cond, *cases, memory_pool: Incomplete | None = ...): ... +def cast( + arr: Table + | Int16Array + | StringViewArray + | UInt64Array + | Date32Array + | JsonArray + | DurationArray + | UInt16Array + | HalfFloatArray + | Date64Array + | Int64Scalar + | ChunkedArray + | StringScalar + | FloatScalar + | FixedShapeTensorArray + | DoubleScalar + | Int32Array + | DictionaryArray + | FloatArray + | TimestampArray + | Decimal128Array + | LargeStringArray + | ListArray + | NullArray + | StringArray + | OpaqueArray + | BinaryScalar + | Int8Array + | UInt32Array + | MapArray + | DoubleArray + | Time32Array + | LargeListArray + | BooleanArray + | ExtensionArray + | BinaryArray + | TimestampScalar + | Bool8Array + | UInt8Array + | Int64Array + | FixedSizeBinaryArray + | Time64Array + | StructArray + | FixedSizeListArray + | Int8Scalar, + target_type: Incomplete | None = ..., + safe: Incomplete | None = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +) -> pyarrow.Array: ... +def ceil(x, *, memory_pool: Incomplete | None = ...): ... +def ceil_temporal( + timestamps, + multiple: int = ..., + unit: str = ..., + *, + week_starts_monday: bool = ..., + ceil_is_strictly_greater: bool = ..., + calendar_based_origin: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def choose(indices, *values, memory_pool: Incomplete | None = ...): ... +def coalesce(*values, memory_pool: Incomplete | None = ...): ... +def cos(x, *, memory_pool: Incomplete | None = ...): ... +def cos_checked(x, *, memory_pool: Incomplete | None = ...): ... +def cosh(x, *, memory_pool: Incomplete | None = ...): ... +def count( + array, + mode: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def count_distinct( + array, + mode: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def count_substring( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def count_substring_regex( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def cumulative_max( + values, + start: Incomplete | None = ..., + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def cumulative_mean( + values, + start: Incomplete | None = ..., + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def cumulative_min( + values, + start: Incomplete | None = ..., + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def cumulative_prod( + values, + start: Incomplete | None = ..., + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def cumulative_prod_checked( + values, + start: Incomplete | None = ..., + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def cumulative_sum( + values, + start: Incomplete | None = ..., + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def cumulative_sum_checked( + values, + start: Incomplete | None = ..., + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def day(values, *, memory_pool: Incomplete | None = ...): ... +def day_of_week( + values, + *, + count_from_zero: bool = ..., + week_start: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def day_of_year(values, *, memory_pool: Incomplete | None = ...): ... +def day_time_interval_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def days_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def dictionary_decode(dictionary_array, *, memory_pool: Incomplete | None = ...): ... +def dictionary_encode( + array, + null_encoding: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def divide(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def divide_checked(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def drop_null(input, *, memory_pool: Incomplete | None = ...): ... +def ends_with( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def equal(x, y, *, memory_pool: Incomplete | None = ...): ... +def exp(exponent, *, memory_pool: Incomplete | None = ...): ... +def expm1(exponent, *, memory_pool: Incomplete | None = ...): ... +def extract_regex( + strings, + pattern: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def extract_regex_span( + strings, + pattern: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def fill_null_backward(values, *, memory_pool: Incomplete | None = ...): ... +def fill_null_forward(values, *, memory_pool: Incomplete | None = ...): ... +def filter( + input, + selection_filter, + null_selection_behavior: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def find_substring( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def find_substring_regex( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def first( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def first_last( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def floor(x, *, memory_pool: Incomplete | None = ...): ... +def floor_temporal( + timestamps, + multiple: int = ..., + unit: str = ..., + *, + week_starts_monday: bool = ..., + ceil_is_strictly_greater: bool = ..., + calendar_based_origin: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def greater(x, y, *, memory_pool: Incomplete | None = ...): ... +def greater_equal(x, y, *, memory_pool: Incomplete | None = ...): ... +def hour(values, *, memory_pool: Incomplete | None = ...): ... +def hours_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def hypot(x, y, *, memory_pool: Incomplete | None = ...): ... +def if_else(cond, left, right, *, memory_pool: Incomplete | None = ...): ... +def index( + data: ChunkedArray | Int64Array, + value: Int64Scalar | int | Int8Scalar | None, + start: Incomplete | None = ..., + end: Incomplete | None = ..., + *, + memory_pool: Incomplete | None = ..., +) -> int: ... +def index_in( + values, + value_set: pyarrow.Array, + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def index_in_meta_binary( + values, value_set, *, memory_pool: Incomplete | None = ... +): ... +def indices_nonzero(values, *, memory_pool: Incomplete | None = ...): ... +def inverse_permutation( + indices, + max_index: int = ..., + output_type: Incomplete | None = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def invert(values, *, memory_pool: Incomplete | None = ...): ... +def is_dst(values, *, memory_pool: Incomplete | None = ...): ... +def is_finite(values, *, memory_pool: Incomplete | None = ...): ... +def is_in( + values, + value_set: pyarrow.Array, + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def is_in_meta_binary(values, value_set, *, memory_pool: Incomplete | None = ...): ... +def is_inf(values, *, memory_pool: Incomplete | None = ...): ... +def is_leap_year(values, *, memory_pool: Incomplete | None = ...): ... +def is_nan(values, *, memory_pool: Incomplete | None = ...): ... +def is_null( + values, + *, + nan_is_null: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def is_valid(values, *, memory_pool: Incomplete | None = ...): ... +def iso_calendar(values, *, memory_pool: Incomplete | None = ...): ... +def iso_week(values, *, memory_pool: Incomplete | None = ...): ... +def iso_year(values, *, memory_pool: Incomplete | None = ...): ... +def kurtosis( + array, + *, + skip_nulls: bool = ..., + biased: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def last( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def less(x, y, *, memory_pool: Incomplete | None = ...): ... +def less_equal(x, y, *, memory_pool: Incomplete | None = ...): ... +def list_element(lists, index, *, memory_pool: Incomplete | None = ...): ... +def list_flatten( + lists, + recursive: bool = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def list_parent_indices(lists, *, memory_pool: Incomplete | None = ...): ... +def list_slice( + lists, + start: int, + stop: Incomplete | None = ..., + step: int = ..., + return_fixed_size_list: Incomplete | None = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def list_value_length(lists, *, memory_pool: Incomplete | None = ...): ... +def ln(x, *, memory_pool: Incomplete | None = ...): ... +def ln_checked(x, *, memory_pool: Incomplete | None = ...): ... +def local_timestamp(values, *, memory_pool: Incomplete | None = ...): ... +def log10(x, *, memory_pool: Incomplete | None = ...): ... +def log10_checked(x, *, memory_pool: Incomplete | None = ...): ... +def log1p(x, *, memory_pool: Incomplete | None = ...): ... +def log1p_checked(x, *, memory_pool: Incomplete | None = ...): ... +def log2(x, *, memory_pool: Incomplete | None = ...): ... +def log2_checked(x, *, memory_pool: Incomplete | None = ...): ... +def logb(x, b, *, memory_pool: Incomplete | None = ...): ... +def logb_checked(x, b, *, memory_pool: Incomplete | None = ...): ... +def make_struct( + *args, + field_names: tuple = ..., + field_nullability: Incomplete | None = ..., + field_metadata: Incomplete | None = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def map_lookup( + container, + query_key, + occurrence: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def match_like( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def match_substring( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def match_substring_regex( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def max( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def max_element_wise( + *args, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def mean( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def microsecond(values, *, memory_pool: Incomplete | None = ...): ... +def microseconds_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def millisecond(values, *, memory_pool: Incomplete | None = ...): ... +def milliseconds_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def min( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def min_element_wise( + *args, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def min_max( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def minute(values, *, memory_pool: Incomplete | None = ...): ... +def minutes_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def mode( + array, + n: int = ..., + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def modulo(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def modulo_checked(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def month(values, *, memory_pool: Incomplete | None = ...): ... +def month_day_nano_interval_between( + start, end, *, memory_pool: Incomplete | None = ... +): ... +def month_interval_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def multiply(x, y, *, memory_pool: Incomplete | None = ...): ... +def multiply_checked(x, y, *, memory_pool: Incomplete | None = ...): ... +def nanosecond(values, *, memory_pool: Incomplete | None = ...): ... +def nanoseconds_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def negate(x, *, memory_pool: Incomplete | None = ...): ... +def negate_checked(x, *, memory_pool: Incomplete | None = ...): ... +def not_equal(x, y, *, memory_pool: Incomplete | None = ...): ... +def or_(x, y, *, memory_pool: Incomplete | None = ...): ... +def or_kleene(x, y, *, memory_pool: Incomplete | None = ...): ... +def pairwise_diff( + input, + period: int = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def pairwise_diff_checked( + input, + period: int = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def partition_nth_indices( + array, + pivot: int, + *, + null_placement: str = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def pivot_wider( + pivot_keys, + pivot_values, + key_names: Sequence[str], + *, + unexpected_key_behavior: str = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def power(base, exponent, *, memory_pool: Incomplete | None = ...): ... +def power_checked(base, exponent, *, memory_pool: Incomplete | None = ...): ... +def product( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def quantile( + array, + q: float = ..., + *, + interpolation: str = ..., + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def quarter(values, *, memory_pool: Incomplete | None = ...): ... +def quarters_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def random( + n: int, + *, + initializer: str = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +) -> DoubleArray: ... +def rank( + input, + sort_keys: str = ..., + *, + null_placement: Incomplete | None = ..., + tiebreaker: str = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def rank_normal( + input, + sort_keys: str = ..., + *, + null_placement: Incomplete | None = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def rank_quantile( + input, + sort_keys: str = ..., + *, + null_placement: Incomplete | None = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def remainder(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def remainder_checked(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def replace_substring( + strings, + pattern: str, + replacement: str, + *, + max_replacements: Incomplete | None = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def replace_substring_regex( + strings, + pattern: str, + replacement: str, + *, + max_replacements: Incomplete | None = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def replace_with_mask( + values, mask, replacements, *, memory_pool: Incomplete | None = ... +): ... +def round( + x, + ndigits: int = ..., + round_mode: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def round_binary( + x, + s, + round_mode: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def round_temporal( + timestamps, + multiple: int = ..., + unit: str = ..., + *, + week_starts_monday: bool = ..., + ceil_is_strictly_greater: bool = ..., + calendar_based_origin: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def round_to_multiple( + x, + multiple: float = ..., + round_mode: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def run_end_decode(array, *, memory_pool: Incomplete | None = ...): ... +def run_end_encode( + array, + run_end_type: pyarrow.lib.DataType = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def scatter( + values, + indices, + max_index: int = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def search_sorted( + values, + needles, + side: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def second(values, *, memory_pool: Incomplete | None = ...): ... +def seconds_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def select_k_unstable( + input, + k: int, + sort_keys, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def shift_left(x, y, *, memory_pool: Incomplete | None = ...): ... +def shift_left_checked(x, y, *, memory_pool: Incomplete | None = ...): ... +def shift_right(x, y, *, memory_pool: Incomplete | None = ...): ... +def shift_right_checked(x, y, *, memory_pool: Incomplete | None = ...): ... +def sign(x, *, memory_pool: Incomplete | None = ...): ... +def sin(x, *, memory_pool: Incomplete | None = ...): ... +def sin_checked(x, *, memory_pool: Incomplete | None = ...): ... +def sinh(x, *, memory_pool: Incomplete | None = ...): ... +def skew( + array, + *, + skip_nulls: bool = ..., + biased: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def sort_indices( + input, + sort_keys: tuple = ..., + *, + null_placement: Incomplete | None = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def split_pattern( + strings, + pattern: str, + *, + max_splits: Incomplete | None = ..., + reverse: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def split_pattern_regex( + strings, + pattern: str, + *, + max_splits: Incomplete | None = ..., + reverse: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def sqrt(x, *, memory_pool: Incomplete | None = ...): ... +def sqrt_checked(x, *, memory_pool: Incomplete | None = ...): ... +def starts_with( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def stddev( + array, + *, + ddof: int = ..., + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def strftime( + timestamps, + format: str = ..., + locale: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def string_is_ascii(strings, *, memory_pool: Incomplete | None = ...): ... +def strptime( + strings, + format: str, + unit: str, + error_is_null: bool = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def struct_field( + values, + indices: list[str] | list[bytes] | list[int] | Expression | bytes | str | int, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def subsecond(values, *, memory_pool: Incomplete | None = ...): ... +def subtract(x, y, *, memory_pool: Incomplete | None = ...): ... +def subtract_checked(x, y, *, memory_pool: Incomplete | None = ...): ... +def sum( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def take( + data: pyarrow.Array | pyarrow.ChunkedArray | pyarrow.RecordBatch | pyarrow.Table, + indices: pyarrow.Array | pyarrow.ChunkedArray, + *, + boundscheck: bool = ..., + memory_pool: Incomplete | None = ..., +) -> ( + Table + | Int16Array + | UInt64Array + | UInt16Array + | ChunkedArray + | Int32Array + | DictionaryArray + | FloatArray + | ListArray + | NullArray + | StringArray + | Int8Array + | UInt32Array + | DoubleArray + | LargeBinaryArray + | LargeListArray + | BooleanArray + | ExtensionArray + | BinaryArray + | UInt8Array + | Int64Array + | FixedSizeBinaryArray + | RecordBatch + | StructArray +): ... +def tan(x, *, memory_pool: Incomplete | None = ...): ... +def tan_checked(x, *, memory_pool: Incomplete | None = ...): ... +def tanh(x, *, memory_pool: Incomplete | None = ...): ... +def tdigest( + array, + q: float = ..., + *, + delta: int = ..., + buffer_size: int = ..., + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def true_unless_null(values, *, memory_pool: Incomplete | None = ...): ... +def trunc(x, *, memory_pool: Incomplete | None = ...): ... +def unique(array, *, memory_pool: Incomplete | None = ...): ... +def us_week(values, *, memory_pool: Incomplete | None = ...): ... +def us_year(values, *, memory_pool: Incomplete | None = ...): ... +def utf8_capitalize(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_center( + strings, + width: int, + padding: str = ..., + lean_left_on_odd_padding: bool = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_is_alnum(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_alpha(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_decimal(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_digit(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_lower(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_numeric(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_printable(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_space(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_title(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_upper(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_length(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_lower(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_lpad( + strings, + width: int, + padding: str = ..., + lean_left_on_odd_padding: bool = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_ltrim( + strings, + characters: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_ltrim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_normalize( + strings, + form: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_replace_slice( + strings, + start: int, + stop: int, + replacement: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_reverse(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_rpad( + strings, + width: int, + padding: str = ..., + lean_left_on_odd_padding: bool = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_rtrim( + strings, + characters: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_rtrim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_slice_codeunits( + strings, + start: int, + stop: Incomplete | None = ..., + step: int = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_split_whitespace( + strings, + *, + max_splits: Incomplete | None = ..., + reverse: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_swapcase(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_title(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_trim( + strings, + characters: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_trim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_upper(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_zero_fill( + strings, + width: int, + padding: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def value_counts(array, *, memory_pool: Incomplete | None = ...): ... +def variance( + array, + *, + ddof: int = ..., + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def week( + values, + *, + week_starts_monday: bool = ..., + count_from_zero: bool = ..., + first_week_is_fully_in_year: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def weeks_between( + start, + end, + *, + count_from_zero: bool = ..., + week_start: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def winsorize( + array, + lower_limit, + upper_limit, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def xor(x, y, *, memory_pool: Incomplete | None = ...): ... +def year(values, *, memory_pool: Incomplete | None = ...): ... +def year_month_day(values, *, memory_pool: Incomplete | None = ...): ... +def years_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def utf8_zfill( + strings, + width: int, + padding: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def fill_null( + values: ChunkedArray + | Int16Array + | UInt64Array + | StringArray + | NullArray + | UInt8Array + | Int64Array + | Int8Array + | DoubleArray + | Int32Array + | FloatArray + | LargeBinaryArray + | UInt16Array + | BooleanArray, + fill_value: Int64Scalar + | NullScalar + | FloatScalar + | bool + | str + | Int8Array + | UInt16Scalar + | UInt8Scalar + | UInt64Scalar + | DoubleScalar + | Int16Scalar + | int + | Int8Scalar + | None, +) -> ( + ChunkedArray + | Int16Array + | UInt64Array + | StringArray + | NullArray + | UInt8Array + | Int64Array + | Int8Array + | DoubleArray + | Int32Array + | FloatArray + | LargeBinaryArray + | UInt16Array + | BooleanArray +): ... +def top_k_unstable( + values: pyarrow.Array | pyarrow.ChunkedArray | pyarrow.RecordBatch | pyarrow.Table, + k: int, + sort_keys: Incomplete | None = ..., + null_placements: Incomplete | None = ..., + *, + memory_pool: Incomplete | None = ..., +) -> pyarrow.Array: ... +def bottom_k_unstable( + values: pyarrow.Array | pyarrow.ChunkedArray | pyarrow.RecordBatch | pyarrow.Table, + k: int, + sort_keys: Incomplete | None = ..., + null_placements: Incomplete | None = ..., + *, + memory_pool: Incomplete | None = ..., +) -> UInt64Array: ... +def field(*name_or_index) -> Expression: ... +def scalar(value: bool | float | str) -> Expression: ... + +class _OptionsClassDoc(tuple): + _fields: ClassVar[tuple] = ... + _field_defaults: ClassVar[dict] = ... + __match_args__: ClassVar[tuple] = ... + params: Incomplete + + def __init__(self, _cls, params) -> None: ... + def __replace__(self, **kwds): ... + def __getnewargs__(self): ... diff --git a/python/pyarrow-stubs/pyarrow/csv.pyi b/python/pyarrow-stubs/pyarrow/csv.pyi new file mode 100644 index 000000000000..2ea79dda6b89 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/csv.pyi @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyarrow._csv import ( + CSVStreamingReader as CSVStreamingReader, + CSVWriter as CSVWriter, + ConvertOptions as ConvertOptions, + ISO8601 as ISO8601, + InvalidRow as InvalidRow, + ParseOptions as ParseOptions, + ReadOptions as ReadOptions, + WriteOptions as WriteOptions, + open_csv as open_csv, + read_csv as read_csv, + write_csv as write_csv, +) diff --git a/python/pyarrow-stubs/pyarrow/cuda.pyi b/python/pyarrow-stubs/pyarrow/cuda.pyi new file mode 100644 index 000000000000..707a2e56d89f --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/cuda.pyi @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyarrow._cuda import ( + BufferReader as BufferReader, + BufferWriter as BufferWriter, + Context as Context, + CudaBuffer as CudaBuffer, + HostBuffer as HostBuffer, + IpcMemHandle as IpcMemHandle, + new_host_buffer as new_host_buffer, + read_message as read_message, + read_record_batch as read_record_batch, + serialize_record_batch as serialize_record_batch, +) diff --git a/python/pyarrow-stubs/pyarrow/dataset.pyi b/python/pyarrow-stubs/pyarrow/dataset.pyi new file mode 100644 index 000000000000..b59431d9bb16 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/dataset.pyi @@ -0,0 +1,240 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyarrow._fs import ( + FileSelector, + LocalFileSystem, + PyFileSystem, + SubTreeFileSystem, + _MockFileSystem, +) +from pyarrow.lib import RecordBatch, RecordBatchReader, Schema, Table +import os +import pyarrow +from typing import Any +from collections.abc import Callable +import pyarrow.dataset +import pyarrow.fs +from pyarrow._dataset import ( + CsvFileFormat as CsvFileFormat, + CsvFragmentScanOptions as CsvFragmentScanOptions, + Dataset as Dataset, + DatasetFactory as DatasetFactory, + DirectoryPartitioning as DirectoryPartitioning, + FeatherFileFormat as FeatherFileFormat, + FileFormat as FileFormat, + FileFragment as FileFragment, + FileSystemDataset as FileSystemDataset, + FileSystemDatasetFactory as FileSystemDatasetFactory, + FileSystemFactoryOptions as FileSystemFactoryOptions, + FileWriteOptions as FileWriteOptions, + FilenamePartitioning as FilenamePartitioning, + Fragment as Fragment, + FragmentScanOptions as FragmentScanOptions, + HivePartitioning as HivePartitioning, + InMemoryDataset as InMemoryDataset, + IpcFileFormat as IpcFileFormat, + IpcFileWriteOptions as IpcFileWriteOptions, + JsonFileFormat as JsonFileFormat, + JsonFragmentScanOptions as JsonFragmentScanOptions, + Partitioning as Partitioning, + PartitioningFactory as PartitioningFactory, + Scanner as Scanner, + TaggedRecordBatch as TaggedRecordBatch, + UnionDataset as UnionDataset, + UnionDatasetFactory as UnionDatasetFactory, + WrittenFile as WrittenFile, + _filesystemdataset_write as _filesystemdataset_write, + get_partition_keys as get_partition_keys, +) +from pyarrow._dataset_orc import OrcFileFormat as OrcFileFormat +from pyarrow._dataset_parquet import ( + ParquetDatasetFactory as ParquetDatasetFactory, + ParquetFactoryOptions as ParquetFactoryOptions, + ParquetFileFormat as ParquetFileFormat, + ParquetFileFragment as ParquetFileFragment, + ParquetFileWriteOptions as ParquetFileWriteOptions, + ParquetFragmentScanOptions as ParquetFragmentScanOptions, + ParquetReadOptions as ParquetReadOptions, + RowGroupInfo as RowGroupInfo, +) +from pyarrow._dataset_parquet_encryption import ( + ParquetDecryptionConfig as ParquetDecryptionConfig, + ParquetEncryptionConfig as ParquetEncryptionConfig, +) +from pyarrow.compute import Expression as Expression, field as field, scalar as scalar +from pyarrow.util import ( + _is_iterable as _is_iterable, + _is_path_like as _is_path_like, + _stringify_path as _stringify_path, +) + +_orc_available: bool +_orc_msg: str +_parquet_available: bool +_parquet_msg: str + +def __getattr__(name: str) -> None: ... +def partitioning( + schema: pyarrow.Schema | None = None, + field_names: list[str] | None = None, + flavor: str | None = None, + dictionaries: str | None = None, +) -> Partitioning | PartitioningFactory: ... +def _ensure_partitioning( + scheme: str + | list[str] + | PartitioningFactory + | FilenamePartitioning + | DirectoryPartitioning + | int + | HivePartitioning + | None, +) -> ( + FilenamePartitioning + | PartitioningFactory + | HivePartitioning + | DirectoryPartitioning + | None +): ... +def _ensure_format( + obj: str + | OrcFileFormat + | JsonFileFormat + | IpcFileFormat + | CsvFileFormat + | ParquetFileFormat, +) -> ( + OrcFileFormat + | JsonFileFormat + | IpcFileFormat + | CsvFileFormat + | FeatherFileFormat + | ParquetFileFormat +): ... +def _ensure_multiple_sources( + paths: list[os.PathLike[str]], filesystem: pyarrow.fs.FileSystem | str | None = None +) -> ( + tuple[_MockFileSystem, list[str]] + | tuple[LocalFileSystem, list[Any]] + | tuple[LocalFileSystem, list[str]] + | tuple[SubTreeFileSystem, list[str]] +): ... +def _ensure_single_source( + path: os.PathLike[str], filesystem: pyarrow.fs.FileSystem | str | None = None +) -> ( + tuple[PyFileSystem, FileSelector] + | tuple[_MockFileSystem, FileSelector] + | tuple[LocalFileSystem, list[str]] + | tuple[LocalFileSystem, FileSelector] + | tuple[SubTreeFileSystem, FileSelector] + | tuple[SubTreeFileSystem, list[str]] +): ... +def _filesystem_dataset( + source, + schema: Schema | None = None, + filesystem: _MockFileSystem + | PyFileSystem + | str + | SubTreeFileSystem + | LocalFileSystem + | None = None, + partitioning: FilenamePartitioning + | str + | list[str] + | PartitioningFactory + | HivePartitioning + | DirectoryPartitioning + | int + | None = None, + format: str + | OrcFileFormat + | JsonFileFormat + | IpcFileFormat + | CsvFileFormat + | ParquetFileFormat + | None = None, + partition_base_dir=None, + exclude_invalid_files=None, + selector_ignore_prefixes: list[str] | None = None, +) -> FileSystemDataset: ... +def _in_memory_dataset( + source: Table | list[RecordBatch] | RecordBatch | list[Table] | RecordBatchReader, + schema=None, + **kwargs, +) -> InMemoryDataset: ... +def _union_dataset( + children: list[FileSystemDataset] + | tuple[InMemoryDataset, InMemoryDataset] + | tuple[FileSystemDataset, FileSystemDataset], + schema: Schema | None = None, + **kwargs, +) -> UnionDataset: ... +def parquet_dataset( + metadata_path: str, + schema: pyarrow.Schema | None = None, + filesystem=None, + format: ParquetFileFormat | None = None, + partitioning: Partitioning | PartitioningFactory | str | list[str] | None = None, + partition_base_dir: str | None = None, +) -> FileSystemDataset: ... +def dataset( + source, + schema: pyarrow.Schema | None = None, + format: FileFormat | str | None = None, + filesystem: _MockFileSystem + | PyFileSystem + | str + | SubTreeFileSystem + | LocalFileSystem + | None = None, + partitioning: Partitioning | PartitioningFactory | str | list[str] | None = None, + partition_base_dir: str | None = None, + exclude_invalid_files: bool | None = None, + ignore_prefixes: list | None = None, +) -> Dataset: ... +def _ensure_write_partitioning( + part: list[str] + | FilenamePartitioning + | DirectoryPartitioning + | HivePartitioning + | None, + schema: Schema, + flavor: str | None, +) -> DirectoryPartitioning | HivePartitioning | FilenamePartitioning: ... +def write_dataset( + data: Table | FileSystemDataset | Scanner | list[Table] | InMemoryDataset, + base_dir: str, + *, + basename_template: str | None = None, + format: FileFormat | str | None = None, + partitioning: Partitioning | list[str] | None = None, + partitioning_flavor: str | None = None, + schema: pyarrow.Schema | None = None, + filesystem: pyarrow.fs.FileSystem | None = None, + file_options: pyarrow.dataset.FileWriteOptions | None = None, + use_threads: bool = True, + preserve_order: bool = False, + max_partitions: int | None = None, + max_open_files: int | None = None, + max_rows_per_file: int | None = None, + min_rows_per_group: int | None = None, + max_rows_per_group: int | None = None, + file_visitor: Callable[..., Any] | None = None, + existing_data_behavior: str = "error", + create_dir: bool = True, +) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/feather.pyi b/python/pyarrow-stubs/pyarrow/feather.pyi new file mode 100644 index 000000000000..548d319c8506 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/feather.pyi @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from io import BytesIO +from pathlib import Path +from pyarrow.lib import ChunkedArray +from typing import Any +from collections.abc import Iterator, Sequence +from _typeshed import Incomplete +from pyarrow import _feather as _feather +from pyarrow._feather import FeatherError as FeatherError +from pyarrow.lib import ( + Codec as Codec, + Table as Table, + concat_tables as concat_tables, + schema as schema, +) +from pyarrow.pandas_compat import _pandas_api as _pandas_api + +def check_chunked_overflow(name: str, col: ChunkedArray) -> None: ... + +_FEATHER_SUPPORTED_CODECS: Incomplete + +def write_feather( + df, + dest: str, + compression: str | None = None, + compression_level: int | None = None, + chunksize: int | None = None, + version: int = 2, +) -> None: ... +def read_feather( + source: BytesIO | Path | str, + columns: Sequence[Any] | None = None, + use_threads: bool = True, + memory_map: bool = False, + **kwargs, +): ... +def _read_table_internal( + source: Path | BytesIO | str, + columns: list[int] | Iterator[Any] | list[str] | None = None, + memory_map: bool = False, + use_threads: bool = True, +) -> Table: ... +def read_table( + source: Path | str, + columns: Sequence[Any] | None = None, + memory_map: bool = False, + use_threads: bool = True, +) -> Table: ... + +class FeatherDataset: + paths: Incomplete + validate_schema: Incomplete + + def __init__( + self, path_or_paths: list[str], validate_schema: bool = True + ) -> None: ... + _tables: Incomplete + schema: Incomplete + + def read_table(self, columns: list[str] | None = None) -> Table: ... + def validate_schemas(self, piece: str, table: Table) -> None: ... + def read_pandas( + self, columns: list[str] | None = None, use_threads: bool = True + ): ... diff --git a/python/pyarrow-stubs/pyarrow/flight.pyi b/python/pyarrow-stubs/pyarrow/flight.pyi new file mode 100644 index 000000000000..2ce10eeaf679 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/flight.pyi @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyarrow._flight import ( + Action as Action, + ActionType as ActionType, + BasicAuth as BasicAuth, + CallInfo as CallInfo, + CertKeyPair as CertKeyPair, + ClientAuthHandler as ClientAuthHandler, + ClientMiddleware as ClientMiddleware, + ClientMiddlewareFactory as ClientMiddlewareFactory, + DescriptorType as DescriptorType, + FlightCallOptions as FlightCallOptions, + FlightCancelledError as FlightCancelledError, + FlightClient as FlightClient, + FlightDataStream as FlightDataStream, + FlightDescriptor as FlightDescriptor, + FlightEndpoint as FlightEndpoint, + FlightError as FlightError, + FlightInfo as FlightInfo, + FlightInternalError as FlightInternalError, + FlightMetadataReader as FlightMetadataReader, + FlightMetadataWriter as FlightMetadataWriter, + FlightMethod as FlightMethod, + FlightServerBase as FlightServerBase, + FlightServerError as FlightServerError, + FlightStreamChunk as FlightStreamChunk, + FlightStreamReader as FlightStreamReader, + FlightStreamWriter as FlightStreamWriter, + FlightTimedOutError as FlightTimedOutError, + FlightUnauthenticatedError as FlightUnauthenticatedError, + FlightUnauthorizedError as FlightUnauthorizedError, + FlightUnavailableError as FlightUnavailableError, + FlightWriteSizeExceededError as FlightWriteSizeExceededError, + GeneratorStream as GeneratorStream, + Location as Location, + MetadataRecordBatchReader as MetadataRecordBatchReader, + MetadataRecordBatchWriter as MetadataRecordBatchWriter, + RecordBatchStream as RecordBatchStream, + Result as Result, + SchemaResult as SchemaResult, + ServerAuthHandler as ServerAuthHandler, + ServerCallContext as ServerCallContext, + ServerMiddleware as ServerMiddleware, + ServerMiddlewareFactory as ServerMiddlewareFactory, + Ticket as Ticket, + TracingServerMiddlewareFactory as TracingServerMiddlewareFactory, + connect as connect, +) diff --git a/python/pyarrow-stubs/pyarrow/fs.pyi b/python/pyarrow-stubs/pyarrow/fs.pyi new file mode 100644 index 000000000000..4e183355b039 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/fs.pyi @@ -0,0 +1,127 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from io import BufferedReader, BufferedWriter, BytesIO +from pyarrow.lib import Buffer, BufferOutputStream, BufferReader, NativeFile, OSFile +from typing import Any +from collections.abc import Mapping +from _typeshed import Incomplete +from pyarrow._azurefs import AzureFileSystem as AzureFileSystem +from pyarrow._fs import ( + FileInfo as FileInfo, + FileSelector as FileSelector, + FileSystem as FileSystem, + FileSystemHandler as FileSystemHandler, + FileType as FileType, + LocalFileSystem as LocalFileSystem, + PyFileSystem as PyFileSystem, + SubTreeFileSystem as SubTreeFileSystem, + _MockFileSystem as _MockFileSystem, + _copy_files as _copy_files, + _copy_files_selector as _copy_files_selector, +) +from pyarrow._gcsfs import GcsFileSystem as GcsFileSystem +from pyarrow._hdfs import HadoopFileSystem as HadoopFileSystem +from pyarrow._s3fs import ( + AwsDefaultS3RetryStrategy as AwsDefaultS3RetryStrategy, + AwsStandardS3RetryStrategy as AwsStandardS3RetryStrategy, + S3FileSystem as S3FileSystem, + S3LogLevel as S3LogLevel, + S3RetryStrategy as S3RetryStrategy, + ensure_s3_finalized as ensure_s3_finalized, + ensure_s3_initialized as ensure_s3_initialized, + finalize_s3 as finalize_s3, + initialize_s3 as initialize_s3, + resolve_s3_region as resolve_s3_region, +) +from pyarrow.util import ( + _is_path_like as _is_path_like, + _stringify_path as _stringify_path, +) + +FileStats = FileInfo +_not_imported: Incomplete + +def __getattr__(name: str) -> None: ... +def _ensure_filesystem( + filesystem: PyFileSystem + | _MockFileSystem + | str + | SubTreeFileSystem + | LocalFileSystem, + *, + use_mmap: bool = False, +) -> PyFileSystem | SubTreeFileSystem | _MockFileSystem | LocalFileSystem: ... +def _resolve_filesystem_and_path( + path, + filesystem: _MockFileSystem + | PyFileSystem + | str + | SubTreeFileSystem + | LocalFileSystem + | None = None, + *, + memory_map: bool = False, +) -> ( + tuple[PyFileSystem, str] + | tuple[None, BufferedReader] + | tuple[None, NativeFile] + | tuple[None, BufferOutputStream] + | tuple[LocalFileSystem, str] + | tuple[_MockFileSystem, str] + | tuple[None, None] + | tuple[None, BytesIO] + | tuple[None, OSFile] + | tuple[SubTreeFileSystem, str] + | tuple[None, Buffer] + | tuple[None, BufferedWriter] + | tuple[None, BufferReader] +): ... +def copy_files( + source: str, + destination: str, + source_filesystem: FileSystem | None = None, + destination_filesystem: FileSystem | None = None, + *, + chunk_size: int = ..., + use_threads: bool = True, +) -> None: ... + +class FSSpecHandler(FileSystemHandler): + fs: Incomplete + + def __init__(self, fs) -> None: ... + def __eq__(self, other): ... + def __ne__(self, other): ... + def get_type_name(self): ... + def normalize_path(self, path: str): ... + @staticmethod + def _create_file_info(path, info): ... + def get_file_info(self, paths: list[str]): ... + def get_file_info_selector(self, selector: FileSelector): ... + def create_dir(self, path: str, recursive: bool) -> None: ... + def delete_dir(self, path: str) -> None: ... + def _delete_dir_contents(self, path, missing_dir_ok) -> None: ... + def delete_dir_contents(self, path: str, missing_dir_ok: bool) -> None: ... + def delete_root_dir_contents(self) -> None: ... + def delete_file(self, path: str) -> None: ... + def move(self, src: str, dest: str) -> None: ... + def copy_file(self, src: str, dest: str) -> None: ... + def open_input_stream(self, path: str): ... + def open_input_file(self, path: str): ... + def open_output_stream(self, path: str, metadata: Mapping[Any, Any]): ... + def open_append_stream(self, path: str, metadata: Mapping[Any, Any]): ... diff --git a/python/pyarrow-stubs/pyarrow/gandiva.pyi b/python/pyarrow-stubs/pyarrow/gandiva.pyi new file mode 100644 index 000000000000..a80e03671e58 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/gandiva.pyi @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated by stubgen-pyx from python/pyarrow/gandiva.pyx + +from pyarrow.includes.libarrow import * +from pyarrow.lib import DataType, Field, MemoryPool, RecordBatch, Schema, _Weakrefable + +class Node(_Weakrefable): + def __init__(self): ... + def __str__(self): ... + def __repr__(self): ... + def return_type(self): ... + +class Expression(_Weakrefable): + def __str__(self): ... + def __repr__(self): ... + def root(self): ... + def result(self): ... + +class Condition(_Weakrefable): + def __init__(self): ... + def __str__(self): ... + def __repr__(self): ... + def root(self): ... + def result(self): ... + +class SelectionVector(_Weakrefable): + def __init__(self): ... + def to_array(self): ... + +class Projector(_Weakrefable): + def __init__(self): ... + @property + def llvm_ir(self): ... + def evaluate( + self, batch: RecordBatch, selection: SelectionVector | None = None + ): ... + +class Filter(_Weakrefable): + def __init__(self): ... + @property + def llvm_ir(self): ... + def evaluate(self, batch: RecordBatch, pool: MemoryPool, dtype="int32"): ... + +class TreeExprBuilder(_Weakrefable): + def make_literal(self, value, dtype): ... + def make_expression(self, root_node: Node, return_field: Field): ... + def make_function(self, name, children, return_type: DataType): ... + def make_field(self, field: Field): ... + def make_if( + self, condition: Node, this_node: Node, else_node: Node, return_type: DataType + ): ... + def make_and(self, children): ... + def make_or(self, children): ... + def _make_in_expression_int32(self, node: Node, values): ... + def _make_in_expression_int64(self, node: Node, values): ... + def _make_in_expression_time32(self, node: Node, values): ... + def _make_in_expression_time64(self, node: Node, values): ... + def _make_in_expression_date32(self, node: Node, values): ... + def _make_in_expression_date64(self, node: Node, values): ... + def _make_in_expression_timestamp(self, node: Node, values): ... + def _make_in_expression_binary(self, node: Node, values): ... + def _make_in_expression_string(self, node: Node, values): ... + def make_in_expression(self, node: Node, values, dtype): ... + def make_condition(self, condition: Node): ... + +class Configuration(_Weakrefable): + def __init__(self, optimize: bool = True, dump_ir: bool = False): ... + +class FunctionSignature(_Weakrefable): + def __init__(self): ... + def return_type(self): ... + def param_types(self): ... + def name(self): ... + def __repr__(self): ... + +def make_projector( + schema: Schema, + children, + pool: MemoryPool, + selection_mode: str = "NONE", + configuration: Configuration | None = None, +): ... +def make_filter( + schema: Schema, condition: Condition, configuration: Configuration | None = None +): ... +def get_registered_function_signatures(): ... diff --git a/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi b/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi new file mode 100644 index 000000000000..3d04612d9651 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .from_dataframe import from_dataframe as from_dataframe diff --git a/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi b/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi new file mode 100644 index 000000000000..4fa39a8cdbf1 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import enum +import pyarrow as pa +from _typeshed import Incomplete + +class DlpackDeviceType(enum.IntEnum): + CPU = 1 + CUDA = 2 + CPU_PINNED = 3 + OPENCL = 4 + VULKAN = 7 + METAL = 8 + VPI = 9 + ROCM = 10 + +class _PyArrowBuffer: + _x: Incomplete + + def __init__(self, x: pa.Buffer, allow_copy: bool = True) -> None: ... + @property + def bufsize(self) -> int: ... + @property + def ptr(self) -> int: ... + def __dlpack__(self) -> None: ... + def __dlpack_device__(self) -> tuple[DlpackDeviceType, int | None]: ... diff --git a/python/pyarrow-stubs/pyarrow/interchange/column.pyi b/python/pyarrow-stubs/pyarrow/interchange/column.pyi new file mode 100644 index 000000000000..5790a3f9d05e --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/interchange/column.pyi @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import enum +import pyarrow as pa +from _typeshed import Incomplete +from pyarrow.interchange.buffer import _PyArrowBuffer as _PyArrowBuffer +from typing import Any, TypedDict, TypeAlias +from collections.abc import Iterable + +_PYARROW_KINDS: Incomplete + +class DtypeKind(enum.IntEnum): + INT = 0 + UINT = 1 + FLOAT = 2 + BOOL = 20 + STRING = 21 + DATETIME = 22 + CATEGORICAL = 23 + +Dtype: TypeAlias = tuple[DtypeKind, int, str, str] + +class ColumnNullType(enum.IntEnum): + NON_NULLABLE = 0 + USE_NAN = 1 + USE_SENTINEL = 2 + USE_BITMASK = 3 + USE_BYTEMASK = 4 + +class ColumnBuffers(TypedDict): + data: tuple[_PyArrowBuffer, Dtype] + validity: tuple[_PyArrowBuffer, Dtype] | None + offsets: tuple[_PyArrowBuffer, Dtype] | None + +class CategoricalDescription(TypedDict): + is_ordered: bool + is_dictionary: bool + categories: _PyArrowColumn | None + +class Endianness: + LITTLE: str + BIG: str + NATIVE: str + NA: str + +class NoBufferPresent(Exception): ... + +class _PyArrowColumn: + _allow_copy: Incomplete + _dtype: Incomplete + _col: Incomplete + + def __init__( + self, column: pa.Array | pa.ChunkedArray, allow_copy: bool = True + ) -> None: ... + def size(self) -> int: ... + @property + def offset(self) -> int: ... + @property + def dtype(self) -> tuple[DtypeKind, int, str, str]: ... + def _dtype_from_arrowdtype( + self, dtype: pa.DataType, bit_width: int + ) -> tuple[DtypeKind, int, str, str]: ... + @property + def describe_categorical(self) -> CategoricalDescription: ... + @property + def describe_null(self) -> tuple[ColumnNullType, Any]: ... + @property + def null_count(self) -> int: ... + @property + def metadata(self) -> dict[str, Any]: ... + def num_chunks(self) -> int: ... + def get_chunks(self, n_chunks: int | None = None) -> Iterable[_PyArrowColumn]: ... + def get_buffers(self) -> ColumnBuffers: ... + def _get_data_buffer(self) -> tuple[_PyArrowBuffer, Any]: ... + def _get_validity_buffer(self) -> tuple[_PyArrowBuffer, Any]: ... + def _get_offsets_buffer(self) -> tuple[_PyArrowBuffer, Any]: ... diff --git a/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi b/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi new file mode 100644 index 000000000000..283aea084506 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pyarrow as pa +from _typeshed import Incomplete +from pyarrow.interchange.column import _PyArrowColumn as _PyArrowColumn +from typing import Any +from collections.abc import Iterable, Sequence + +class _PyArrowDataFrame: + _df: Incomplete + _nan_as_null: Incomplete + _allow_copy: Incomplete + + def __init__( + self, + df: pa.Table | pa.RecordBatch, + nan_as_null: bool = False, + allow_copy: bool = True, + ) -> None: ... + def __dataframe__( + self, nan_as_null: bool = False, allow_copy: bool = True + ) -> _PyArrowDataFrame: ... + @property + def metadata(self) -> dict[str, Any]: ... + def num_columns(self) -> int: ... + def num_rows(self) -> int: ... + def num_chunks(self) -> int: ... + def column_names(self) -> Iterable[str]: ... + def get_column(self, i: int) -> _PyArrowColumn: ... + def get_column_by_name(self, name: str) -> _PyArrowColumn: ... + def get_columns(self) -> Iterable[_PyArrowColumn]: ... + def select_columns(self, indices: Sequence[int]) -> _PyArrowDataFrame: ... + def select_columns_by_name(self, names: Sequence[str]) -> _PyArrowDataFrame: ... + def get_chunks( + self, n_chunks: int | None = None + ) -> Iterable[_PyArrowDataFrame]: ... diff --git a/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi b/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi new file mode 100644 index 000000000000..f0a05cbcde65 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyarrow.lib import DataType, TimestampType +import pyarrow +import pyarrow as pa +from pyarrow.interchange.column import ( + ColumnBuffers as ColumnBuffers, + ColumnNullType as ColumnNullType, + Dtype as Dtype, + DtypeKind as DtypeKind, +) +from typing import Any, TypeAlias + +DataFrameObject: TypeAlias = Any +ColumnObject: TypeAlias = Any +BufferObject: TypeAlias = Any +_PYARROW_DTYPES: dict[DtypeKind, dict[int, Any]] + +def from_dataframe(df: DataFrameObject, allow_copy: bool = True) -> pa.Table: ... +def _from_dataframe(df: DataFrameObject, allow_copy: bool = True) -> pyarrow.Table: ... +def protocol_df_chunk_to_pyarrow( + df: DataFrameObject, allow_copy: bool = True +) -> pa.RecordBatch: ... +def column_to_array(col: ColumnObject, allow_copy: bool = True) -> pa.Array: ... +def bool_column_to_array(col: ColumnObject, allow_copy: bool = True) -> pa.Array: ... +def categorical_column_to_dictionary( + col: ColumnObject, allow_copy: bool = True +) -> pa.DictionaryArray: ... +def parse_datetime_format_str(format_str: str) -> tuple[str, str]: ... +def map_date_type( + data_type: tuple[DtypeKind, int, str, str], +) -> DataType | TimestampType: ... +def buffers_to_array( + buffers: ColumnBuffers, + data_type: tuple[DtypeKind, int, str, str], + length: int, + describe_null: ColumnNullType, + offset: int = 0, + allow_copy: bool = True, +) -> pa.Array: ... +def validity_buffer_from_mask( + validity_buff: BufferObject, + validity_dtype: Dtype, + describe_null: ColumnNullType, + length: int, + offset: int = 0, + allow_copy: bool = True, +) -> pa.Buffer: ... +def validity_buffer_nan_sentinel( + data_pa_buffer: BufferObject, + data_type: Dtype, + describe_null: ColumnNullType, + length: int, + offset: int = 0, + allow_copy: bool = True, +) -> pa.Buffer: ... diff --git a/python/pyarrow-stubs/pyarrow/ipc.pyi b/python/pyarrow-stubs/pyarrow/ipc.pyi new file mode 100644 index 000000000000..bd5d5147d671 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/ipc.pyi @@ -0,0 +1,144 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from io import BufferedWriter, BytesIO +from pathlib import Path +from pyarrow.lib import ( + Buffer, + BufferOutputStream, + BufferReader, + CompressedInputStream, + CompressedOutputStream, + MockOutputStream, + NativeFile, + OSFile, + Schema, +) +import pyarrow +from typing import Any, IO +import pyarrow.ipc +from pyarrow.lib import __doc__ as __doc__ +import pyarrow.lib as lib +from _typeshed import Incomplete +from pyarrow.lib import ( + Alignment as Alignment, + IpcReadOptions as IpcReadOptions, + IpcWriteOptions as IpcWriteOptions, + Message as Message, + MessageReader as MessageReader, + MetadataVersion as MetadataVersion, + ReadStats as ReadStats, + RecordBatchReader as RecordBatchReader, + WriteStats as WriteStats, + _ReadPandasMixin as _ReadPandasMixin, + get_record_batch_size as get_record_batch_size, + get_tensor_size as get_tensor_size, + read_message as read_message, + read_record_batch as read_record_batch, + read_schema as read_schema, + read_tensor as read_tensor, + write_tensor as write_tensor, +) + +_ipc_writer_class_doc: str +_ipc_file_writer_class_doc: Incomplete + +def _get_legacy_format_default(options: IpcWriteOptions | None) -> IpcWriteOptions: ... +def _ensure_default_ipc_read_options( + options: IpcReadOptions | bool | None, +) -> IpcReadOptions: ... +def new_stream( + sink: str | pyarrow.NativeFile | IO[Any], + schema: pyarrow.Schema, + *, + options: pyarrow.ipc.IpcWriteOptions | None = None, +) -> RecordBatchStreamWriter: ... +def open_stream( + source: BytesIO | bytes | BufferReader | Buffer, + *, + options: pyarrow.ipc.IpcReadOptions | None = None, + memory_pool: pyarrow.MemoryPool | None = None, +) -> RecordBatchStreamReader: ... +def new_file( + sink: str | pyarrow.NativeFile | IO[Any], + schema: pyarrow.Schema, + *, + options: pyarrow.ipc.IpcWriteOptions | None = None, + metadata: dict | pyarrow.KeyValueMetadata | None = None, +) -> RecordBatchFileWriter: ... +def open_file( + source: Path | BufferReader | Buffer | BytesIO | OSFile | bytes, + footer_offset: int | None = None, + *, + options: pyarrow.ipc.IpcReadOptions | None = None, + memory_pool: pyarrow.MemoryPool | None = None, +) -> RecordBatchFileReader: ... +def serialize_pandas( + df, *, nthreads: int | None = None, preserve_index: bool | None = None +) -> Buffer: ... +def deserialize_pandas(buf: Buffer, *, use_threads: bool = True): ... + +class RecordBatchStreamReader(lib._RecordBatchStreamReader): + def __init__( + self, + source: BufferReader | CompressedInputStream | Buffer | BytesIO | bytes, + *, + options=None, + memory_pool=None, + ) -> None: ... + +class RecordBatchStreamWriter(lib._RecordBatchStreamWriter): + __doc__: Incomplete + + def __init__( + self, + sink: BufferOutputStream + | BufferedWriter + | MockOutputStream + | CompressedOutputStream + | BytesIO, + schema: Schema, + *, + options=None, + ) -> None: ... + +class RecordBatchFileReader(lib._RecordBatchFileReader): + def __init__( + self, + source: Path | BufferReader | Buffer | BytesIO | OSFile | bytes, + footer_offset=None, + *, + options=None, + memory_pool=None, + ) -> None: ... + +class RecordBatchFileWriter(lib._RecordBatchFileWriter): + __doc__: Incomplete + + def __init__( + self, + sink: BufferOutputStream + | MockOutputStream + | str + | BytesIO + | NativeFile + | OSFile, + schema: Schema, + *, + options=None, + metadata=None, + ) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/json.pyi b/python/pyarrow-stubs/pyarrow/json.pyi new file mode 100644 index 000000000000..d67cf434b233 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/json.pyi @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyarrow._json import ( + ParseOptions as ParseOptions, + ReadOptions as ReadOptions, + open_json as open_json, + read_json as read_json, +) diff --git a/python/pyarrow-stubs/pyarrow/jvm.pyi b/python/pyarrow-stubs/pyarrow/jvm.pyi new file mode 100644 index 000000000000..657fa8234396 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/jvm.pyi @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pyarrow +from _typeshed import Incomplete + +def jvm_buffer(jvm_buf) -> pyarrow.Buffer: ... +def _from_jvm_int_type(jvm_type) -> pyarrow.DataType: ... +def _from_jvm_float_type(jvm_type): ... +def _from_jvm_time_type(jvm_type): ... +def _from_jvm_timestamp_type(jvm_type): ... +def _from_jvm_date_type(jvm_type): ... +def field(jvm_field) -> pyarrow.Field: ... +def schema(jvm_schema) -> pyarrow.Schema: ... +def array(jvm_array) -> pyarrow.Array: ... +def record_batch(jvm_vector_schema_root): ... + +class _JvmBufferNanny: + ref_manager: Incomplete + + def __init__(self, jvm_buf) -> None: ... + def __del__(self) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/lib.pyi b/python/pyarrow-stubs/pyarrow/lib.pyi new file mode 100644 index 000000000000..5c5988186f42 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/lib.pyi @@ -0,0 +1,2511 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Generated by stubgen-pyx from python/pyarrow/lib.pyx + +import os +import pyarrow +from typing import IO, NamedTuple, Self +from collections.abc import Callable, Iterable, Iterator +import pyarrow.compute +from _typeshed import Incomplete +from collections.abc import Mapping, Sequence +from enum import IntEnum +from typing import Any + +MonthDayNano: Incomplete +Type_NA: Incomplete +Type_BOOL: Incomplete +Type_UINT8: Incomplete +Type_INT8: Incomplete +Type_UINT16: Incomplete +Type_INT16: Incomplete +Type_UINT32: Incomplete +Type_INT32: Incomplete +Type_UINT64: Incomplete +Type_INT64: Incomplete +Type_HALF_FLOAT: Incomplete +Type_FLOAT: Incomplete +Type_DOUBLE: Incomplete +Type_DECIMAL32: Incomplete +Type_DECIMAL64: Incomplete +Type_DECIMAL128: Incomplete +Type_DECIMAL256: Incomplete +Type_DATE32: Incomplete +Type_DATE64: Incomplete +Type_TIMESTAMP: Incomplete +Type_TIME32: Incomplete +Type_TIME64: Incomplete +Type_DURATION: Incomplete +Type_INTERVAL_MONTH_DAY_NANO: Incomplete +Type_BINARY: Incomplete +Type_STRING: Incomplete +Type_LARGE_BINARY: Incomplete +Type_LARGE_STRING: Incomplete +Type_FIXED_SIZE_BINARY: Incomplete +Type_BINARY_VIEW: Incomplete +Type_STRING_VIEW: Incomplete +Type_LIST: Incomplete +Type_LARGE_LIST: Incomplete +Type_LIST_VIEW: Incomplete +Type_LARGE_LIST_VIEW: Incomplete +Type_MAP: Incomplete +Type_FIXED_SIZE_LIST: Incomplete +Type_STRUCT: Incomplete +Type_SPARSE_UNION: Incomplete +Type_DENSE_UNION: Incomplete +Type_DICTIONARY: Incomplete +Type_RUN_END_ENCODED: Incomplete +Type_INTERVAL_MONTHS: Incomplete +Type_INTERVAL_DAY_TIME: Incomplete +UnionMode_SPARSE: Incomplete +UnionMode_DENSE: Incomplete +__pc: Incomplete +__pac: Incomplete +__cuda_loaded: Incomplete +ordered_dict = dict +ArrowIOError = IOError +have_signal_refcycle: bool +build_info: BuildInfo +cpp_build_info: CppBuildInfo +cpp_version: str +cpp_version_info: VersionInfo +_pandas_api: Incomplete +_python_extension_types_registry: list[Incomplete] +_registry_nanny: _ExtensionRegistryNanny +DEFAULT_BUFFER_SIZE: int + +def cpu_count(): ... +def set_cpu_count(count: int) -> None: ... +def is_opentelemetry_enabled() -> bool: ... +def is_threading_enabled() -> bool: ... +def _pc(): ... +def _pac(): ... +def _ensure_cuda_loaded() -> None: ... +def _gdb_test_session() -> None: ... +def encode_file_path(path): ... +def tobytes(o: str | bytes): ... +def frombytes(o, *, safe: bool = False): ... +def enable_signal_handlers(enable: bool) -> None: ... +def runtime_info() -> RuntimeInfo: ... +def _build_info() -> BuildInfo: ... +def set_timezone_db_path(path: str) -> None: ... +def default_memory_pool() -> MemoryPool: ... +def proxy_memory_pool(parent: MemoryPool) -> ProxyMemoryPool: ... +def logging_memory_pool(parent: MemoryPool) -> LoggingMemoryPool: ... +def system_memory_pool() -> MemoryPool: ... +def jemalloc_memory_pool() -> MemoryPool: ... +def mimalloc_memory_pool() -> MemoryPool: ... +def set_memory_pool(pool: MemoryPool) -> None: ... +def log_memory_allocations(enable: bool = True) -> None: ... +def total_allocated_bytes(): ... +def jemalloc_set_decay_ms(decay_ms: int) -> None: ... +def supported_memory_backends() -> list: ... +def default_cpu_memory_manager() -> MemoryManager: ... +def _get_pandas_type_map(): ... +def _is_primitive(type: int): ... +def _is_nested(type: int): ... +def _get_pandas_type(arrow_type, coerce_to_ns: bool = False): ... +def _get_pandas_tz_type(arrow_type, coerce_to_ns: bool = False): ... +def _to_pandas_dtype(arrow_type, options=None): ... +def register_extension_type(ext_type) -> None: ... +def unregister_extension_type(type_name: str) -> None: ... +def ensure_metadata(meta: object, allow_none: bool = False) -> KeyValueMetadata: ... +def unify_schemas( + schemas: list[Schema], *, promote_options: str = "default" +) -> Schema: ... +def field( + name: str | bytes, + type: DataType | str | None = None, + nullable: bool | None = None, + metadata: dict | None = None, +) -> Field: ... +def null() -> DataType: ... +def bool_() -> DataType: ... +def uint8() -> DataType: ... +def int8() -> DataType: ... +def uint16() -> DataType: ... +def int16() -> DataType: ... +def uint32() -> DataType: ... +def int32() -> DataType: ... +def uint64() -> DataType: ... +def int64() -> DataType: ... +def tzinfo_to_string(tz) -> str: ... +def string_to_tzinfo(name, *, prefer_zoneinfo: bool = True): ... +def timestamp(unit: str, tz: str | None = None) -> TimestampType: ... +def time32(unit: str) -> Time32Type: ... +def time64(unit: str) -> Time64Type: ... +def duration(unit: str) -> DurationType: ... +def month_day_nano_interval() -> DataType: ... +def date32() -> DataType: ... +def date64() -> DataType: ... +def float16() -> DataType: ... +def float32() -> DataType: ... +def float64() -> DataType: ... +def decimal32(precision: int, scale: int = 0) -> DataType: ... +def decimal64(precision: int, scale: int = 0) -> DataType: ... +def decimal128(precision: int, scale: int = 0) -> DataType: ... +def decimal256(precision: int, scale: int = 0) -> DataType: ... +def string() -> DataType: ... +def utf8(): ... +def binary(length: int = -1) -> DataType: ... +def large_binary() -> DataType: ... +def large_string() -> DataType: ... +def large_utf8(): ... +def binary_view() -> DataType: ... +def string_view() -> DataType: ... +def list_(value_type: DataType | Field, list_size: int = -1) -> DataType: ... +def large_list(value_type: DataType | Field) -> LargeListType: ... +def list_view(value_type: DataType | Field) -> ListViewType: ... +def large_list_view(value_type: DataType | Field) -> LargeListViewType: ... +def map_( + key_type: DataType | Field, item_type: DataType | Field, keys_sorted: bool = False +) -> MapType: ... +def dictionary( + index_type: DataType, value_type: DataType, ordered: bool = False +) -> DictionaryType: ... +def struct(fields) -> DataType: ... +def sparse_union(child_fields, type_codes: list[int] | None = None) -> DataType: ... +def dense_union(child_fields, type_codes: list[int] | None = None) -> DataType: ... +def union( + child_fields, mode: str, type_codes: list[int] | None = None +) -> UnionType: ... +def run_end_encoded(run_end_type: DataType, value_type: DataType) -> DataType: ... +def json_(storage_type: DataType = ...) -> JsonType: ... +def uuid() -> UuidType: ... +def fixed_shape_tensor( + value_type: DataType, + shape: tuple | list[int], + dim_names: tuple | list[str] | None = None, + permutation=None, +) -> FixedShapeTensorType: ... +def bool8() -> Bool8Type: ... +def opaque(storage_type: DataType, type_name: str, vendor_name: str) -> OpaqueType: ... +def type_for_alias(name: str) -> DataType: ... +def ensure_type(ty: object, allow_none: bool = False) -> DataType: ... +def schema(fields, metadata: dict | None = None) -> Schema: ... +def from_numpy_dtype(dtype: object) -> DataType: ... +def is_boolean_value(obj: object): ... +def is_integer_value(obj: object): ... +def is_float_value(obj: object): ... +def _datetime_from_int(value: int, unit: int, tzinfo=None): ... +def scalar( + value: Any, + type: DataType | None = None, + *, + from_pandas: bool | None = None, + memory_pool: MemoryPool | None = None, +) -> Scalar: ... +def _ndarray_to_arrow_type(values: object, type: DataType) -> DataType: ... +def _handle_arrow_array_protocol(obj, type, mask, size): ... +def array( + obj: object, + type: DataType | None = None, + mask=None, + size=None, + from_pandas: bool | None = None, + safe: bool = True, + memory_pool: MemoryPool | None = None, +) -> Array | ChunkedArray: ... +def asarray(values, type: str | DataType | None = None) -> Array | ChunkedArray: ... +def nulls( + size: int, type: DataType | None = None, memory_pool: MemoryPool | None = None +) -> Array: ... +def repeat(value, size: int, memory_pool: MemoryPool | None = None) -> Array: ... +def infer_type(values, mask=None, from_pandas: bool = False) -> DataType: ... +def arange( + start: int, stop: int, step: int = 1, *, memory_pool: MemoryPool | None = None +) -> Array: ... +def _normalize_slice(arrow_obj: object, key: slice): ... +def _restore_array(data) -> Array: ... +def _compute_binary_op(func_name, left, right): ... +def concat_arrays( + arrays: Iterable[Array], memory_pool: MemoryPool | None = None +) -> Array: ... +def _empty_array(type: DataType) -> DictionaryArray: ... +def chunked_array(arrays, type=None) -> ChunkedArray: ... +def _reconstruct_record_batch(columns, schema) -> RecordBatch: ... +def table_to_blocks(options, table: Table, categories, extension_columns): ... +def _reconstruct_table(arrays, schema) -> Table: ... +def record_batch( + data, + names: list | None = None, + schema: Schema | None = None, + metadata: dict | Mapping[Any, Any] | None = None, +) -> RecordBatch: ... +def table( + data, + names: list | None = None, + schema: Schema | None = None, + metadata: dict | Mapping[Any, Any] | None = None, + nthreads: int | None = None, +) -> Table: ... +def concat_tables( + tables, + memory_pool: MemoryPool | None = None, + promote_options: str = "none", + **kwargs, +) -> Table: ... +def concat_batches( + recordbatches, memory_pool: MemoryPool | None = None +) -> RecordBatch: ... +def _from_pydict( + cls, + mapping: dict | Mapping[Any, Any], + schema: Schema | None, + metadata: dict | Mapping[Any, Any] | None, +) -> Self: ... +def _from_pylist( + cls, mapping, schema: Schema | None, metadata: dict | Mapping[Any, Any] | None +) -> Self: ... +def io_thread_count(): ... +def set_io_thread_count(count: int) -> None: ... +def memory_map(path: str, mode: str = "r") -> MemoryMappedFile: ... +def create_memory_map(path: str, size: int) -> MemoryMappedFile: ... +def allocate_buffer( + size: int, memory_pool: MemoryPool | None = None, resizable: bool = False +) -> ResizableBuffer | Buffer: ... +def transcoding_input_stream( + stream: NativeFile, src_encoding: str, dest_encoding: str +): ... +def py_buffer(obj: object) -> Buffer: ... +def foreign_buffer(address: int, size: int, base=None) -> Buffer: ... +def as_buffer(o: object): ... +def _detect_compression(path) -> str: ... +def compress( + buf: object, + codec: str = "lz4", + asbytes: bool = False, + memory_pool: MemoryPool | None = None, +) -> Buffer | bytes: ... +def decompress( + buf: object, + decompressed_size: int | None = None, + codec: str = "lz4", + asbytes: bool = False, + memory_pool: MemoryPool | None = None, +) -> Buffer | bytes: ... +def input_stream( + source, compression: str | None = "detect", buffer_size: int | None = None +) -> NativeFile: ... +def output_stream( + source, compression: str | None = "detect", buffer_size: int | None = None +) -> NativeFile: ... +def get_tensor_size(tensor: Tensor) -> int: ... +def get_record_batch_size(batch: RecordBatch) -> int: ... +def write_tensor(tensor: Tensor, dest: NativeFile) -> int: ... +def read_tensor(source: NativeFile) -> Tensor: ... +def read_message(source) -> Message: ... +def read_schema(obj, dictionary_memo: DictionaryMemo | None = None) -> Schema: ... +def read_record_batch( + obj, schema: Schema, dictionary_memo: DictionaryMemo | None = None +) -> RecordBatch: ... +def benchmark_PandasObjectIsNull(obj: list[Any]) -> None: ... + +_NULL: NullScalar +NA: NullScalar + +class VersionInfo(NamedTuple): + major: Incomplete + minor: Incomplete + patch: Incomplete + +class RuntimeInfo(NamedTuple): + simd_level: Incomplete + detected_simd_level: Incomplete + +class BuildInfo(NamedTuple): + build_type: Incomplete + cpp_build_info: Incomplete + +class CppBuildInfo(NamedTuple): + version: Incomplete + version_info: Incomplete + so_version: Incomplete + full_so_version: Incomplete + compiler_id: Incomplete + compiler_version: Incomplete + compiler_flags: Incomplete + git_id: Incomplete + git_description: Incomplete + package_kind: Incomplete + build_type: Incomplete + +class _WriteStats(NamedTuple): + num_messages: Incomplete + num_record_batches: Incomplete + num_dictionary_batches: Incomplete + num_dictionary_deltas: Incomplete + num_replaced_dictionaries: Incomplete + +class _ReadStats(NamedTuple): + num_messages: Incomplete + num_record_batches: Incomplete + num_dictionary_batches: Incomplete + num_dictionary_deltas: Incomplete + num_replaced_dictionaries: Incomplete + +class _RecordBatchWithMetadata(NamedTuple): + batch: Incomplete + custom_metadata: Incomplete + +class DeviceAllocationType(IntEnum): + CPU = ... + CUDA = ... + CUDA_HOST = ... + OPENCL = ... + VULKAN = ... + METAL = ... + VPI = ... + ROCM = ... + ROCM_HOST = ... + EXT_DEV = ... + CUDA_MANAGED = ... + ONEAPI = ... + WEBGPU = ... + HEXAGON = ... + +class MetadataVersion(IntEnum): + V1 = ... + V2 = ... + V3 = ... + V4 = ... + V5 = ... + +class Alignment(IntEnum): + Any = ... + DataTypeSpecific = ... + At64Byte = ... + +class ArrowException(Exception): ... +class ArrowInvalid(ValueError, ArrowException): ... +class ArrowMemoryError(MemoryError, ArrowException): ... + +class ArrowKeyError(KeyError, ArrowException): + def __str__(self): ... + +class ArrowTypeError(TypeError, ArrowException): ... +class ArrowNotImplementedError(NotImplementedError, ArrowException): ... +class ArrowCapacityError(ArrowException): ... +class ArrowIndexError(IndexError, ArrowException): ... +class ArrowSerializationError(ArrowException): ... + +class ArrowCancelled(ArrowException): + def __init__(self, message, signum=None): ... + +class StopToken: ... + +class SignalStopHandler: + def __init__(self): ... + def _init_signals(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type, exc_value, exc_tb) -> None: ... + @property + def stop_token(self): ... + +class _PandasAPIShim(object): + _loose_version: object + _version: object + _pd: object + _types_api: object + _compat_module: object + _data_frame: object + _index: object + _series: object + _categorical_type: object + _datetimetz_type: object + _extension_array: object + _extension_dtype: object + _array_like_types: object + _lock: object + _is_ge_v23: bool + _is_ge_v3: bool + _is_ge_v3_strict: bool + + def __init__(self): ... + def series(self, *args, **kwargs): ... + def data_frame(self, *args, **kwargs): ... + @property + def have_pandas(self): ... + @property + def compat(self): ... + @property + def pd(self): ... + def infer_dtype(self, obj): ... + def pandas_dtype(self, dtype): ... + @property + def loose_version(self): ... + @property + def version(self): ... + def is_ge_v23(self): ... + def is_ge_v3(self): ... + def is_ge_v3_strict(self): ... + def uses_string_dtype(self): ... + @property + def categorical_type(self): ... + @property + def datetimetz_type(self): ... + @property + def extension_dtype(self): ... + def is_array_like(self, obj): ... + def is_categorical(self, obj): ... + def is_datetimetz(self, obj): ... + def is_extension_array_dtype(self, obj): ... + def is_sparse(self, obj): ... + def is_data_frame(self, obj): ... + def is_series(self, obj): ... + def is_index(self, obj): ... + def get_values(self, obj): ... + +class _Weakrefable: ... + +class MemoryPool(_Weakrefable): + def __init__(self): ... + def release_unused(self) -> None: ... + def bytes_allocated(self): ... + def total_bytes_allocated(self): ... + def max_memory(self): ... + def num_allocations(self): ... + def print_stats(self) -> None: ... + @property + def backend_name(self) -> str: ... + +class LoggingMemoryPool(MemoryPool): + def __init__(self): ... + +class ProxyMemoryPool(MemoryPool): + def __init__(self): ... + +class Device(_Weakrefable): + def __init__(self): ... + def __eq__(self, other): ... + @property + def type_name(self) -> str: ... + @property + def device_id(self): ... + @property + def is_cpu(self): ... + @property + def device_type(self): ... + +class MemoryManager(_Weakrefable): + def __init__(self): ... + @property + def device(self) -> Device: ... + @property + def is_cpu(self): ... + +class DataType(_Weakrefable): + def __init__(self): ... + def field(self, i: int) -> Field: ... + @property + def id(self): ... + @property + def bit_width(self): ... + @property + def byte_width(self): ... + @property + def num_fields(self): ... + @property + def num_buffers(self): ... + @property + def has_variadic_buffers(self): ... + def __hash__(self): ... + def __reduce__(self) -> tuple: ... + def __eq__(self, other): ... + def equals(self, other, *, check_metadata: bool = False) -> bool: ... + def to_pandas_dtype(self): ... + def _export_to_c(self, out_ptr) -> None: ... + @staticmethod + def _import_from_c(in_ptr) -> DataType: ... + def __arrow_c_schema__(self): ... + @staticmethod + def _import_from_c_capsule(schema) -> DataType: ... + +class DictionaryMemo(_Weakrefable): + def __init__(self): ... + +class DictionaryType(DataType): + def __reduce__(self) -> tuple: ... + @property + def ordered(self): ... + @property + def index_type(self) -> DataType: ... + @property + def value_type(self) -> DataType: ... + +class ListType(DataType): + def __reduce__(self) -> tuple: ... + @property + def value_field(self) -> Field: ... + @property + def value_type(self) -> DataType: ... + +class LargeListType(DataType): + def __reduce__(self) -> tuple: ... + @property + def value_field(self) -> Field: ... + @property + def value_type(self) -> DataType: ... + +class ListViewType(DataType): + def __reduce__(self) -> tuple: ... + @property + def value_field(self) -> Field: ... + @property + def value_type(self) -> DataType: ... + +class LargeListViewType(DataType): + def __reduce__(self) -> tuple: ... + @property + def value_field(self) -> Field: ... + @property + def value_type(self) -> DataType: ... + +class MapType(DataType): + def __reduce__(self) -> tuple: ... + @property + def key_field(self) -> Field: ... + @property + def key_type(self) -> DataType: ... + @property + def item_field(self) -> Field: ... + @property + def item_type(self) -> DataType: ... + @property + def keys_sorted(self): ... + +class FixedSizeListType(DataType): + def __reduce__(self) -> tuple: ... + @property + def value_field(self) -> Field: ... + @property + def value_type(self) -> DataType: ... + @property + def list_size(self): ... + +class StructType(DataType): + def get_field_index(self, name: str) -> int: ... + def field(self, i: int | str) -> Field: ... + def get_all_field_indices(self, name: str) -> list[int]: ... + def __len__(self): ... + def __iter__(self) -> Iterator[Any]: ... + def __getitem__(self, i): ... + def __reduce__(self) -> tuple: ... + @property + def names(self) -> list: ... + @property + def fields(self) -> list: ... + +class UnionType(DataType): + @property + def mode(self) -> str: ... + @property + def type_codes(self): ... + def __len__(self): ... + def __iter__(self) -> Iterator[Any]: ... + def field(self, i: int) -> Field: ... + def __getitem__(self, i): ... + def __reduce__(self) -> tuple: ... + +class SparseUnionType(UnionType): ... +class DenseUnionType(UnionType): ... + +class TimestampType(DataType): + @property + def unit(self): ... + @property + def tz(self) -> str | None: ... + def __reduce__(self) -> tuple: ... + +class Time32Type(DataType): + @property + def unit(self): ... + +class Time64Type(DataType): + @property + def unit(self): ... + +class DurationType(DataType): + @property + def unit(self): ... + +class FixedSizeBinaryType(DataType): + def __reduce__(self) -> tuple: ... + +class Decimal32Type(FixedSizeBinaryType): + def __reduce__(self) -> tuple: ... + @property + def precision(self): ... + @property + def scale(self): ... + +class Decimal64Type(FixedSizeBinaryType): + def __reduce__(self) -> tuple: ... + @property + def precision(self): ... + @property + def scale(self): ... + +class Decimal128Type(FixedSizeBinaryType): + def __reduce__(self) -> tuple: ... + @property + def precision(self): ... + @property + def scale(self): ... + +class Decimal256Type(FixedSizeBinaryType): + def __reduce__(self) -> tuple: ... + @property + def precision(self): ... + @property + def scale(self): ... + +class RunEndEncodedType(DataType): + def __reduce__(self) -> tuple: ... + @property + def run_end_type(self) -> DataType: ... + @property + def value_type(self) -> DataType: ... + +class BaseExtensionType(DataType): + def __arrow_ext_class__(self): ... + def __arrow_ext_scalar_class__(self): ... + @property + def extension_name(self) -> str: ... + @property + def storage_type(self) -> DataType: ... + @property + def byte_width(self): ... + @property + def bit_width(self): ... + def wrap_array(self, storage: Array | ChunkedArray) -> Array | ChunkedArray: ... + +class ExtensionType(BaseExtensionType): + def __init__(self, storage_type: DataType, extension_name): ... + def __eq__(self, other): ... + def __arrow_ext_serialize__(self): ... + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): ... + def __reduce__(self) -> tuple: ... + def __arrow_ext_class__(self): ... + def __arrow_ext_scalar_class__(self): ... + +class JsonType(BaseExtensionType): + def __arrow_ext_class__(self): ... + def __reduce__(self) -> tuple: ... + def __arrow_ext_scalar_class__(self): ... + +class UuidType(BaseExtensionType): + def __arrow_ext_class__(self): ... + def __reduce__(self) -> tuple: ... + def __arrow_ext_scalar_class__(self): ... + +class FixedShapeTensorType(BaseExtensionType): + @property + def value_type(self) -> DataType: ... + @property + def shape(self): ... + @property + def dim_names(self) -> list | None: ... + @property + def permutation(self): ... + def __arrow_ext_class__(self): ... + def __reduce__(self) -> tuple: ... + def __arrow_ext_scalar_class__(self): ... + +class Bool8Type(BaseExtensionType): + def __arrow_ext_class__(self): ... + def __reduce__(self) -> tuple: ... + def __arrow_ext_scalar_class__(self): ... + +class OpaqueType(BaseExtensionType): + @property + def type_name(self) -> str: ... + @property + def vendor_name(self) -> str: ... + def __arrow_ext_class__(self): ... + def __reduce__(self) -> tuple: ... + def __arrow_ext_scalar_class__(self): ... + +class UnknownExtensionType(ExtensionType): + def __init__(self, storage_type: DataType, serialized): ... + def __arrow_ext_serialize__(self): ... + @classmethod + def __arrow_ext_deserialize__( + cls, storage_type, serialized + ) -> UnknownExtensionType: ... + +class _Metadata(_Weakrefable): ... + +class KeyValueMetadata(_Metadata, Mapping): + def __init__(self, __arg0__=None, **kwargs): ... + def equals(self, other: KeyValueMetadata) -> bool: ... + def __eq__(self, other): ... + def __len__(self): ... + def __contains__(self, key): ... + def __getitem__(self, key): ... + def __iter__(self): ... + def __reduce__(self) -> tuple: ... + def key(self, i: int): ... + def value(self, i: int): ... + def keys(self) -> Iterator[Any]: ... + def values(self) -> Iterator[Any]: ... + def items(self) -> Iterator[tuple]: ... + def get_all(self, key: str) -> list: ... + def to_dict(self): ... + +class Field(_Weakrefable): + type: DataType + + def __init__(self): ... + def equals(self, other: Field, check_metadata: bool = False) -> bool: ... + def __eq__(self, other): ... + def __reduce__(self) -> tuple: ... + def __repr__(self): ... + def __hash__(self): ... + @property + def nullable(self): ... + @property + def name(self) -> str: ... + @property + def metadata(self) -> dict | None: ... + def with_metadata(self, metadata: dict) -> Field: ... + def remove_metadata(self) -> Field: ... + def with_type(self, new_type: DataType) -> Field: ... + def with_name(self, name: str) -> Field: ... + def with_nullable(self, nullable: bool) -> Field: ... + def flatten(self) -> list: ... + def _export_to_c(self, out_ptr) -> None: ... + @staticmethod + def _import_from_c(in_ptr) -> Field: ... + def __arrow_c_schema__(self): ... + @staticmethod + def _import_from_c_capsule(schema) -> Field: ... + +class Schema(_Weakrefable): + def __init__(self): ... + def __len__(self): ... + def __getitem__(self, key): ... + def __iter__(self) -> Iterator[Any]: ... + def __reduce__(self) -> tuple: ... + def __hash__(self): ... + def __sizeof__(self): ... + @property + def pandas_metadata(self): ... + @property + def names(self) -> list: ... + @property + def types(self) -> list: ... + @property + def metadata(self): ... + def __eq__(self, other): ... + def empty_table(self) -> Table: ... + def equals(self, other: Schema, check_metadata: bool = False) -> bool: ... + @classmethod + def from_pandas(cls, df, preserve_index: bool | None = None) -> Schema: ... + def field(self, i: int | str) -> Field: ... + def _field(self, i: int) -> Field: ... + def field_by_name(self, name: str) -> None | Field: ... + def get_field_index(self, name: str) -> int: ... + def get_all_field_indices(self, name: str) -> list[int]: ... + def append(self, field: Field): ... + def insert(self, i: int, field: Field) -> Schema: ... + def remove(self, i: int) -> Schema: ... + def set(self, i: int, field: Field) -> Schema: ... + def add_metadata(self, metadata: dict): ... + def with_metadata(self, metadata: dict) -> Schema: ... + def serialize(self, memory_pool: MemoryPool | None = None) -> Buffer: ... + def remove_metadata(self) -> Schema: ... + def to_string( + self, + truncate_metadata: bool = True, + show_field_metadata: bool = True, + show_schema_metadata: bool = True, + element_size_limit: int = 100, + ) -> str: ... + def _export_to_c(self, out_ptr) -> None: ... + @staticmethod + def _import_from_c(in_ptr) -> Schema: ... + def __str__(self): ... + def __repr__(self): ... + def __arrow_c_schema__(self): ... + @staticmethod + def _import_from_c_capsule(schema) -> Schema: ... + +class _ExtensionRegistryNanny(_Weakrefable): + def __init__(self): ... + def release_registry(self) -> None: ... + +class Scalar(_Weakrefable): + def __init__(self): ... + @property + def type(self) -> DataType: ... + @property + def is_valid(self): ... + def cast( + self, + target_type: object | None = None, + safe: bool | None = None, + options: pyarrow.compute.CastOptions | None = None, + memory_pool: MemoryPool | None = None, + ): ... + def validate(self, *, full: bool = False) -> None: ... + def equals(self, other: Scalar) -> bool: ... + def __eq__(self, other): ... + def __hash__(self): ... + def __reduce__(self) -> tuple: ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __abs__(self): ... + def __add__(self, other: object): ... + def __radd__(self, other: object): ... + def __truediv__(self, other: object): ... + def __rtruediv__(self, other: object): ... + def __mul__(self, other: object): ... + def __rmul__(self, other: object): ... + def __neg__(self): ... + def __pow__(self, other: object, mod: None = None): ... + def __rpow__(self, other: object, mod: None = None): ... + def __sub__(self, other: object): ... + def __rsub__(self, other: object): ... + def __and__(self, other: object): ... + def __rand__(self, other: object): ... + def __or__(self, other: object): ... + def __ror__(self, other: object): ... + def __xor__(self, other: object): ... + def __rxor__(self, other: object): ... + def __lshift__(self, other: object): ... + def __rlshift__(self, other: object): ... + def __rshift__(self, other: object): ... + def __rrshift__(self, other: object): ... + +class NullScalar(Scalar): + def __init__(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None) -> None: ... + +class BooleanScalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __bool__(self): ... + +class UInt8Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __index__(self): ... + +class Int8Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __index__(self): ... + +class UInt16Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __index__(self): ... + +class Int16Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __index__(self): ... + +class UInt32Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __index__(self): ... + +class Int32Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __index__(self): ... + +class UInt64Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __index__(self): ... + +class Int64Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __index__(self): ... + +class HalfFloatScalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __float__(self): ... + def __int__(self) -> int: ... + +class FloatScalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __float__(self): ... + def __int__(self) -> int: ... + +class DoubleScalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __float__(self): ... + def __int__(self) -> int: ... + +class Decimal32Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + +class Decimal64Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + +class Decimal128Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + +class Decimal256Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + +class Date32Scalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + +class Date64Scalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + +class Time32Scalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + +class Time64Scalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + +class TimestampScalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + +class DurationScalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + +class MonthDayNanoIntervalScalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + +class BinaryScalar(Scalar): + def as_buffer(self) -> Buffer | None: ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __bytes__(self): ... + +class LargeBinaryScalar(BinaryScalar): ... +class FixedSizeBinaryScalar(BinaryScalar): ... + +class StringScalar(BinaryScalar): + def as_py(self, *, maps_as_pydicts: str | None = None) -> None | str: ... + +class LargeStringScalar(StringScalar): ... +class BinaryViewScalar(BinaryScalar): ... +class StringViewScalar(StringScalar): ... + +class ListScalar(Scalar, Sequence): + @property + def values(self) -> Array | None: ... + def __len__(self) -> int: ... + def __getitem__(self, i): ... + def __iter__(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + +class FixedSizeListScalar(ListScalar): ... +class LargeListScalar(ListScalar): ... +class ListViewScalar(ListScalar): ... +class LargeListViewScalar(ListScalar): ... + +class StructScalar(Scalar, Mapping): + def __len__(self): ... + def __iter__(self) -> Iterator[str]: ... + def items(self): ... + def __contains__(self, key) -> bool: ... + def __getitem__(self, key) -> Scalar: ... + def as_py(self, *, maps_as_pydicts: str | None = None) -> dict | None: ... + def _as_py_tuple(self) -> list | None: ... + +class MapScalar(ListScalar, Mapping): + def __getitem__(self, i): ... + def __iter__(self) -> Iterator[tuple]: ... + def as_py(self, *, maps_as_pydicts: str | None = None) -> None | list | dict: ... + def keys(self) -> list: ... + +class DictionaryScalar(Scalar): + @staticmethod + def _reconstruct(type, is_valid, index, dictionary) -> Scalar: ... + def __reduce__(self) -> tuple: ... + @property + def index(self) -> Scalar: ... + @property + def value(self) -> Scalar: ... + @property + def dictionary(self) -> Array: ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + +class RunEndEncodedScalar(Scalar): + @property + def value(self) -> Scalar: ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + +class UnionScalar(Scalar): + @property + def value(self) -> Scalar | None: ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + @property + def type_code(self): ... + +class ExtensionScalar(Scalar): + @property + def value(self) -> Scalar | None: ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + @staticmethod + def from_storage(typ: BaseExtensionType, value: object) -> Scalar: ... + +class JsonScalar(ExtensionScalar): ... + +class UuidScalar(ExtensionScalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + +class FixedShapeTensorScalar(ExtensionScalar): + def to_numpy(self): ... + def to_tensor(self) -> Tensor: ... + def __dlpack__( + self, *, stream=None, max_version=None, dl_device=None, copy=None + ): ... + def __dlpack_device__(self) -> tuple: ... + +class OpaqueScalar(ExtensionScalar): ... + +class Bool8Scalar(ExtensionScalar): + def as_py(self, *, maps_as_pydicts: str | None = None) -> None | bool: ... + +class ArrayStatistics(_Weakrefable): + def __init__(self): ... + @property + def null_count(self): ... + @property + def is_null_count_exact(self): ... + @property + def distinct_count(self): ... + @property + def is_distinct_count_exact(self): ... + @property + def min(self): ... + @property + def is_min_exact(self): ... + @property + def max(self): ... + @property + def is_max_exact(self): ... + +class _PandasConvertible(_Weakrefable): + def to_pandas( + self, + memory_pool: MemoryPool | None = None, + categories: list | None = None, + strings_to_categorical: bool = False, + zero_copy_only: bool = False, + integer_object_nulls: bool = False, + date_as_object: bool = True, + timestamp_as_object: bool = False, + use_threads: bool = True, + deduplicate_objects: bool = True, + ignore_metadata: bool = False, + safe: bool = True, + split_blocks: bool = False, + self_destruct: bool = False, + maps_as_pydicts: str | None = None, + types_mapper: Callable[..., Any] | None = None, + coerce_temporal_nanoseconds: bool = False, + ): ... + +class Array(_PandasConvertible): + type: DataType + _name: object + + def __init__(self): ... + def _debug_print(self) -> None: ... + def diff(self, other: Array) -> str: ... + def cast( + self, + target_type: object | None = None, + safe: bool | None = None, + options: pyarrow.compute.CastOptions | None = None, + memory_pool: MemoryPool | None = None, + ) -> Array: ... + def view(self, target_type: object) -> Array: ... + def sum(self, **kwargs) -> Scalar: ... + def unique(self) -> Array: ... + def dictionary_encode(self, null_encoding: str = "mask") -> DictionaryArray: ... + def value_counts(self) -> StructArray: ... + @staticmethod + def from_pandas( + obj, + mask=None, + type: DataType | None = None, + safe: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Array | ChunkedArray: ... + def __reduce__(self) -> tuple: ... + @staticmethod + def from_buffers( + type: DataType, + length: int, + buffers, + null_count: int = -1, + offset: int = 0, + children: list[Array] | None = None, + ) -> Array: ... + @property + def null_count(self): ... + @property + def nbytes(self): ... + def get_total_buffer_size(self) -> int: ... + def __sizeof__(self): ... + def __iter__(self) -> Iterator[Any]: ... + def to_string( + self, + *, + indent: int = 2, + top_level_indent: int = 0, + window: int = 10, + container_window: int = 2, + skip_new_lines: bool = False, + element_size_limit: int = 100, + ) -> str: ... + def __str__(self): ... + def __eq__(self, other): ... + def equals(self: Array, other: Array) -> bool: ... + def __len__(self): ... + def is_null(self, *, nan_is_null: bool = False): ... + def is_nan(self): ... + def is_valid(self): ... + def fill_null(self, fill_value: Any) -> Array: ... + def __getitem__(self, key) -> Scalar | Array: ... + def slice( + self, offset: int | Scalar = 0, length: int | Scalar | None = None + ) -> Array: ... + def take(self, indices: object) -> Array: ... + def drop_null(self): ... + def filter( + self, mask: object, *, null_selection_behavior: str = "drop" + ) -> Array: ... + def index( + self, + value: Scalar | object, + start: int | None = None, + end: int | None = None, + *, + memory_pool: MemoryPool | None = None, + ) -> Int64Scalar: ... + def sort( + self, order: str = "ascending", null_placement: str = "at_end", **kwargs + ) -> Array: ... + def _to_pandas(self, options, types_mapper=None, **kwargs): ... + def __array__(self, dtype=None, copy=None): ... + def to_numpy(self, zero_copy_only: bool = True, writable: bool = False): ... + def to_tensor(self, *, allow_nulls: bool = False) -> Tensor: ... + def to_pylist(self, *, maps_as_pydicts: str | None = None) -> list: ... + def tolist(self): ... + def validate(self, *, full: bool = False) -> None: ... + @property + def offset(self): ... + def buffers(self) -> list: ... + def copy_to(self, destination: MemoryManager | Device) -> Array: ... + def _export_to_c(self, out_ptr, out_schema_ptr: int = 0) -> None: ... + @staticmethod + def _import_from_c(in_ptr, type) -> Array: ... + def __arrow_c_array__(self, requested_schema=None) -> tuple: ... + @staticmethod + def _import_from_c_capsule(schema_capsule, array_capsule) -> Array: ... + def _export_to_c_device(self, out_ptr, out_schema_ptr: int = 0) -> None: ... + @staticmethod + def _import_from_c_device(in_ptr, type) -> Array: ... + def __arrow_c_device_array__(self, requested_schema=None, **kwargs) -> tuple: ... + @staticmethod + def _import_from_c_device_capsule(schema_capsule, array_capsule) -> Array: ... + @staticmethod + def from_dlpack( + x: object, /, *, device=None, copy: bool | None = None + ) -> Array: ... + def __dlpack__( + self, + *, + stream: int | None = None, + max_version=None, + dl_device=None, + copy: bool | None = None, + ): ... + def __dlpack_device__(self) -> tuple: ... + @property + def device_type(self) -> DeviceAllocationType: ... + @property + def is_cpu(self) -> bool: ... + @property + def statistics(self) -> None | ArrayStatistics: ... + def __abs__(self): ... + def __add__(self, other: object): ... + def __radd__(self, other: object): ... + def __truediv__(self, other: object): ... + def __rtruediv__(self, other: object): ... + def __mul__(self, other: object): ... + def __rmul__(self, other: object): ... + def __neg__(self): ... + def __pow__(self, other: object, mod: None = None): ... + def __rpow__(self, other: object, mod: None = None): ... + def __sub__(self, other: object): ... + def __rsub__(self, other: object): ... + def __and__(self, other: object): ... + def __rand__(self, other: object): ... + def __or__(self, other: object): ... + def __ror__(self, other: object): ... + def __xor__(self, other: object): ... + def __rxor__(self, other: object): ... + def __lshift__(self, other: object): ... + def __rlshift__(self, other: object): ... + def __rshift__(self, other: object): ... + def __rrshift__(self, other: object): ... + +class NullArray(Array): ... + +class BooleanArray(Array): + @property + def false_count(self): ... + @property + def true_count(self): ... + +class NumericArray(Array): ... +class IntegerArray(NumericArray): ... +class FloatingPointArray(NumericArray): ... +class Int8Array(IntegerArray): ... +class UInt8Array(IntegerArray): ... +class Int16Array(IntegerArray): ... +class UInt16Array(IntegerArray): ... +class Int32Array(IntegerArray): ... +class UInt32Array(IntegerArray): ... +class Int64Array(IntegerArray): ... +class UInt64Array(IntegerArray): ... +class Date32Array(NumericArray): ... +class Date64Array(NumericArray): ... +class TimestampArray(NumericArray): ... +class Time32Array(NumericArray): ... +class Time64Array(NumericArray): ... +class DurationArray(NumericArray): ... + +class MonthDayNanoIntervalArray(Array): + def to_pylist(self, *, maps_as_pydicts: str | None = None) -> list: ... + +class HalfFloatArray(FloatingPointArray): ... +class FloatArray(FloatingPointArray): ... +class DoubleArray(FloatingPointArray): ... +class FixedSizeBinaryArray(Array): ... +class Decima32Array(FixedSizeBinaryArray): ... +class Decimal64Array(FixedSizeBinaryArray): ... +class Decimal128Array(FixedSizeBinaryArray): ... +class Decimal256Array(FixedSizeBinaryArray): ... + +class BaseListArray(Array): + def flatten(self, recursive: bool = False) -> Array: ... + def value_parent_indices(self): ... + def value_lengths(self): ... + +class ListArray(BaseListArray): + @staticmethod + def from_arrays( + offsets: Array, + values: Array, + type: DataType | None = None, + pool: MemoryPool | None = None, + mask: Array | None = None, + ) -> Array: ... + @property + def values(self) -> Array: ... + @property + def offsets(self) -> Array: ... + +class LargeListArray(BaseListArray): + @staticmethod + def from_arrays( + offsets: Array, + values: Array, + type: DataType | None = None, + pool: MemoryPool | None = None, + mask: Array | None = None, + ) -> Array: ... + @property + def values(self) -> Array: ... + @property + def offsets(self) -> Array: ... + +class ListViewArray(BaseListArray): + @staticmethod + def from_arrays( + offsets: Array, + sizes: Array, + values: Array, + type: DataType | None = None, + pool: MemoryPool | None = None, + mask: Array | None = None, + ) -> Array: ... + @property + def values(self) -> Array: ... + @property + def offsets(self) -> Array: ... + @property + def sizes(self) -> Array: ... + +class LargeListViewArray(BaseListArray): + @staticmethod + def from_arrays( + offsets: Array, + sizes: Array, + values: Array, + type: DataType | None = None, + pool: MemoryPool | None = None, + mask: Array | None = None, + ) -> Array: ... + @property + def values(self) -> Array: ... + @property + def offsets(self) -> Array: ... + @property + def sizes(self) -> Array: ... + +class MapArray(ListArray): + @staticmethod + def from_arrays( + offsets, + keys, + items, + type: DataType | None = None, + pool: MemoryPool | None = None, + mask: Array | None = None, + ) -> Array: ... + @property + def keys(self) -> Array: ... + @property + def items(self) -> Array: ... + +class FixedSizeListArray(BaseListArray): + @staticmethod + def from_arrays( + values: Array, + list_size: int | None = None, + type: DataType | None = None, + mask: Array | None = None, + ) -> Array: ... + @property + def values(self) -> Array: ... + +class UnionArray(Array): + def child(self, pos: int) -> Field: ... + def field(self, pos: int) -> Array: ... + @property + def type_codes(self) -> Array: ... + @property + def offsets(self) -> Array: ... + @staticmethod + def from_dense( + types: Array, + value_offsets: Array, + children: list[Any], + field_names: list[Any] | None = None, + type_codes: list[Any] | None = None, + ) -> Array: ... + @staticmethod + def from_sparse( + types: Array, + children: list[Any], + field_names: list[Any] | None = None, + type_codes: list[Any] | None = None, + ) -> Array: ... + +class StringArray(Array): + @staticmethod + def from_buffers( + length: int, + value_offsets: Buffer, + data: Buffer, + null_bitmap: Buffer | None = None, + null_count: int = -1, + offset: int = 0, + ) -> Array: ... + +class LargeStringArray(Array): + @staticmethod + def from_buffers( + length: int, + value_offsets: Buffer, + data: Buffer, + null_bitmap: Buffer | None = None, + null_count: int = -1, + offset: int = 0, + ) -> Array: ... + +class StringViewArray(Array): ... + +class BinaryArray(Array): + @property + def total_values_length(self): ... + +class LargeBinaryArray(Array): + @property + def total_values_length(self): ... + +class BinaryViewArray(Array): ... + +class DictionaryArray(Array): + def dictionary_encode(self) -> DictionaryArray: ... + def dictionary_decode(self): ... + @property + def dictionary(self): ... + @property + def indices(self): ... + @staticmethod + def from_buffers( + type: DataType, + length: int, + buffers, + dictionary: Array, + null_count: int = -1, + offset: int = 0, + ) -> Array: ... + @staticmethod + def from_arrays( + indices, + dictionary, + mask=None, + ordered: bool = False, + from_pandas: bool = False, + safe: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Array: ... + +class StructArray(Array): + def field(self, index) -> Array: ... + def _flattened_field( + self, index, memory_pool: MemoryPool | None = None + ) -> Array: ... + def flatten(self, memory_pool: MemoryPool | None = None) -> list: ... + @staticmethod + def from_arrays( + arrays: Sequence[Array], + names: list[str] | None = None, + fields: list[Field] | None = None, + mask=None, + memory_pool: MemoryPool | None = None, + type: StructType | None = None, + ) -> StructArray: ... + def sort( + self, + order: str = "ascending", + null_placement: str = "at_end", + by: str | None = None, + **kwargs, + ) -> StructArray: ... + +class RunEndEncodedArray(Array): + @staticmethod + def _from_arrays( + type, allow_none_for_type, logical_length, run_ends, values, logical_offset + ) -> Array: ... + @staticmethod + def from_arrays( + run_ends, values: Array, type: DataType | None = None + ) -> RunEndEncodedArray: ... + @staticmethod + def from_buffers( + type: DataType, + length: int, + buffers: list[Buffer], + null_count: int = -1, + offset: int = 0, + children: list[Array] | None = None, + ) -> RunEndEncodedArray: ... + @property + def run_ends(self) -> Array: ... + @property + def values(self) -> Array: ... + def find_physical_offset(self): ... + def find_physical_length(self): ... + +class ExtensionArray(Array): + @property + def storage(self) -> Array: ... + @staticmethod + def from_storage(typ: BaseExtensionType, storage: Array) -> Array: ... + +class JsonArray(ExtensionArray): ... +class UuidArray(ExtensionArray): ... + +class FixedShapeTensorArray(ExtensionArray): + def to_numpy_ndarray(self): ... + @staticmethod + def from_tensor(tensor: Tensor) -> Array: ... + @staticmethod + def from_numpy_ndarray( + obj, dim_names: tuple | list[str] | None = None + ) -> ExtensionArray: ... + @staticmethod + def from_dlpack( + x: object, /, *, device=None, copy: bool | None = None + ) -> FixedShapeTensorArray: ... + def __dlpack__( + self, *, stream=None, max_version=None, dl_device=None, copy=None + ): ... + +class OpaqueArray(ExtensionArray): ... + +class Bool8Array(ExtensionArray): + def to_numpy(self, zero_copy_only: bool = True, writable: bool = False): ... + @staticmethod + def from_storage(storage: Int8Array) -> ExtensionArray: ... + @staticmethod + def from_numpy(obj) -> Bool8Array: ... + +class StringBuilder(_Weakrefable): + def __init__(self, memory_pool: MemoryPool | None = None): ... + def append(self, value) -> None: ... + def append_values(self, values) -> None: ... + def finish(self) -> Array: ... + @property + def null_count(self): ... + def __len__(self): ... + +class StringViewBuilder(_Weakrefable): + def __init__(self, memory_pool: MemoryPool | None = None): ... + def append(self, value) -> None: ... + def append_values(self, values) -> None: ... + def finish(self) -> Array: ... + @property + def null_count(self): ... + def __len__(self): ... + +class ChunkedArray(_PandasConvertible): + _name: object + + def __init__(self): ... + def __reduce__(self) -> tuple: ... + @property + def data(self) -> ChunkedArray: ... + @property + def type(self) -> DataType: ... + def length(self): ... + def __len__(self): ... + def to_string( + self, + *, + indent: int = 0, + window: int = 5, + container_window: int = 2, + skip_new_lines: bool = False, + element_size_limit: int = 100, + ) -> str: ... + def format(self, **kwargs) -> str: ... + def __str__(self): ... + def validate(self, *, full: bool = False) -> None: ... + @property + def null_count(self) -> int: ... + @property + def nbytes(self): ... + def get_total_buffer_size(self) -> int: ... + def __sizeof__(self): ... + def __iter__(self) -> Iterator[Any]: ... + def __getitem__(self, key) -> Scalar | ChunkedArray: ... + def is_null(self, *, nan_is_null: bool = False): ... + def is_nan(self): ... + def is_valid(self): ... + def __eq__(self, other): ... + def fill_null(self, fill_value: Any) -> Array | ChunkedArray: ... + def equals(self, other: ChunkedArray) -> bool: ... + def _to_pandas(self, options, types_mapper=None, **kwargs): ... + def to_numpy(self, zero_copy_only: bool = False): ... + def __array__(self, dtype=None, copy=None): ... + def cast( + self, + target_type: object | None = None, + safe: bool | None = None, + options: pyarrow.compute.CastOptions | None = None, + ) -> Array | ChunkedArray: ... + def dictionary_encode(self, null_encoding: str = "mask") -> ChunkedArray: ... + def flatten(self, memory_pool: MemoryPool | None = None) -> list: ... + def combine_chunks(self, memory_pool: MemoryPool | None = None) -> Array: ... + def unique(self) -> Array: ... + def value_counts(self): ... + def slice( + self, offset: int | Scalar = 0, length: int | Scalar | None = None + ) -> ChunkedArray: ... + def filter( + self, mask, null_selection_behavior: object = "drop" + ) -> Array | ChunkedArray: ... + def index( + self, + value: Scalar | object, + start: int | None = None, + end: int | None = None, + *, + memory_pool: MemoryPool | None = None, + ) -> Int64Scalar: ... + def take(self, indices: object) -> Array | ChunkedArray: ... + def drop_null(self): ... + def sort( + self, order: str = "ascending", null_placement: str = "at_end", **kwargs + ) -> ChunkedArray: ... + def unify_dictionaries( + self, memory_pool: MemoryPool | None = None + ) -> ChunkedArray: ... + @property + def num_chunks(self) -> int: ... + def chunk(self, i: int) -> Array: ... + @property + def chunks(self) -> list: ... + def iterchunks(self) -> Iterator[Any]: ... + def to_pylist(self, *, maps_as_pydicts: str | None = None) -> list: ... + def __arrow_c_stream__(self, requested_schema=None): ... + @staticmethod + def _import_from_c_capsule(stream) -> ChunkedArray: ... + @property + def is_cpu(self): ... + def _assert_cpu(self) -> None: ... + +class _Tabular(_PandasConvertible): + def __init__(self): ... + def __array__(self, dtype=None, copy=None): ... + def __dataframe__(self, nan_as_null: bool = False, allow_copy: bool = True): ... + def __eq__(self, other): ... + def __getitem__(self, key): ... + def __len__(self): ... + def __repr__(self): ... + def _column(self, i: int): ... + def _ensure_integer_index(self, i): ... + def _is_initialized(self): ... + def column(self, i: int | str) -> Array | ChunkedArray: ... + @property + def column_names(self) -> list: ... + @property + def columns(self) -> list: ... + def drop_null(self) -> Table | RecordBatch: ... + def field(self, i: int | str) -> Field: ... + @classmethod + def from_pydict( + cls, + mapping: dict | Mapping[Any, Any], + schema: Schema | None = None, + metadata: dict | Mapping[Any, Any] | None = None, + ) -> Table | RecordBatch: ... + @classmethod + def from_pylist( + cls, + mapping, + schema: Schema | None = None, + metadata: dict | Mapping[Any, Any] | None = None, + ) -> Table | RecordBatch: ... + def itercolumns(self) -> Iterator[Any]: ... + @property + def num_columns(self): ... + @property + def num_rows(self): ... + @property + def shape(self) -> tuple: ... + @property + def schema(self): ... + def sort_by(self, sorting, **kwargs) -> Table | RecordBatch: ... + def take(self, indices: object) -> Table | RecordBatch: ... + def filter( + self, mask, null_selection_behavior: object = "drop" + ) -> Table | RecordBatch: ... + def to_pydict(self, *, maps_as_pydicts: str | None = None) -> dict: ... + def to_pylist(self, *, maps_as_pydicts: str | None = None) -> list: ... + def to_string( + self, *, show_metadata: bool = False, preview_cols: int = 0 + ) -> str: ... + def remove_column(self, i: int): ... + def drop_columns(self, columns: str | list[str]) -> _Tabular: ... + def add_column(self, i: int, field_, column): ... + def append_column(self, field_: str | Field, column) -> Table | RecordBatch: ... + +class RecordBatch(_Tabular): + def __init__(self): ... + def _is_initialized(self) -> bool: ... + def __reduce__(self) -> tuple: ... + def validate(self, *, full: bool = False) -> None: ... + def replace_schema_metadata(self, metadata: dict | None = None) -> RecordBatch: ... + @property + def num_columns(self) -> int: ... + @property + def num_rows(self) -> int: ... + @property + def schema(self) -> Schema: ... + def _column(self, i: int) -> Array: ... + @property + def nbytes(self): ... + def get_total_buffer_size(self) -> int: ... + def __sizeof__(self): ... + def add_column(self, i: int, field_: str | Field, column) -> RecordBatch: ... + def remove_column(self, i: int) -> RecordBatch: ... + def set_column(self, i: int, field_: str | Field, column) -> RecordBatch: ... + def rename_columns(self, names) -> RecordBatch: ... + def serialize(self, memory_pool: MemoryPool | None = None) -> Buffer: ... + def slice( + self, offset: int | Scalar = 0, length: int | Scalar | None = None + ) -> RecordBatch: ... + def equals(self, other: object, check_metadata: bool = False) -> bool: ... + def select(self, columns: object) -> RecordBatch: ... + def cast( + self, + target_schema: Schema, + safe: bool | None = None, + options: pyarrow.compute.CastOptions | None = None, + ) -> RecordBatch: ... + def _to_pandas(self, options, **kwargs): ... + @classmethod + def from_pandas( + cls, + df, + schema: Schema | None = None, + preserve_index: bool | None = None, + nthreads: int | None = None, + columns: list | None = None, + ) -> RecordBatch | Self: ... + @staticmethod + def from_arrays( + arrays: list[Any], + names: list[str] | None = None, + schema: Schema | None = None, + metadata: dict | Mapping[Any, Any] | None = None, + ) -> RecordBatch: ... + @staticmethod + def from_struct_array(struct_array: StructArray) -> RecordBatch: ... + def to_struct_array(self) -> Array: ... + def to_tensor( + self, + null_to_nan: bool = False, + row_major: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Tensor: ... + def copy_to(self, destination: MemoryManager | Device) -> RecordBatch: ... + def _export_to_c(self, out_ptr, out_schema_ptr: int = 0) -> None: ... + @staticmethod + def _import_from_c(in_ptr, schema) -> RecordBatch: ... + def __arrow_c_array__(self, requested_schema=None) -> tuple: ... + def __arrow_c_stream__(self, requested_schema=None): ... + @staticmethod + def _import_from_c_capsule(schema_capsule, array_capsule) -> RecordBatch: ... + def _export_to_c_device(self, out_ptr, out_schema_ptr: int = 0) -> None: ... + @staticmethod + def _import_from_c_device(in_ptr, schema) -> RecordBatch: ... + def __arrow_c_device_array__(self, requested_schema=None, **kwargs) -> tuple: ... + @staticmethod + def _import_from_c_device_capsule(schema_capsule, array_capsule) -> RecordBatch: ... + @property + def device_type(self) -> DeviceAllocationType: ... + @property + def is_cpu(self) -> bool: ... + +class Table(_Tabular): + def __init__(self): ... + def _is_initialized(self) -> bool: ... + def validate(self, *, full: bool = False) -> None: ... + def __reduce__(self) -> tuple: ... + def slice( + self, offset: int | Scalar = 0, length: int | Scalar | None = None + ) -> Table: ... + def select(self, columns: object) -> Table: ... + def replace_schema_metadata(self, metadata: dict | None = None) -> Table: ... + def flatten(self, memory_pool: MemoryPool | None = None) -> Table: ... + def combine_chunks(self, memory_pool: MemoryPool | None = None) -> Table: ... + def unify_dictionaries(self, memory_pool: MemoryPool | None = None) -> Table: ... + def equals(self, other: Table, check_metadata: bool = False) -> bool: ... + def cast( + self, + target_schema: Schema, + safe: bool | None = None, + options: pyarrow.compute.CastOptions | None = None, + ) -> Table: ... + @classmethod + def from_pandas( + cls, + df, + schema: Schema | None = None, + preserve_index: bool | None = None, + nthreads: int | None = None, + columns: list | None = None, + safe: bool = True, + ) -> Table | Self: ... + @staticmethod + def from_arrays( + arrays: list[Array] | ChunkedArray, + names: list[str] | None = None, + schema: Schema | None = None, + metadata: dict | Mapping[Any, Any] | None = None, + ) -> Table: ... + @staticmethod + def from_struct_array(struct_array: StructArray | ChunkedArray) -> Table: ... + def to_struct_array(self, max_chunksize: int | None = None) -> ChunkedArray: ... + @staticmethod + def from_batches( + batches: Sequence[Any] | Iterator[RecordBatch], schema: Schema | None = None + ) -> Table: ... + def to_batches(self, max_chunksize: int | None = None) -> list[RecordBatch]: ... + def to_tensor( + self, + null_to_nan: bool = False, + row_major: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Tensor: ... + def to_reader(self, max_chunksize: int | None = None) -> RecordBatchReader: ... + def _to_pandas( + self, options, categories=None, ignore_metadata: bool = False, types_mapper=None + ): ... + @property + def schema(self) -> Schema: ... + def _column(self, i: int) -> ChunkedArray: ... + @property + def num_columns(self) -> int: ... + @property + def num_rows(self) -> int: ... + @property + def nbytes(self): ... + def get_total_buffer_size(self) -> int: ... + def __sizeof__(self): ... + def add_column(self, i: int, field_: str | Field, column) -> Table: ... + def remove_column(self, i: int) -> Table: ... + def set_column(self, i: int, field_: str | Field, column) -> Table: ... + def rename_columns(self, names) -> Table: ... + def drop(self, columns: str | list[str]) -> Table: ... + def group_by( + self, keys: str | list[str], use_threads: bool = True + ) -> TableGroupBy: ... + def join( + self, + right_table: Table, + keys: str | list[str], + right_keys: str | list[str] | None = None, + join_type: str = "left outer", + left_suffix: str | None = None, + right_suffix: str | None = None, + coalesce_keys: bool = True, + use_threads: bool = True, + filter_expression: pyarrow.compute.Expression | None = None, + ) -> Table: ... + def join_asof( + self, + right_table: Table, + on: str, + by: str | list[str], + tolerance: int, + right_on: str | list[str] | None = None, + right_by: str | list[str] | None = None, + ) -> Table: ... + def __arrow_c_stream__(self, requested_schema=None): ... + @property + def is_cpu(self): ... + +class TableGroupBy: + def __init__(self, table, keys, use_threads: bool = True): ... + def aggregate(self, aggregations) -> Table: ... + +class Tensor(_Weakrefable): + type: DataType + _ssize_t_shape: bytes + _ssize_t_strides: bytes + + def __init__(self): ... + def _make_shape_or_strides_buffer(self, values): ... + @staticmethod + def from_numpy(obj, dim_names: list | None = None) -> Tensor: ... + def to_numpy(self): ... + def equals(self, other: Tensor): ... + def __eq__(self, other): ... + def dim_name(self, i: int) -> str: ... + @property + def dim_names(self) -> list: ... + @property + def is_mutable(self): ... + @property + def is_contiguous(self): ... + @property + def ndim(self): ... + @property + def size(self): ... + @property + def shape(self) -> tuple: ... + @property + def strides(self) -> tuple: ... + @staticmethod + def from_dlpack( + x: object, /, *, device=None, copy: bool | None = None + ) -> Tensor: ... + def __dlpack__( + self, + *, + stream: int | None = None, + max_version=None, + dl_device=None, + copy: bool | None = None, + ): ... + def __dlpack_device__(self) -> tuple: ... + +class SparseCOOTensor(_Weakrefable): + type: DataType + + def __init__(self): ... + @classmethod + def from_dense_numpy(cls, obj, dim_names: list[str] | None = None) -> Self: ... + @staticmethod + def from_numpy( + data, coords, shape: tuple, dim_names: list | None = None + ) -> SparseCOOTensor: ... + @staticmethod + def from_scipy(obj, dim_names: list | None = None) -> SparseCOOTensor: ... + @staticmethod + def from_pydata_sparse(obj, dim_names: list | None = None) -> SparseCOOTensor: ... + @staticmethod + def from_tensor(obj: Tensor) -> SparseCOOTensor: ... + def to_numpy(self) -> tuple: ... + def to_scipy(self): ... + def to_pydata_sparse(self): ... + def to_tensor(self) -> Tensor: ... + def equals(self, other: SparseCOOTensor): ... + def __eq__(self, other): ... + @property + def is_mutable(self): ... + @property + def ndim(self): ... + @property + def shape(self) -> tuple: ... + @property + def size(self): ... + def dim_name(self, i: int) -> str: ... + @property + def dim_names(self) -> tuple: ... + @property + def non_zero_length(self): ... + @property + def has_canonical_format(self): ... + +class SparseCSRMatrix(_Weakrefable): + type: DataType + + def __init__(self): ... + @classmethod + def from_dense_numpy(cls, obj, dim_names: list | None = None) -> Self: ... + @staticmethod + def from_numpy( + data, indptr, indices, shape: tuple, dim_names: list | None = None + ) -> SparseCSRMatrix: ... + @staticmethod + def from_scipy(obj, dim_names: list | None = None) -> SparseCSRMatrix: ... + @staticmethod + def from_tensor(obj: Tensor) -> SparseCSRMatrix: ... + def to_numpy(self) -> tuple: ... + def to_scipy(self): ... + def to_tensor(self) -> Tensor: ... + def equals(self, other: SparseCSRMatrix): ... + def __eq__(self, other): ... + @property + def is_mutable(self): ... + @property + def ndim(self): ... + @property + def shape(self) -> tuple: ... + @property + def size(self): ... + def dim_name(self, i: int) -> str: ... + @property + def dim_names(self) -> tuple: ... + @property + def non_zero_length(self): ... + +class SparseCSCMatrix(_Weakrefable): + type: DataType + + def __init__(self): ... + @classmethod + def from_dense_numpy(cls, obj, dim_names: list[str] | None = None) -> Self: ... + @staticmethod + def from_numpy( + data, indptr, indices, shape: tuple, dim_names: list | None = None + ) -> SparseCSCMatrix: ... + @staticmethod + def from_scipy(obj, dim_names: list | None = None) -> SparseCSCMatrix: ... + @staticmethod + def from_tensor(obj: Tensor) -> SparseCSCMatrix: ... + def to_numpy(self) -> tuple: ... + def to_scipy(self): ... + def to_tensor(self) -> Tensor: ... + def equals(self, other: SparseCSCMatrix): ... + def __eq__(self, other): ... + @property + def is_mutable(self): ... + @property + def ndim(self): ... + @property + def shape(self) -> tuple: ... + @property + def size(self): ... + def dim_name(self, i: int) -> str: ... + @property + def dim_names(self) -> tuple: ... + @property + def non_zero_length(self): ... + +class SparseCSFTensor(_Weakrefable): + type: DataType + + def __init__(self): ... + @classmethod + def from_dense_numpy(cls, obj, dim_names: list[str] | None = None) -> Self: ... + @staticmethod + def from_numpy( + data, + indptr, + indices, + shape: tuple, + axis_order: list | None = None, + dim_names: list | None = None, + ) -> SparseCSFTensor: ... + @staticmethod + def from_tensor(obj: Tensor) -> SparseCSFTensor: ... + def to_numpy(self) -> tuple: ... + def to_tensor(self) -> Tensor: ... + def equals(self, other: SparseCSFTensor): ... + def __eq__(self, other): ... + @property + def is_mutable(self): ... + @property + def ndim(self): ... + @property + def shape(self) -> tuple: ... + @property + def size(self): ... + def dim_name(self, i: int) -> str: ... + @property + def dim_names(self) -> tuple: ... + @property + def non_zero_length(self): ... + +class NativeFile(_Weakrefable): + _default_chunk_size: int + + def __init__(self): ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type, exc_value, tb) -> None: ... + @property + def mode(self) -> str: ... + def readable(self): ... + def writable(self): ... + def seekable(self): ... + def isatty(self) -> bool: ... + def fileno(self): ... + @property + def closed(self): ... + def close(self) -> None: ... + def _assert_open(self) -> None: ... + def _assert_readable(self) -> None: ... + def _assert_writable(self) -> None: ... + def _assert_seekable(self) -> None: ... + def size(self) -> int: ... + def metadata(self) -> dict: ... + def tell(self) -> int: ... + def seek(self, position: int, whence: int = 0) -> int: ... + def flush(self) -> None: ... + def write(self, data) -> int: ... + def read(self, nbytes: int | None = None) -> bytes: ... + def get_stream(self, file_offset: int, nbytes: int) -> NativeFile: ... + def read_at(self, nbytes: int, offset: int) -> bytes: ... + def read1(self, nbytes: int | None = None) -> bytes: ... + def readall(self): ... + def readinto(self, b) -> int: ... + def readline(self, size: int | None = None): ... + def readlines(self, hint: int | None = None): ... + def __iter__(self) -> Self: ... + def __next__(self): ... + def read_buffer(self, nbytes: int | None = None): ... + def truncate(self): ... + def writelines(self, lines: Iterable[Any]) -> None: ... + def download( + self, stream_or_path: str | IO[Any], buffer_size: int | None = None + ): ... + def _download_nothreads(self, stream_or_path, buffer_size=None) -> None: ... + def upload(self, stream: IO[Any], buffer_size: int | None = None): ... + def _upload_nothreads(self, stream, buffer_size=None) -> None: ... + +class PythonFile(NativeFile): + def __init__(self, handle, mode=None): ... + def truncate(self, pos: int | None = None) -> None: ... + def readline(self, size: int | None = None): ... + def readlines(self, hint: int | None = None): ... + +class MemoryMappedFile(NativeFile): + @staticmethod + def create(path: str, size: int) -> MemoryMappedFile: ... + def _open(self, path, mode: str = "r") -> None: ... + def resize(self, new_size) -> None: ... + def fileno(self): ... + +class OSFile(NativeFile): + def __init__( + self, path, mode: str = "r", memory_pool: MemoryPool | None = None + ): ... + def fileno(self): ... + +class FixedSizeBufferWriter(NativeFile): + def __init__(self, buffer: Buffer): ... + def set_memcopy_threads(self, num_threads: int) -> None: ... + def set_memcopy_blocksize(self, blocksize: int) -> None: ... + def set_memcopy_threshold(self, threshold: int) -> None: ... + +class Buffer(_Weakrefable): + def __init__(self): ... + def __len__(self): ... + def _assert_cpu(self) -> None: ... + @property + def size(self): ... + @property + def address(self): ... + def hex(self): ... + @property + def is_mutable(self): ... + @property + def is_cpu(self): ... + @property + def device(self) -> Device: ... + @property + def memory_manager(self) -> MemoryManager: ... + @property + def device_type(self) -> DeviceAllocationType: ... + @property + def parent(self) -> None | Buffer: ... + def __getitem__(self, key): ... + def slice(self, offset: int = 0, length: int | None = None) -> Buffer: ... + def equals(self, other: Buffer) -> bool: ... + def __eq__(self, other): ... + def __reduce_ex__(self, protocol) -> tuple: ... + def to_pybytes(self): ... + +class ResizableBuffer(Buffer): + def resize(self, new_size: int, shrink_to_fit: bool = False) -> None: ... + +class BufferOutputStream(NativeFile): + def __init__(self, memory_pool: MemoryPool | None = None): ... + def getvalue(self) -> Buffer: ... + +class MockOutputStream(NativeFile): + def __init__(self): ... + def size(self): ... + +class BufferReader(NativeFile): + def __init__(self, obj): ... + +class CompressedInputStream(NativeFile): + def __init__(self, stream: object, compression: str): ... + +class CompressedOutputStream(NativeFile): + def __init__(self, stream: object, compression: str): ... + +class BufferedInputStream(NativeFile): + def __init__( + self, + stream: NativeFile, + buffer_size: int, + memory_pool: MemoryPool | None = None, + ): ... + def detach(self) -> NativeFile: ... + +class BufferedOutputStream(NativeFile): + def __init__( + self, + stream: NativeFile, + buffer_size: int, + memory_pool: MemoryPool | None = None, + ): ... + def detach(self) -> NativeFile: ... + +class TransformInputStream(NativeFile): + def __init__(self, stream: NativeFile, transform_func): ... + +class Transcoder: + def __init__(self, decoder, encoder): ... + def __call__(self, buf): ... + +class CacheOptions(_Weakrefable): + def __init__( + self, + *, + hole_size_limit=None, + range_size_limit=None, + lazy=None, + prefetch_limit=None, + ): ... + @property + def hole_size_limit(self) -> None: ... + @hole_size_limit.setter + def hole_size_limit(self, hole_size_limit) -> None: ... + @property + def range_size_limit(self) -> None: ... + @range_size_limit.setter + def range_size_limit(self, range_size_limit) -> None: ... + @property + def lazy(self) -> None: ... + @lazy.setter + def lazy(self, lazy) -> None: ... + @property + def prefetch_limit(self) -> None: ... + @prefetch_limit.setter + def prefetch_limit(self, prefetch_limit) -> None: ... + def __eq__(self, other: CacheOptions): ... + @staticmethod + def from_network_metrics( + time_to_first_byte_millis: int, + transfer_bandwidth_mib_per_sec: int, + ideal_bandwidth_utilization_frac: float = 0.9, + max_ideal_request_size_mib: int = 64, + ) -> CacheOptions: ... + @staticmethod + def _reconstruct(kwargs) -> CacheOptions: ... + def __reduce__(self) -> tuple: ... + +class Codec(_Weakrefable): + def __init__(self, compression: str, compression_level=None): ... + @staticmethod + def detect(path: str | os.PathLike[str]) -> Codec: ... + @staticmethod + def is_available(compression: str) -> bool: ... + @staticmethod + def supports_compression_level(compression: str): ... + @staticmethod + def default_compression_level(compression: str): ... + @staticmethod + def minimum_compression_level(compression: str): ... + @staticmethod + def maximum_compression_level(compression: str): ... + @property + def name(self) -> str: ... + @property + def compression_level(self): ... + def compress( + self, buf: object, asbytes: bool = False, memory_pool: MemoryPool | None = None + ) -> Buffer | bytes: ... + def decompress( + self, + buf: object, + decompressed_size: int | None = None, + asbytes: bool = False, + memory_pool: MemoryPool | None = None, + ) -> Buffer | bytes: ... + +class WriteStats(_WriteStats): + __slots__ = () + +class ReadStats(_ReadStats): + __slots__ = () + +class IpcReadOptions(_Weakrefable): + __slots__ = () + + def __init__( + self, + *, + ensure_native_endian: bool = True, + ensure_alignment: Alignment = ..., + use_threads: bool = True, + included_fields: list[Any] | None = None, + ): ... + @property + def ensure_native_endian(self) -> None: ... + @ensure_native_endian.setter + def ensure_native_endian(self, value: bool) -> None: ... + @property + def ensure_alignment(self) -> None: ... + @ensure_alignment.setter + def ensure_alignment(self, value: Alignment) -> None: ... + @property + def use_threads(self) -> None: ... + @use_threads.setter + def use_threads(self, value: bool) -> None: ... + @property + def included_fields(self) -> None: ... + @included_fields.setter + def included_fields(self, value: list[Any]) -> None: ... + +class IpcWriteOptions(_Weakrefable): + __slots__ = () + + def __init__( + self, + *, + metadata_version=..., + allow_64bit: bool = False, + use_legacy_format: bool = False, + compression=None, + use_threads: bool = True, + emit_dictionary_deltas: bool = False, + unify_dictionaries: bool = False, + ): ... + @property + def allow_64bit(self) -> None: ... + @allow_64bit.setter + def allow_64bit(self, value: bool) -> None: ... + @property + def use_legacy_format(self) -> None: ... + @use_legacy_format.setter + def use_legacy_format(self, value: bool) -> None: ... + @property + def metadata_version(self) -> None: ... + @metadata_version.setter + def metadata_version(self, value) -> None: ... + @property + def compression(self) -> None: ... + @compression.setter + def compression(self, value) -> None: ... + @property + def use_threads(self) -> None: ... + @use_threads.setter + def use_threads(self, value: bool) -> None: ... + @property + def emit_dictionary_deltas(self) -> None: ... + @emit_dictionary_deltas.setter + def emit_dictionary_deltas(self, value: bool) -> None: ... + @property + def unify_dictionaries(self) -> None: ... + @unify_dictionaries.setter + def unify_dictionaries(self, value: bool) -> None: ... + +class Message(_Weakrefable): + def __init__(self): ... + @property + def type(self) -> str: ... + @property + def metadata(self) -> Buffer: ... + @property + def metadata_version(self): ... + @property + def body(self) -> None | Buffer: ... + def equals(self, other: Message) -> bool: ... + def serialize_to( + self, + sink: NativeFile, + alignment: int = 8, + memory_pool: MemoryPool | None = None, + ) -> None: ... + def serialize( + self, alignment: int = 8, memory_pool: MemoryPool | None = None + ) -> Buffer: ... + +class MessageReader(_Weakrefable): + def __init__(self): ... + @staticmethod + def open_stream(source) -> MessageReader: ... + def __iter__(self) -> Self: ... + def __next__(self): ... + def read_next_message(self) -> Message: ... + +class _CRecordBatchWriter(_Weakrefable): + def write(self, table_or_batch) -> None: ... + def write_batch( + self, + batch: RecordBatch, + custom_metadata: Mapping[Any, Any] | KeyValueMetadata | None = None, + ) -> None: ... + def write_table(self, table: Table, max_chunksize: int | None = None) -> None: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type, exc_val, exc_tb) -> None: ... + @property + def stats(self): ... + +class _RecordBatchStreamWriter(_CRecordBatchWriter): + def __init__(self): ... + @property + def _use_legacy_format(self): ... + @property + def _metadata_version(self): ... + def _open(self, sink, schema: Schema, options: IpcWriteOptions = ...) -> None: ... + +class _ReadPandasMixin: + def read_pandas(self, **options): ... + +class RecordBatchReader(_Weakrefable): + read_pandas = _ReadPandasMixin.read_pandas + + def __init__(self): ... + def __iter__(self) -> Self: ... + def __next__(self): ... + @property + def schema(self) -> Schema: ... + def read_next_batch(self) -> RecordBatch: ... + def read_next_batch_with_custom_metadata(self): ... + def iter_batches_with_custom_metadata(self) -> Iterator[Any]: ... + def read_all(self) -> Table: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type, exc_val, exc_tb) -> None: ... + def cast(self, target_schema: Schema) -> RecordBatchReader: ... + def _export_to_c(self, out_ptr) -> None: ... + @staticmethod + def _import_from_c(in_ptr) -> RecordBatchReader: ... + def __arrow_c_stream__(self, requested_schema=None): ... + @staticmethod + def _import_from_c_capsule(stream) -> RecordBatchReader: ... + @staticmethod + def from_stream(data, schema: Schema | None = None) -> RecordBatchReader: ... + @staticmethod + def from_batches( + schema: Schema, batches: Iterable[RecordBatch] + ) -> RecordBatchReader: ... + +class _RecordBatchStreamReader(RecordBatchReader): + def __init__(self): ... + def _open( + self, + source, + options: IpcReadOptions = ..., + memory_pool: MemoryPool | None = None, + ) -> None: ... + @property + def stats(self): ... + +class _RecordBatchFileWriter(_RecordBatchStreamWriter): + def _open( + self, sink, schema: Schema, options: IpcWriteOptions = ..., metadata=None + ) -> None: ... + +class RecordBatchWithMetadata(_RecordBatchWithMetadata): + __slots__ = () + +class _RecordBatchFileReader(_Weakrefable): + read_pandas = _ReadPandasMixin.read_pandas + schema: Schema + + def __init__(self): ... + def _open( + self, + source, + footer_offset=None, + options: IpcReadOptions = ..., + memory_pool: MemoryPool | None = None, + ) -> None: ... + @property + def num_record_batches(self): ... + def count_rows(self) -> int: ... + def get_batch(self, i: int) -> RecordBatch: ... + get_record_batch = get_batch + + def get_batch_with_custom_metadata(self, i: int): ... + def read_all(self) -> Table: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type, exc_value, traceback) -> None: ... + @property + def stats(self): ... + @property + def metadata(self): ... + +class Decimal32Array(FixedSizeBinaryArray): ... diff --git a/python/pyarrow-stubs/pyarrow/orc.pyi b/python/pyarrow-stubs/pyarrow/orc.pyi new file mode 100644 index 000000000000..77df7e448d52 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/orc.pyi @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from io import BytesIO +from pathlib import Path +from pyarrow.lib import BufferReader, NativeFile, Schema +import pyarrow +from typing import Any, IO +import pyarrow.fs +from pyarrow.lib import __doc__ as __doc__ +from _typeshed import Incomplete +from pyarrow.fs import _resolve_filesystem_and_path as _resolve_filesystem_and_path +from pyarrow.lib import Table as Table + +_orc_writer_args_docs: str + +def read_table( + source: str | pyarrow.NativeFile | IO[Any], + columns: list | None = None, + filesystem: pyarrow.fs.FileSystem | None = None, +) -> Table: ... +def write_table( + table: Table, + where, + *, + file_version: str = "0.12", + batch_size: int = 1024, + stripe_size: int = ..., + compression: str = "uncompressed", + compression_block_size: int = 65536, + compression_strategy: str = "speed", + row_index_stride: int = 10000, + padding_tolerance: float = 0.0, + dictionary_key_size_threshold: float = 0.0, + bloom_filter_columns=None, + bloom_filter_fpp: float = 0.05, +) -> None: ... + +class ORCFile: + reader: Incomplete + + def __init__( + self, source: Path | BufferReader | str | BytesIO | NativeFile + ) -> None: ... + @property + def metadata(self): ... + @property + def schema(self) -> Schema: ... + @property + def nrows(self) -> int: ... + @property + def nstripes(self) -> int: ... + @property + def file_version(self) -> str: ... + @property + def software_version(self): ... + @property + def compression(self) -> str: ... + @property + def compression_size(self) -> int: ... + @property + def writer(self): ... + @property + def writer_version(self): ... + @property + def row_index_stride(self) -> int: ... + @property + def nstripe_statistics(self): ... + @property + def content_length(self): ... + @property + def stripe_statistics_length(self): ... + @property + def file_footer_length(self): ... + @property + def file_postscript_length(self): ... + @property + def file_length(self): ... + def _select_names( + self, columns: list[int] | list[str] | None = None + ) -> list[str]: ... + def read_stripe( + self, n: int, columns: list | None = None + ) -> pyarrow.RecordBatch: ... + def read(self, columns: list | None = None) -> Table: ... + +class ORCWriter: + __doc__: Incomplete + is_open: bool + writer: Incomplete + + def __init__( + self, + where, + *, + file_version: str = "0.12", + batch_size: int = 1024, + stripe_size=..., + compression: str = "uncompressed", + compression_block_size: int = 65536, + compression_strategy: str = "speed", + row_index_stride: int = 10000, + padding_tolerance: float = 0.0, + dictionary_key_size_threshold: float = 0.0, + bloom_filter_columns=None, + bloom_filter_fpp: float = 0.05, + ) -> None: ... + def __del__(self) -> None: ... + def __enter__(self): ... + def __exit__(self, *args, **kwargs) -> None: ... + def write(self, table: Table) -> None: ... + def close(self) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/pandas_compat.pyi b/python/pyarrow-stubs/pyarrow/pandas_compat.pyi new file mode 100644 index 000000000000..1b4b91c148a6 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/pandas_compat.pyi @@ -0,0 +1,719 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyarrow.lib import ( + BinaryArray, + BooleanArray, + ChunkedArray, + DataType, + Date32Array, + Decimal128Array, + Decimal128Type, + DictionaryArray, + DictionaryType, + DoubleArray, + DurationArray, + DurationType, + ExtensionArray, + FixedSizeBinaryType, + FixedSizeListType, + FloatArray, + HalfFloatArray, + Int16Array, + Int32Array, + Int64Array, + Int8Array, + LargeListType, + LargeListViewType, + LargeStringArray, + ListArray, + ListType, + ListViewType, + MapArray, + MapType, + NullArray, + RunEndEncodedType, + Schema, + StringArray, + StringViewArray, + StructArray, + StructType, + Table, + Time32Type, + Time64Type, + TimestampArray, + TimestampType, + UInt16Array, + UInt32Array, + UInt64Array, + UInt8Array, + UuidArray, +) +import pyarrow +from typing import Any +from _typeshed import Incomplete +from pyarrow.lib import ( + _pandas_api as _pandas_api, + frombytes as frombytes, + is_threading_enabled as is_threading_enabled, +) + +_logical_type_map: Incomplete +_numpy_logical_type_map: Incomplete +_pandas_logical_type_map: Incomplete + +def get_logical_type_map() -> dict[int, str]: ... +def get_logical_type(arrow_type) -> str: ... +def get_numpy_logical_type_map(): ... +def get_logical_type_from_numpy(pandas_collection) -> str: ... +def get_extension_dtype_info( + column, +) -> ( + tuple[str, dict[str, str]] | tuple[str, dict[str, int | bool]] | tuple[str, None] +): ... +def get_column_metadata( + column, name: str, arrow_type: pyarrow.DataType, field_name: str +) -> dict: ... +def construct_metadata( + columns_to_convert, + df, + column_names: list[Any | str], + index_levels, + index_descriptors: list[dict], + preserve_index: bool, + types: list[pyarrow.DataType], + column_field_names: list[Any | str] | None = None, +) -> dict: ... +def _get_simple_index_descriptor( + level, name: str | None +) -> ( + dict[str, str | dict[str, str] | None] + | dict[str, str | dict[str, str]] + | dict[str, str | None] +): ... +def _column_name_to_strings(name: str | tuple) -> str | tuple: ... +def _index_level_name(index, i: int, column_names: list[Any | str]) -> str: ... +def _get_columns_to_convert( + df, schema: Schema | None, preserve_index: bool | None, columns: list[str] | None +): ... +def _get_columns_to_convert_given_schema( + df, schema: Schema, preserve_index: bool | None +): ... +def _get_index_level(df, name: str): ... +def _level_name(name) -> int | str | None: ... +def _get_range_index_descriptor( + level, +) -> dict[str, str | int] | dict[str, str | int | None]: ... +def _get_index_level_values(index): ... +def _resolve_columns_of_interest( + df, schema: Schema | None, columns: list[str] | None +): ... +def dataframe_to_types(df, preserve_index: bool | None, columns=None): ... +def dataframe_to_arrays( + df, + schema: Schema | None, + preserve_index: bool | None, + nthreads: int = 1, + columns: list[str] | None = None, + safe: bool = True, +) -> ( + tuple[list[MapArray | StringArray], Schema, None] + | tuple[ + list[Int64Array | FloatArray | DoubleArray | BooleanArray | LargeStringArray], + Schema, + None, + ] + | tuple[list[LargeStringArray | Int64Array | TimestampArray], Schema, None] + | tuple[list[DoubleArray], Schema, None] + | tuple[list[DoubleArray | LargeStringArray], Schema, None] + | tuple[ + list[ + UInt8Array + | UInt16Array + | UInt32Array + | UInt64Array + | Int16Array + | Int32Array + | Int64Array + | HalfFloatArray + | FloatArray + | DoubleArray + | BooleanArray + | TimestampArray + | DurationArray + | LargeStringArray + | NullArray + | ListArray + ], + Schema, + None, + ] + | tuple[ + list[ + TimestampArray + | DictionaryArray + | LargeStringArray + | BooleanArray + | DoubleArray + | Int64Array + | ExtensionArray + ], + Schema, + None, + ] + | tuple[list[Date32Array], Schema, None] + | tuple[list[StringArray | Int64Array | DoubleArray], Schema, None] + | tuple[list[UInt32Array], Schema, None] + | tuple[list[Int16Array], Schema, None] + | tuple[list[Int64Array | LargeStringArray], Schema, None] + | tuple[list[Date32Array | Int64Array | DoubleArray | StringArray], Schema, None] + | tuple[list[Any], Schema, None] + | tuple[list[ExtensionArray], Schema, None] + | tuple[list[ListArray], Schema, None] + | tuple[list[StructArray], Schema, None] + | tuple[ + list[ + UInt8Array + | UInt32Array + | Int16Array + | Int32Array + | HalfFloatArray + | DoubleArray + | TimestampArray + | LargeStringArray + | ListArray + ], + Schema, + None, + ] + | tuple[list[Int64Array | DoubleArray], Schema, None] + | tuple[list[TimestampArray], Schema, None] + | tuple[ + list[Int64Array | UInt32Array | DoubleArray | LargeStringArray | BooleanArray], + Schema, + None, + ] + | tuple[ + list[ + UInt8Array + | UInt16Array + | UInt32Array + | UInt64Array + | Int8Array + | Int16Array + | Int32Array + | Int64Array + | FloatArray + | DoubleArray + | BooleanArray + ], + Schema, + None, + ] + | tuple[list[StructArray | LargeStringArray], Schema, None] + | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array], Schema, None] + | tuple[list[DurationArray], Schema, None] + | tuple[ + list[ + UInt8Array + | UInt16Array + | UInt32Array + | UInt64Array + | Int16Array + | Int32Array + | Int64Array + | FloatArray + | DoubleArray + | BooleanArray + | TimestampArray + | DurationArray + | LargeStringArray + ], + Schema, + None, + ] + | tuple[list[LargeStringArray | ListArray], Schema, None] + | tuple[list[UInt64Array], Schema, None] + | tuple[list[DoubleArray | ListArray | Int32Array], Schema, None] + | tuple[list[LargeStringArray | Decimal128Array], Schema, None] + | tuple[list[StringViewArray], Schema, None] + | tuple[list[Int64Array | StructArray], Schema, None] + | tuple[list[DoubleArray | LargeStringArray | Int64Array], Schema, None] + | tuple[list[UInt8Array], Schema, None] + | tuple[ + list[ + UInt8Array + | UInt16Array + | UInt32Array + | UInt64Array + | Int8Array + | Int16Array + | Int32Array + | Int64Array + | FloatArray + | DoubleArray + | BooleanArray + | LargeStringArray + | NullArray + ], + Schema, + None, + ] + | tuple[ + list[UInt32Array | DoubleArray | BooleanArray | LargeStringArray], Schema, None + ] + | tuple[list[FloatArray | DoubleArray], Schema, None] + | tuple[list[Int16Array | FloatArray | StringArray], Schema, None] + | tuple[ + list[ + UInt8Array + | UInt16Array + | Int64Array + | UInt64Array + | Int8Array + | Int16Array + | Int32Array + | FloatArray + | DoubleArray + | BooleanArray + | LargeStringArray + | NullArray + ], + Schema, + None, + ] + | tuple[list[Int64Array | BooleanArray], Schema, None] + | tuple[ + list[ + LargeStringArray + | Int64Array + | UInt8Array + | DoubleArray + | BooleanArray + | DictionaryArray + | TimestampArray + ], + Schema, + None, + ] + | tuple[list[Int64Array | ListArray | DoubleArray], Schema, None] + | tuple[ + list[ + UInt8Array + | UInt16Array + | UInt32Array + | UInt64Array + | Int16Array + | Int32Array + | Int64Array + | FloatArray + | DoubleArray + | BooleanArray + ], + Schema, + None, + ] + | tuple[list[LargeStringArray], Schema, None] + | tuple[list[NullArray], Schema, None] + | tuple[list[DictionaryArray], Schema, None] + | tuple[list[Int32Array | StringArray], Schema, None] + | tuple[ + list[DictionaryArray | Int64Array | LargeStringArray | BinaryArray], + Schema, + None, + ] + | tuple[list[FloatArray | TimestampArray], Schema, None] + | tuple[list[StringArray | Int64Array | Int32Array | TimestampArray], Schema, None] + | tuple[list[BooleanArray], Schema, None] + | tuple[ + list[LargeStringArray | Int64Array | DoubleArray | TimestampArray], Schema, None + ] + | tuple[list[Int64Array | ListArray], Schema, None] + | tuple[list[DictionaryArray | Int64Array], Schema, None] + | tuple[ + list[ + UInt8Array + | UInt16Array + | UInt32Array + | UInt64Array + | Int16Array + | Int32Array + | Int64Array + | FloatArray + | DoubleArray + | BooleanArray + | LargeStringArray + ], + Schema, + None, + ] + | tuple[list[StringArray], Schema, None] + | tuple[list[Any], Schema, int] + | tuple[list[Int8Array], Schema, None] + | tuple[list[Decimal128Array], Schema, None] + | tuple[list[NullArray | DoubleArray], Schema, None] + | tuple[list[UInt16Array], Schema, None] + | tuple[list[UuidArray], Schema, None] + | tuple[list[BinaryArray], Schema, None] + | tuple[list[LargeStringArray | DictionaryArray], Schema, None] + | tuple[ + list[ + UInt8Array + | UInt16Array + | UInt32Array + | UInt64Array + | Int16Array + | Int32Array + | Int64Array + | HalfFloatArray + | FloatArray + | DoubleArray + | BooleanArray + | TimestampArray + | DurationArray + | LargeStringArray + | NullArray + | ListArray + | DictionaryArray + ], + Schema, + None, + ] + | tuple[list[Int32Array], Schema, None] + | tuple[list[DoubleArray | DictionaryArray], Schema, None] + | tuple[list[StringArray | FloatArray], Schema, None] + | tuple[ + list[ + Int8Array + | Int16Array + | Int32Array + | Int64Array + | UInt8Array + | UInt16Array + | UInt32Array + | UInt64Array + ], + Schema, + None, + ] + | tuple[list[FloatArray], Schema, None] + | tuple[list[Int64Array], Schema, None] + | tuple[list[ChunkedArray | StringArray], Schema, None] + | tuple[list[TimestampArray | Int64Array], Schema, None] +): ... +def get_datetimetz_type( + values, + dtype, + type_: Decimal128Type + | LargeListViewType + | StructType + | LargeListType + | MapType + | RunEndEncodedType + | DictionaryType + | ListType + | TimestampType + | FixedSizeListType + | DataType + | FixedSizeBinaryType + | DurationType + | Time64Type + | Time32Type + | ListViewType + | None, +): ... +def _reconstruct_block( + item: dict, + columns: list[str] | None = None, + extension_columns: dict | None = None, + return_block: bool = True, +): ... +def make_datetimetz(unit: str, tz: str): ... +def table_to_dataframe( + options: dict[str, bool | None], + table: Table, + categories=None, + ignore_metadata: bool = False, + types_mapper=None, +): ... + +_pandas_supported_numpy_types: Incomplete + +def _get_extension_dtypes( + table: Table, + columns_metadata: list[ + dict[str, str | dict[str, int]] + | dict[str, str | None] + | dict[str, str | dict[str, int | bool]] + | Any + | dict[str, str | dict[str, str]] + | dict[str, str | dict[str, str] | None] + ], + types_mapper, + options: dict[str, bool | None], + categories, +): ... +def _check_data_column_metadata_consistency( + all_columns: list[ + dict[str, str | dict[str, int]] + | dict[str, str | None] + | dict[str, str | dict[str, int | bool]] + | Any + | dict[str, str | dict[str, str]] + | dict[str, str | dict[str, str] | None] + ], +) -> None: ... +def _deserialize_column_index( + block_table: Table, + all_columns: list[ + dict[str, str | dict[str, int]] + | dict[str, str | None] + | dict[str, str | dict[str, int | bool]] + | Any + | dict[str, str | dict[str, str]] + | dict[str, str | dict[str, str] | None] + ], + column_indexes: list[ + dict[str, str | dict[str, str]] + | dict[str, str | dict[str, str] | None] + | dict[str, str | None] + | Any + ], +): ... +def _reconstruct_index( + table: Table, + index_descriptors: list[ + Any | dict[str, str | int] | dict[str, str | int | None] | str + ], + all_columns: list[ + dict[str, str | dict[str, int]] + | dict[str, str | None] + | dict[str, str | dict[str, int | bool]] + | Any + | dict[str, str | dict[str, str]] + | dict[str, str | dict[str, str] | None] + ], + types_mapper=None, +): ... +def _extract_index_level( + table: Table, + result_table: Table, + field_name: str, + field_name_to_metadata: dict[ + str, dict[str, str | None] | dict[str, str | dict[str, int | bool]] + ] + | dict[str, dict[str, str | None] | dict[str, str | dict[str, str]]] + | dict[ + str, dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None] + ] + | dict[str, dict[str, str | None]], + types_mapper=None, +): ... +def _backwards_compatible_index_name(raw_name: str, logical_name: str) -> str: ... +def _is_generated_index_name(name: str) -> bool: ... +def get_pandas_logical_type_map(): ... +def _pandas_type_to_numpy_type(pandas_type: str): ... +def _reconstruct_columns_from_metadata( + columns, + column_indexes: list[ + dict[str, str | dict[str, str]] + | dict[str, str | None] + | dict[str, str | dict[str, str] | None] + ], +): ... +def _add_any_metadata( + table: Table, + pandas_metadata: dict[ + str, + list[str] | str | list[dict[str, str | None] | dict[str, str | dict[str, str]]], + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | None] | dict[str, str | dict[str, str]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | dict[str, int]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[Any] + | list[dict[str, str | None] | dict[str, str | dict[str, int | bool]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[Any] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | None]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[str, list[str] | list[dict[str, str | None]] | str] + | dict[ + str, + list[str] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | None]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[str, list[dict[str, str | None]] | list[dict[str, str | int]] | str] + | dict[ + str, + list[str] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | None]] + | str, + ] + | dict[ + str, + list[Any] + | list[dict[str, str | dict[str, int | bool]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | dict[str, str]]] + | list[dict[str, str | None]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[str] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | None] | dict[str, str | dict[str, int | bool]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[Any] + | list[dict[str, str | None] | dict[str, str | dict[str, int]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | None]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[str] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[str] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str, + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | dict[str, str]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | None]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | None] | dict[str, str | dict[str, int | bool]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[Any] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | None] | dict[str, str | dict[str, str]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | dict[str, int | bool]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | None]] + | list[Any] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[Any] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str, + ] + | dict[ + str, + list[Any] + | list[dict[str, str | dict[str, str]]] + | dict[Any, Any] + | dict[str, str] + | str, + ], +) -> Table: ... +def make_tz_aware(series, tz: str): ... diff --git a/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi b/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi new file mode 100644 index 000000000000..14d5874f5c46 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .core import * diff --git a/python/pyarrow-stubs/pyarrow/parquet/core.pyi b/python/pyarrow-stubs/pyarrow/parquet/core.pyi new file mode 100644 index 000000000000..d86961927fc8 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/parquet/core.pyi @@ -0,0 +1,368 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pathlib import Path +from pyarrow._fs import LocalFileSystem as LocalFileSystem, SubTreeFileSystem +from pyarrow.lib import RecordBatch, Schema, Table +import pyarrow +from typing import Any, IO, Literal +from collections.abc import Callable, Sequence +import pyarrow.compute +import pyarrow.dataset +import pyarrow.fs +from pyarrow.lib import __doc__ as __doc__ +from _typeshed import Incomplete +from pyarrow._parquet import ( + ColumnChunkMetaData as ColumnChunkMetaData, + ColumnSchema as ColumnSchema, + FileDecryptionProperties as FileDecryptionProperties, + FileEncryptionProperties as FileEncryptionProperties, + FileMetaData as FileMetaData, + ParquetLogicalType as ParquetLogicalType, + ParquetReader as ParquetReader, + ParquetSchema as ParquetSchema, + RowGroupMetaData as RowGroupMetaData, + SortingColumn as SortingColumn, + Statistics as Statistics, +) +from collections import defaultdict + +__all__ = [ + "ParquetReader", + "Statistics", + "FileMetaData", + "RowGroupMetaData", + "ColumnChunkMetaData", + "ParquetSchema", + "ColumnSchema", + "ParquetLogicalType", + "FileEncryptionProperties", + "FileDecryptionProperties", + "SortingColumn", + "filters_to_expression", + "_filters_to_expression", + "ParquetFile", + "ParquetWriter", + "ParquetDataset", + "read_table", + "read_pandas", + "write_table", + "write_to_dataset", + "write_metadata", + "read_metadata", + "read_schema", +] + +def filters_to_expression( + filters: list[tuple] | list[list[tuple]], +) -> pyarrow.compute.Expression: ... + +_filters_to_expression: Incomplete + +def read_table( + source: str | list[str] | pyarrow.NativeFile | IO[Any], + *, + columns: list | None = None, + use_threads: bool = True, + schema: pyarrow.Schema | None = None, + use_pandas_metadata: bool = False, + read_dictionary: list | None = None, + binary_type: pyarrow.DataType | None = None, + list_type=None, + memory_map: bool = False, + buffer_size: int = 0, + partitioning: str = "hive", + filesystem: pyarrow.fs.FileSystem | None = None, + filters: pyarrow.compute.Expression | list[tuple] | list[list[tuple]] | None = None, + ignore_prefixes: list | None = None, + pre_buffer: bool = True, + coerce_int96_timestamp_unit: str | None = None, + decryption_properties: FileDecryptionProperties | None = None, + thrift_string_size_limit: int | None = None, + thrift_container_size_limit: int | None = None, + schema_depth_limit: int | None = None, + page_checksum_verification: bool = False, + arrow_extensions_enabled: bool = True, +) -> pyarrow.Table: ... +def read_pandas( + source: str | list[str] | pyarrow.NativeFile | IO[Any], + columns: list | None = None, + **kwargs, +) -> pyarrow.Table: ... +def write_table( + table: pyarrow.Table, + where: str | pyarrow.NativeFile, + row_group_size: int | None = None, + version: str = "2.6", + use_dictionary: bool = True, + compression: str = "snappy", + write_statistics: bool = True, + use_deprecated_int96_timestamps: bool | None = None, + coerce_timestamps: str | None = None, + allow_truncated_timestamps: bool = False, + data_page_size: int | None = None, + flavor: Literal["spark"] | None = None, + filesystem: pyarrow.fs.FileSystem | None = None, + compression_level: int | dict | None = None, + use_byte_stream_split: bool = False, + column_encoding: str | dict | None = None, + data_page_version: str = "1.0", + use_compliant_nested_type: bool = True, + encryption_properties: FileEncryptionProperties | None = None, + write_batch_size: int | None = None, + dictionary_pagesize_limit: int | None = None, + store_schema: bool = True, + write_page_index: bool = False, + write_page_checksum: bool = False, + sorting_columns: Sequence[SortingColumn] | None = None, + store_decimal_as_integer: bool = False, + write_time_adjusted_to_utc: bool = False, + max_rows_per_page: int | None = None, + bloom_filter_options: dict | None = None, + use_content_defined_chunking: bool = False, + **kwargs, +) -> None: ... +def write_to_dataset( + table: pyarrow.Table, + root_path: Path | str, + partition_cols: list | None = None, + filesystem: pyarrow.fs.FileSystem | None = None, + schema: pyarrow.Schema | None = None, + partitioning: pyarrow.dataset.Partitioning | list[str] | None = None, + basename_template: str | None = None, + use_threads: bool | None = None, + file_visitor: Callable[..., Any] | None = None, + existing_data_behavior: Literal["overwrite_or_ignore", "error", "delete_matching"] + | None = None, + **kwargs, +) -> None: ... +def write_metadata( + schema: pyarrow.Schema, + where: str | pyarrow.NativeFile, + metadata_collector: list | None = None, + filesystem: pyarrow.fs.FileSystem | None = None, + **kwargs, +) -> None: ... +def read_metadata( + where: str | IO[Any], + memory_map: bool = False, + decryption_properties: FileDecryptionProperties | None = None, + filesystem: pyarrow.fs.FileSystem | None = None, + arrow_extensions_enabled: bool = True, +) -> FileMetaData: ... +def read_schema( + where: str | IO[Any], + memory_map: bool = False, + decryption_properties: FileDecryptionProperties | None = None, + filesystem: pyarrow.fs.FileSystem | None = None, + arrow_extensions_enabled: bool = True, +) -> pyarrow.Schema: ... + +EXCLUDED_PARQUET_PATHS: set[str] + +class ParquetFile: + _close_source: Incomplete + reader: Incomplete + common_metadata: Incomplete + _nested_paths_by_prefix: Incomplete + + def __init__( + self, + source, + *, + metadata=None, + common_metadata=None, + read_dictionary=None, + binary_type=None, + list_type=None, + memory_map: bool = False, + buffer_size: int = 0, + pre_buffer: bool = True, + coerce_int96_timestamp_unit=None, + decryption_properties=None, + thrift_string_size_limit=None, + thrift_container_size_limit=None, + schema_depth_limit=None, + filesystem=None, + page_checksum_verification: bool = False, + arrow_extensions_enabled: bool = True, + ) -> None: ... + def __enter__(self): ... + def __exit__(self, *args, **kwargs) -> None: ... + def _build_nested_paths( + self, + ) -> defaultdict[str, list[int]] | defaultdict[Any, Any]: ... + @property + def metadata(self) -> FileMetaData: ... + @property + def schema(self) -> ParquetSchema: ... + @property + def schema_arrow(self) -> Schema: ... + @property + def num_row_groups(self) -> int: ... + def close(self, force: bool = False): ... + @property + def closed(self) -> bool: ... + def read_row_group( + self, + i: int, + columns: list | None = None, + use_threads: bool = True, + use_pandas_metadata: bool = False, + ) -> Table: ... + def read_row_groups( + self, + row_groups: list, + columns: list | None = None, + use_threads: bool = True, + use_pandas_metadata: bool = False, + ) -> Table: ... + def iter_batches( + self, + batch_size: int = 65536, + row_groups: list | None = None, + columns: list | None = None, + use_threads: bool = True, + use_pandas_metadata: bool = False, + ): ... + def read( + self, + columns: list | None = None, + use_threads: bool = True, + use_pandas_metadata: bool = False, + ) -> Table: ... + def scan_contents( + self, columns: list[int] | None = None, batch_size: int = 65536 + ) -> int: ... + def _get_column_indices( + self, column_names, use_pandas_metadata: bool = False + ) -> list[int]: ... + +class ParquetWriter: + __doc__: Incomplete + flavor: Incomplete + schema_changed: bool + schema: Incomplete + where: Incomplete + file_handle: Incomplete + _metadata_collector: Incomplete + writer: Incomplete + is_open: bool + + def __init__( + self, + where, + schema: Schema | None, + filesystem: LocalFileSystem | None = None, + flavor: str | None = None, + version: str = "2.6", + use_dictionary: bool = True, + compression: str = "snappy", + write_statistics: bool = True, + use_deprecated_int96_timestamps: bool | None = None, + compression_level: int | dict[str, int] | None = None, + use_byte_stream_split: bool = False, + column_encoding: dict[str, str] | bool | str | None = None, + writer_engine_version=None, + data_page_version: str = "1.0", + use_compliant_nested_type: bool = True, + encryption_properties: FileEncryptionProperties | None = None, + write_batch_size: int | None = None, + dictionary_pagesize_limit: int | str | None = None, + store_schema: bool = True, + write_page_index: bool = False, + write_page_checksum: bool = False, + sorting_columns: tuple[SortingColumn, SortingColumn] | None = None, + store_decimal_as_integer: bool = False, + write_time_adjusted_to_utc: bool = False, + max_rows_per_page: int | None = None, + bloom_filter_options: bool + | dict[str, dict[str, str | int]] + | dict[str, dict[Any, Any]] + | dict[str, dict[str, float]] + | dict[str, dict[str, str | float]] + | dict[str, dict[str, int]] + | dict[str, bool] + | dict[str, dict[str, int | float]] + | None = None, + use_content_defined_chunking: bool = False, + **options, + ) -> None: ... + def __del__(self) -> None: ... + def __enter__(self): ... + def __exit__(self, *args, **kwargs): ... + def write( + self, table_or_batch: Table | RecordBatch, row_group_size: int | None = None + ) -> None: ... + def write_batch( + self, batch: pyarrow.RecordBatch, row_group_size: int | None = None + ) -> None: ... + def write_table( + self, table: pyarrow.Table, row_group_size: int | None = None + ) -> None: ... + def close(self) -> None: ... + def add_key_value_metadata(self, key_value_metadata: dict) -> None: ... + +class ParquetDataset: + __doc__: Incomplete + _filter_expression: Incomplete + _base_dir: Incomplete + _dataset: Incomplete + + def __init__( + self, + path_or_paths, + filesystem: SubTreeFileSystem | LocalFileSystem | str | None = None, + schema: Schema | None = None, + *, + filters=None, + read_dictionary=None, + binary_type=None, + list_type=None, + memory_map: bool = False, + buffer_size=None, + partitioning: str = "hive", + ignore_prefixes=None, + pre_buffer: bool = True, + coerce_int96_timestamp_unit=None, + decryption_properties=None, + thrift_string_size_limit=None, + thrift_container_size_limit=None, + schema_depth_limit=None, + page_checksum_verification: bool = False, + arrow_extensions_enabled: bool = True, + ) -> None: ... + def equals(self, other) -> bool: ... + def __eq__(self, other) -> bool: ... + @property + def schema(self) -> Schema: ... + def read( + self, + columns: list[str] | None = None, + use_threads: bool = True, + use_pandas_metadata: bool = False, + ) -> pyarrow.Table: ... + def _get_common_pandas_metadata(self) -> dict[bytes, bytes]: ... + def read_pandas(self, **kwargs) -> Table: ... + @property + def fragments(self): ... + @property + def files(self) -> list[str]: ... + @property + def filesystem(self) -> LocalFileSystem: ... + @property + def partitioning(self): ... diff --git a/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi b/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi new file mode 100644 index 000000000000..fceed598f204 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyarrow._parquet_encryption import ( + CryptoFactory as CryptoFactory, + DecryptionConfiguration as DecryptionConfiguration, + EncryptionConfiguration as EncryptionConfiguration, + KmsClient as KmsClient, + KmsConnectionConfig as KmsConnectionConfig, + create_decryption_properties as create_decryption_properties, + create_encryption_properties as create_encryption_properties, +) diff --git a/python/pyarrow-stubs/pyarrow/substrait.pyi b/python/pyarrow-stubs/pyarrow/substrait.pyi new file mode 100644 index 000000000000..58bf52066092 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/substrait.pyi @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyarrow._substrait import ( + BoundExpressions as BoundExpressions, + SubstraitSchema as SubstraitSchema, + deserialize_expressions as deserialize_expressions, + deserialize_schema as deserialize_schema, + get_supported_functions as get_supported_functions, + run_query as run_query, + serialize_expressions as serialize_expressions, + serialize_schema as serialize_schema, +) diff --git a/python/pyarrow-stubs/pyarrow/types.pyi b/python/pyarrow-stubs/pyarrow/types.pyi new file mode 100644 index 000000000000..864bbb851717 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/types.pyi @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pyarrow +from _typeshed import Incomplete +from enum import IntEnum +from pyarrow.lib import ( + is_boolean_value as is_boolean_value, + is_float_value as is_float_value, + is_integer_value as is_integer_value, +) +from pyarrow.util import doc as doc + +_SIGNED_INTEGER_TYPES: Incomplete +_UNSIGNED_INTEGER_TYPES: Incomplete +_INTEGER_TYPES: set[Incomplete] +_FLOATING_TYPES: Incomplete +_DECIMAL_TYPES: Incomplete +_DATE_TYPES: Incomplete +_TIME_TYPES: Incomplete +_INTERVAL_TYPES: Incomplete +_TEMPORAL_TYPES: Incomplete +_UNION_TYPES: Incomplete + +def is_null(t: pyarrow.DataType) -> bool: ... +def is_boolean(t: pyarrow.DataType) -> bool: ... +def is_integer(t: pyarrow.DataType) -> bool: ... +def is_signed_integer(t: pyarrow.DataType) -> bool: ... +def is_unsigned_integer(t: pyarrow.DataType) -> bool: ... +def is_int8(t: pyarrow.DataType) -> bool: ... +def is_int16(t: pyarrow.DataType) -> bool: ... +def is_int32(t: pyarrow.DataType) -> bool: ... +def is_int64(t: pyarrow.DataType) -> bool: ... +def is_uint8(t: pyarrow.DataType) -> bool: ... +def is_uint16(t: pyarrow.DataType) -> bool: ... +def is_uint32(t: pyarrow.DataType) -> bool: ... +def is_uint64(t: pyarrow.DataType) -> bool: ... +def is_floating(t: pyarrow.DataType) -> bool: ... +def is_float16(t: pyarrow.DataType) -> bool: ... +def is_float32(t: pyarrow.DataType) -> bool: ... +def is_float64(t: pyarrow.DataType) -> bool: ... +def is_list(t: pyarrow.DataType) -> bool: ... +def is_large_list(t: pyarrow.DataType) -> bool: ... +def is_fixed_size_list(t: pyarrow.DataType) -> bool: ... +def is_list_view(t: pyarrow.DataType) -> bool: ... +def is_large_list_view(t: pyarrow.DataType) -> bool: ... +def is_struct(t: pyarrow.DataType) -> bool: ... +def is_union(t: pyarrow.DataType) -> bool: ... +def is_nested(t: pyarrow.DataType) -> bool: ... +def is_run_end_encoded(t: pyarrow.DataType) -> bool: ... +def is_temporal(t: pyarrow.DataType) -> bool: ... +def is_timestamp(t: pyarrow.DataType) -> bool: ... +def is_duration(t: pyarrow.DataType) -> bool: ... +def is_time(t: pyarrow.DataType) -> bool: ... +def is_time32(t: pyarrow.DataType) -> bool: ... +def is_time64(t: pyarrow.DataType) -> bool: ... +def is_binary(t: pyarrow.DataType) -> bool: ... +def is_large_binary(t: pyarrow.DataType) -> bool: ... +def is_unicode(t: pyarrow.DataType) -> bool: ... +def is_string(t: pyarrow.DataType) -> bool: ... +def is_large_unicode(t: pyarrow.DataType): ... +def is_large_string(t: pyarrow.DataType) -> bool: ... +def is_fixed_size_binary(t: pyarrow.DataType) -> bool: ... +def is_binary_view(t: pyarrow.DataType) -> bool: ... +def is_string_view(t: pyarrow.DataType) -> bool: ... +def is_date(t: pyarrow.DataType) -> bool: ... +def is_date32(t: pyarrow.DataType) -> bool: ... +def is_date64(t: pyarrow.DataType) -> bool: ... +def is_map(t: pyarrow.DataType) -> bool: ... +def is_decimal(t: pyarrow.DataType) -> bool: ... +def is_decimal32(t: pyarrow.DataType) -> bool: ... +def is_decimal64(t: pyarrow.DataType) -> bool: ... +def is_decimal128(t: pyarrow.DataType) -> bool: ... +def is_decimal256(t: pyarrow.DataType) -> bool: ... +def is_dictionary(t: pyarrow.DataType) -> bool: ... +def is_interval(t: pyarrow.DataType) -> bool: ... +def is_primitive(t: pyarrow.DataType) -> bool: ... + +class TypesEnum(IntEnum): + NA = ... + BOOL = ... + INT8 = ... + INT16 = ... + INT32 = ... + INT64 = ... + UINT8 = ... + UINT16 = ... + UINT32 = ... + UINT64 = ... + HALF_FLOAT = ... + FLOAT = ... + DOUBLE = ... + BINARY = ... + BINARY_VIEW = ... + LARGE_BINARY = ... + STRING = ... + STRING_VIEW = ... + LARGE_STRING = ... + FIXED_SIZE_BINARY = ... + DECIMAL32 = ... + DECIMAL64 = ... + DECIMAL128 = ... + DECIMAL256 = ... + LIST = ... + LARGE_LIST = ... + LIST_VIEW = ... + LARGE_LIST_VIEW = ... + MAP = ... + FIXED_SIZE_LIST = ... + STRUCT = ... + SPARSE_UNION = ... + DENSE_UNION = ... + RUN_END_ENCODED = ... + DATE32 = ... + DATE64 = ... + TIME32 = ... + TIME64 = ... + TIMESTAMP = ... + INTERVAL_MONTHS = ... + INTERVAL_DAY_TIME = ... + INTERVAL_MONTH_DAY_NANO = ... + DURATION = ... + DICTIONARY = ... diff --git a/python/pyarrow-stubs/pyarrow/util.pyi b/python/pyarrow-stubs/pyarrow/util.pyi new file mode 100644 index 000000000000..7b2fbd29ef13 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/util.pyi @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyarrow._dataset import FileSystemDataset, InMemoryDataset, Scanner +from typing import Any +from collections.abc import Callable, Iterator + +_DEPR_MSG: str + +def doc(*docstrings, **params) -> Callable: ... +def _deprecate_api( + old_name: str, new_name: str, api: Callable, next_version: str, type=... +) -> Callable: ... +def _deprecate_class(old_name, new_class, next_version, instancecheck: bool = True): ... +def _is_iterable( + obj: FileSystemDataset | Iterator[Any] | InMemoryDataset | Scanner, +): ... +def _is_path_like(path) -> bool: ... +def _stringify_path(path) -> str: ... +def product(seq): ... +def get_contiguous_span(shape: tuple, strides: tuple, itemsize: int) -> int: ... +def find_free_port() -> int: ... +def guid() -> str: ... +def _break_traceback_cycle_from_frame(frame) -> None: ... +def _download_urllib(url, out_path) -> None: ... +def _download_requests(url, out_path) -> None: ... +def download_tzdata_on_windows() -> None: ... diff --git a/python/pyproject.toml b/python/pyproject.toml index 68c1a807dd51..7c73679e3c55 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -112,23 +112,24 @@ git_describe_command = 'git describe --dirty --tags --long --match "apache-arrow fallback_version = '26.0.0a0' # TODO: Enable type checking once stubs are merged +# The test suite is checked against the stubs: it calls the API with real +# values, so it validates the annotations. Test bodies are unannotated, hence +# check_untyped_defs. The library sources themselves are not checked yet. [tool.mypy] -files = ["pyarrow-stubs"] +files = ["pyarrow-stubs", "pyarrow/tests"] mypy_path = "$MYPY_CONFIG_FILE_DIR/pyarrow-stubs" +check_untyped_defs = true exclude = [ - "^pyarrow/", "^benchmarks/", "^examples/", "^scripts/", ] -# TODO: Enable type checking once stubs are merged [tool.pyright] pythonPlatform = "All" pythonVersion = "3.11" -include = ["pyarrow-stubs"] +include = ["pyarrow-stubs", "pyarrow/tests"] exclude = [ - "pyarrow", "benchmarks", "examples", "scripts", @@ -136,16 +137,30 @@ exclude = [ ] stubPath = "pyarrow-stubs" typeCheckingMode = "basic" +# The tests import optional dependencies as `x = None` on failure and then +# use them unguarded; every report of that idiom would be noise here. +reportOptionalMemberAccess = false -# TODO: Enable type checking once stubs are merged [tool.ty.environment] extra-paths = ["pyarrow-stubs"] [tool.ty.src] -include = ["pyarrow-stubs"] +include = ["pyarrow-stubs", "pyarrow/tests"] exclude = [ - "pyarrow", "benchmarks", "examples", "scripts", ] + +# ruff formats and lints the generated stubs only; the pre-commit hooks and +# scripts/add_types_to_stubs.py run it. The stubs follow the typeshed +# conventions its flake8-pyi rules describe. +[tool.ruff] +include = ["pyarrow-stubs/**/*.pyi"] +line-length = 88 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "W", "PYI", "UP006", "UP007", "UP035", "UP037", "UP045"] +# Star imports mirror the sources; in a stub they re-export. +ignore = ["F403", "F405"] diff --git a/python/scripts/add_types_to_stubs.py b/python/scripts/add_types_to_stubs.py new file mode 100755 index 000000000000..7d48a17b5405 --- /dev/null +++ b/python/scripts/add_types_to_stubs.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Add type annotations to python/pyarrow-stubs. + +Existing annotations are kept. Missing ones are filled from, in order: + +1. what the Cython sources return, read from their return statements + (see cython_return_types); +2. the numpydoc "Parameters" and "Returns" sections of docstrings, which + cover the Cython modules as well; +3. default values of parameters; +4. MonkeyType traces of the test suite, which cover the pure Python modules. + +Leaked Cython names are resolved first, since an unknown base class makes a +whole class unknown to type checkers. Module attributes are declared with the +type of their runtime value, and namedtuples as NamedTuple classes. The stubs +are then fixed, formatted and linted with ruff as configured in pyproject.toml, +which the pre-commit hooks run too. The script prints what it annotated, the +docstring type phrases it could not map, what ruff still reports, and pyright's +type completeness score for the result. + +Run from the repository root with python/.venv-stubgen/bin/python after +scripts/generate_stubs.sh. Pass --no-traces to redo everything but the slow +test suite tracing, for instance after changing this script. +""" + +import argparse +import ast +import builtins +import collections +import enum +import functools +import importlib +import inspect +import logging +import os +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +import types +import typing + +import monkeytype +import pyarrow +from monkeytype.config import DefaultConfig +from monkeytype.stubs import build_module_stubs_from_traces +from pyarrow.vendored.docscrape import NumpyDocString + +import cython_return_types + +log = logging.getLogger("add_types_to_stubs") + +STUBS = pathlib.Path("python/pyarrow-stubs") +SOURCES = pathlib.Path("python/pyarrow") +SUBMODULES = ("compute", "dataset", "fs", "parquet", + "flight", "csv", "json", "ipc", "orc") +BUILTINS = {"int": "int", "integer": "int", "bool": "bool", "boolean": "bool", + "str": "str", "string": "str", "float": "float", "bytes": "bytes", + "dict": "dict", "list": "list", "tuple": "tuple", "set": "set", + "object": "object", "none": "None", "any": "Any", "self": "Self", + "callable": "Callable[..., Any]", "function": "Callable[..., Any]", + "iterable": "Iterable[Any]", "iterator": "Iterator[Any]", + "sequence": "Sequence[Any]", "mapping": "Mapping[Any, Any]", + "integers": "int", "strings": "str", "file-like": "IO[Any]", + "file-like object": "IO[Any]", "file-like python object": "IO[Any]", + "path-like": "os.PathLike[str]"} +CONTAINERS = {"list of": "list[{}]", "sequence of": "Sequence[{}]", + "iterable of": "Iterable[{}]", "iterator of": "Iterator[{}]", + "tuple of": "tuple[{}, ...]", "set of": "set[{}]"} +GENERICS = {"list": "list", "tuple": "tuple", "dict": "dict", "set": "set", + "sequence": "Sequence", "iterable": "Iterable", "iterator": "Iterator"} +# Modules whose names traced annotations may refer to; others would make the +# stubs depend on optional packages such as pandas. +TRACE_IMPORTS = ("pyarrow", "typing", "collections", + "datetime", "decimal", "pathlib", "os", "io") +TYPING_IMPORT = ("from typing import Any, Callable, IO, Iterable, Iterator, " + "Literal, Mapping, NamedTuple, Self, Sequence") +# Cython C types that leak into the stubs through declared parameter and +# attribute types, and what Python passes for them; `Type` and `TimeUnit` are +# C enums. +CYTHON_TYPES = {"c_bool": "bool", "c_string": "bytes", "std_vector": "list", + "Type": "int", "TimeUnit": "int"} +# Annotations for module attributes, by the type of their runtime value. +VALUE_TYPES = {int: "int", float: "float", str: "str", bytes: "bytes", + bool: "bool", list: "list[Incomplete]", set: "set[Incomplete]", + frozenset: "frozenset[Incomplete]", tuple: "tuple[Incomplete, ...]", + dict: "dict[Incomplete, Incomplete]", re.Pattern: "re.Pattern[str]"} + +stats = collections.Counter() +unresolved = collections.Counter() +undefined = collections.Counter() + + +class TraceConfig(DefaultConfig): + """MonkeyType configuration: trace the installed package, not its tests.""" + + def code_filter(self): + return lambda code: ("/pyarrow/" in code.co_filename + and "/tests/" not in code.co_filename) + + +CONFIG = TraceConfig() + + +# This module doubles as a pytest plugin so that every pytest-xdist worker +# traces the tests it runs, each into its own database under MT_DB_DIR. + +def pytest_configure(config): + worker = os.environ.get("PYTEST_XDIST_WORKER", "main") + database = pathlib.Path(os.environ["MT_DB_DIR"]) / f"{worker}.sqlite3" + os.environ["MT_DB_PATH"] = str(database) + config.monkeytype = monkeytype.trace(CONFIG) + config.monkeytype.__enter__() + + +def pytest_unconfigure(config): + if hasattr(config, "monkeytype"): + config.monkeytype.__exit__(None, None, None) + + +# --- docstring types ------------------------------------------------------- + +def resolve(text, local, imports): + """Map one numpydoc type phrase to an annotation, or None if unknown.""" + text = re.sub(r"\s*\(.*?\)", "", text).strip(" `.") + if text.lower() in BUILTINS: + return BUILTINS[text.lower()] + for prefix, template in CONTAINERS.items(): + if text.lower().startswith(prefix + " "): + inner = resolve(text[len(prefix) + 1:], local, imports) + return template.format(inner) if inner else None + literal = re.fullmatch( + r"\{(.*)\}", text) or re.fullmatch(r"('[^']*'|\"[^\"]*\")", text) + if literal: + items = [i.strip() for i in literal.group(1).split(",")] + if all(re.fullmatch(r"'[^']*'|\"[^\"]*\"", i) for i in items): + return f"Literal[{', '.join(items)}]" + generic = re.fullmatch(r"(\w+)\[(.*)\]", text) + if generic and generic.group(1).lower() in GENERICS: + args = [resolve(a, local, imports) for a in generic.group(2).split(",")] + outer = GENERICS[generic.group(1).lower()] + return f"{outer}[{', '.join(args)}]" if all(args) else None + name = re.sub(r"^(pyarrow|pa)\.(lib\.)?", "", text) + if name in local: + return name + if re.fullmatch(r"[\w.]+", name): + for prefix in ("", *(f"{sub}." for sub in SUBMODULES)): + obj = pyarrow + for part in f"{prefix}{name}".split("."): + obj = getattr(obj, part, None) + if isinstance(obj, type): + if "." in prefix + name: + imports.add("pyarrow." + (prefix + name).rsplit(".", 1)[0]) + return f"pyarrow.{prefix}{name}" + unresolved[text] += 1 + return None + + +def annotation(text, local, imports): + """Map a numpydoc type string such as "int or str, optional".""" + match = re.search(r",?\s+((optional|default)\b.*)$", text) + body, tail = (text[:match.start()], match.group(1)) if match else (text, "") + parts = [resolve(p, local, imports) + for p in re.split(r",?\s+or\s+|,\s*|\s*\|\s*", body) if p.strip()] + if not parts or None in parts: + return None + result = " | ".join(dict.fromkeys(parts)) + if re.match(r"optional|default\s+`?None`?$", tail) and "None" not in parts: + result += " | None" + return result + + +def from_default(node): + if isinstance(node, ast.Constant): + if type(node.value) in (bool, int, float, str, bytes): + return type(node.value).__name__ + return None + + +# --- stub rewriting -------------------------------------------------------- + +class Cleanup(ast.NodeTransformer): + """Drop cimports of .pxd declarations, and make imports from pyarrow + explicit re-exports where the runtime module has the name, as + `from pyarrow.lib import Table` in a stub is not one.""" + + def __init__(self, module): + self.module = module + + def visit_ImportFrom(self, node): + origin = node.module or "" + if origin.startswith("pyarrow.includes"): + return None + if origin.startswith("pyarrow"): + for alias in node.names: + if alias.asname is None and hasattr(self.module, alias.name): + alias.asname = alias.name + return node + + +def is_namedtuple(node): + return (isinstance(node, ast.Call) and len(node.args) == 2 + and ast.unparse(node.func).endswith("namedtuple") + and isinstance(node.args[1], (ast.Tuple, ast.List))) + + +def namedtuple_fields(node): + return [ast.AnnAssign(ast.Name(f.value), ast.Name("Incomplete"), None, simple=1) + for f in node.args[1].elts] + + +def dedupe_imports(nodes): + """Keep the last import of each name: the stub's own import rather than + one added before it.""" + seen = set() + for node in reversed(nodes): + if isinstance(node, ast.ImportFrom) and node.names[0].name != "*": + node.names = [a for a in node.names if (a.asname or a.name) not in seen] + seen |= {a.asname or a.name for a in node.names} + return [n for n in nodes if not isinstance(n, ast.ImportFrom) or n.names] + + +def order_assignments(body): + """Move assignments after the definitions their values refer to; only + annotations may refer forward in a stub.""" + position = {} + for i, node in enumerate(body): + for name in defined_names(ast.Module([node], [])): + position[name] = i + ordered, waiting = [], collections.defaultdict(list) + for i, node in enumerate(body): + if isinstance(node, (ast.Assign, ast.AnnAssign)) and node.value is not None: + later = [position[n.id] for n in ast.walk(node.value) + if isinstance(n, ast.Name) and position.get(n.id, -1) > i] + if later: + waiting[max(later)].append(node) + continue + ordered.append(node) + ordered.extend(waiting.pop(i, [])) + return ordered + + +class Names(ast.NodeTransformer): + """Spell the names an annotation uses so that the stub knows them.""" + + def __init__(self, tidy): + self.tidy = tidy + + def visit_Name(self, node): + if node.id in self.tidy.known: + return node + if node.id in CYTHON_TYPES: + stats["Cython types spelled"] += 1 + return ast.Name(CYTHON_TYPES[node.id]) + if qualified := self.tidy.lookup(node.id): + stats["annotation names qualified"] += 1 + return ast.parse(qualified, mode="eval").body + undefined[node.id] += 1 + return ast.Name("Incomplete") + + +class Tidy(ast.NodeTransformer): + """Say what the stub can about attributes, using only names it knows. + + Module assignments become declarations typed from the runtime value: + namedtuples become NamedTuple classes, type aliases stay, and the rest + get the value's type or Incomplete; class constants likewise. Leaked + Cython types get their Python spelling, classes other modules define get + qualified, and other names an annotation does not know become + Incomplete. Imports come first and once, assignments move after the + names they alias, and `__enter__` and its kind return Self.""" + + SELF_RETURNING = {"__enter__", "__aenter__", "__new__", + "__iter__", "__aiter__"} + + def __init__(self, module, known): + self.module, self.known = module, known | set(dir(builtins)) + self.owner, self.owners, self.imports = None, [module], set() + + def lookup(self, name): + """A class of that name in pyarrow or a submodule, qualified.""" + for prefix in ("", *(f"{sub}." for sub in SUBMODULES)): + obj = pyarrow + for part in f"{prefix}{name}".split("."): + obj = getattr(obj, part, None) + if isinstance(obj, type): + if prefix: + self.imports.add(f"pyarrow.{prefix[:-1]}") + return f"pyarrow.{prefix}{name}" + return None + + def visit_Module(self, node): + body = order_assignments([self.declare(n) for n in node.body]) + imports = [n for n in body if isinstance(n, (ast.Import, ast.ImportFrom))] + node.body = dedupe_imports(imports) + [n for n in body if n not in imports] + self.generic_visit(node) + return node + + def declare(self, node): + if not (isinstance(node, ast.Assign) and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name)): + return node + name = node.targets[0].id + if name.startswith("__") and name.endswith("__"): + return node + owner = self.owners[-1] + if owner is not self.module: # class constants; members and aliases stay + constant = isinstance(node.value, (ast.Constant, ast.BinOp, ast.UnaryOp)) + member = isinstance(owner, type) and issubclass(owner, enum.Enum) + if not constant or member: + return node + value = getattr(owner, name, None) + if is_namedtuple(node.value): + stats["namedtuples declared"] += 1 + return ast.ClassDef(name=name, bases=[ast.Name("NamedTuple")], keywords=[], + body=namedtuple_fields(node.value) or [ast.Pass()], + decorator_list=[], type_params=[]) + if (isinstance(value, (type, types.UnionType, typing.TypeVar)) + or value is typing.Any or typing.get_origin(value) is not None): + return node + typed = from_default(node.value) or VALUE_TYPES.get(type(value)) + cls = type(value) + if not typed and cls.__name__ in self.known \ + and getattr(self.module, cls.__name__, None) is cls: + typed = cls.__name__ + stats["attributes typed from runtime" if typed + else "attributes left Incomplete"] += 1 + return ast.AnnAssign(ast.Name(name), ast.parse( + typed or "Incomplete", mode="eval").body, None, simple=1) + + def visit_ClassDef(self, node): + for i, base in enumerate(node.bases): + if is_namedtuple(base): + node.bases[i] = ast.Name("NamedTuple") + node.body[:0] = namedtuple_fields(base) + stats["namedtuples declared"] += 1 + self.owners.append(getattr(self.owners[-1], node.name, None)) + node.body = order_assignments([self.declare(n) for n in node.body]) + self.owner, outer = node.name, self.owner + self.generic_visit(node) + self.owner = outer + self.owners.pop() + return node + + def visit_FunctionDef(self, node): + self.generic_visit(node) + node.returns = self.annotation(node.returns) + returns = node.returns + if (self.owner and node.name in self.SELF_RETURNING + and isinstance(returns, ast.Name) and returns.id == self.owner): + node.returns = ast.Name("Self") + stats["returns changed to Self"] += 1 + return node + + visit_AsyncFunctionDef = visit_FunctionDef + + def visit_arg(self, node): + node.annotation = self.annotation(node.annotation) + return node + + def visit_AnnAssign(self, node): + node.annotation = self.annotation(node.annotation) + return node + + def annotation(self, node): + return Names(self).visit(node) if node else None + + +class Annotator(ast.NodeTransformer): + def __init__(self, module, local, sourced, traced): + self.module, self.local = module, local + self.sourced, self.traced = sourced, traced + self.imports, self.owners = set(), [(module, None)] + + def visit_ClassDef(self, node): + self.owners.append((getattr(self.owners[-1][0], node.name, None), node.name)) + self.generic_visit(node) + self.owners.pop() + return node + + def visit_FunctionDef(self, node): + runtime, owner = self.owners[-1] + try: + doc = NumpyDocString(inspect.getdoc( + getattr(runtime, node.name, None)) or "") + except Exception: + doc = {"Parameters": [], "Returns": []} + params = { + n.strip("* "): p.type for p in doc["Parameters"] for n in p.name.split(",")} + traced_params, traced_return = self.traced.get((owner, node.name), ({}, None)) + positional = node.args.posonlyargs + node.args.args + defaults = dict(zip(reversed(positional), reversed(node.args.defaults))) + defaults.update((a, d) for a, d in zip( + node.args.kwonlyargs, node.args.kw_defaults) if d) + for arg in positional + node.args.kwonlyargs: + if arg.arg in ("self", "cls"): + continue + stats["parameters"] += 1 + if arg.annotation: + stats["parameters already typed"] += 1 + continue + default = defaults.get(arg) + typed = annotation(params[arg.arg], self.local, + self.imports) if params.get(arg.arg) else None + source = "parameters typed from docstring" + if not typed and from_default(default): + typed, source = from_default(default), "parameters typed from default" + # Traces only show what the tests passed; a default they never + # saw, such as another class, means the observed type is too narrow. + unknown_default = default is not None and not ( + isinstance(default, ast.Constant) and default.value is not ...) + if not typed and arg.arg in traced_params and not unknown_default: + typed, source = ast.unparse( + traced_params[arg.arg]), "parameters typed from traces" + if typed: + if isinstance(default, ast.Constant) and default.value is None: + if "None" not in typed: + typed += " | None" + default_type = from_default(default) + if default_type and default_type not in typed.split(" | "): + typed += " | " + default_type + arg.annotation = ast.parse(typed, mode="eval").body + stats[source] += 1 + if node.name != "__init__": + stats["returns"] += 1 + if node.returns: + stats["returns already typed"] += 1 + elif (typed := annotation(self.sourced.get((owner, node.name), ""), + self.local, self.imports)): + node.returns = ast.parse(typed, mode="eval").body + stats["returns typed from Cython sources"] += 1 + elif len(doc["Returns"]) == 1 and (typed := annotation( + doc["Returns"][0].type or doc["Returns"][0].name, + self.local, self.imports)): + node.returns = ast.parse(typed, mode="eval").body + stats["returns typed from docstring"] += 1 + elif traced_return is not None: + node.returns = traced_return + stats["returns typed from traces"] += 1 + return node + + visit_AsyncFunctionDef = visit_FunctionDef + + +def defined_names(tree): + names = {n.name for n in tree.body if isinstance( + n, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef))} + names |= {a.asname or a.name for n in tree.body if isinstance( + n, ast.ImportFrom) for a in n.names} + names |= {a.asname or a.name.split(".")[0] for n in tree.body if isinstance( + n, ast.Import) for a in n.names} + names |= {t.id for n in tree.body if isinstance( + n, ast.Assign) for t in n.targets if isinstance(t, ast.Name)} + names |= {n.target.id for n in tree.body if isinstance( + n, ast.AnnAssign) and isinstance(n.target, ast.Name)} + return names + + +def order_classes(tree): + """Define classes after their bases; pyright treats a base defined later + in a stub as unknown. Everything else keeps its order.""" + classes = {n.name: n for n in tree.body if isinstance(n, ast.ClassDef)} + ordered, done = [], set() + + def emit(node): + if node.name in done: + return + done.add(node.name) + for base in node.bases: + if isinstance(base, ast.Name) and base.id in classes: + emit(classes[base.id]) + ordered.append(node) + + for node in classes.values(): + emit(node) + tree.body = [n for n in tree.body if not isinstance(n, ast.ClassDef)] + ordered + return tree + + +def rewrite(path, module, sourced, traced): + source = path.read_text() + header = "".join(line for line in source.splitlines( + keepends=True)[:20] if line.startswith("#")) + tree = ast.parse(source) + defined = defined_names(tree) + # `from pyarrow.lib import *` skips private names such as the base class + # of most Cython classes. + missing = {n.id for n in ast.walk(tree) + if isinstance(n, ast.Name) and n.id.startswith("_") + and n.id not in defined and hasattr(pyarrow.lib, n.id)} + local = {n for n in defined if isinstance(getattr(module, n, None), type)} + annotator = Annotator(module, local, sourced, traced.get("functions", {})) + tree = order_classes(annotator.visit(tree)) + imports = ["import os", "import re", "import pyarrow", TYPING_IMPORT, + "from _typeshed import Incomplete"] + imports += sorted(f"import {m}" for m in annotator.imports) + imports += traced.get("imports", []) + if missing and module is not pyarrow.lib: + imports.append(f"from pyarrow.lib import {', '.join(sorted(missing))}") + # Names the stub imports already stay with that import; ruff drops the + # unused ones afterwards. + for node in reversed([ast.parse(line).body[0] for line in imports]): + node.names = [a for a in node.names + if (a.asname or a.name.split(".")[0]) not in defined] + if node.names: + tree.body.insert(0, node) + tree = Cleanup(module).visit(tree) + known = defined_names(tree) + for node in tree.body: # star imports bring in the runtime module's names + if isinstance(node, ast.ImportFrom) and node.names[0].name == "*": + known |= set(dir(importlib.import_module( + "." * node.level + (node.module or ""), module.__name__))) + tidy = Tidy(module, known) + tree = tidy.visit(tree) + tree.body[:0] = [ast.parse(f"import {m}").body[0] for m in sorted(tidy.imports)] + path.write_text(header + "\n" + ast.unparse(tree) + "\n") + + +# --- MonkeyType traces ----------------------------------------------------- + +class Modernize(ast.NodeTransformer): + """Spell the typing module forms MonkeyType renders the way the rest of + the stubs do: `X | None`, `A | B` and `list[X]`, and the platform's path + class as `Path`.""" + + BUILTIN = {"List": "list", "Dict": "dict", "Tuple": "tuple", "Set": "set", + "FrozenSet": "frozenset", "Type": "type"} + PORTABLE = {"PosixPath": "Path", "WindowsPath": "Path"} + + def visit_Name(self, node): + node.id = self.PORTABLE.get(node.id, node.id) + return node + + def visit_Subscript(self, node): + self.generic_visit(node) + if not isinstance(node.value, ast.Name): + return node + if node.value.id == "Optional": + return ast.BinOp(node.slice, ast.BitOr(), ast.Constant(None)) + if node.value.id == "Union": + parts = node.slice.elts if isinstance( + node.slice, ast.Tuple) else [node.slice] + return functools.reduce(lambda a, b: ast.BinOp(a, ast.BitOr(), b), parts) + node.value.id = self.BUILTIN.get(node.value.id, node.value.id) + return node + + +def trace_tests(directory): + """Run the shipped test suite in parallel under MonkeyType, recording + into one database per worker in directory.""" + log.info("Tracing the test suite with MonkeyType") + env = {**os.environ, "MT_DB_DIR": str(directory), + "PYTHONPATH": str(pathlib.Path(__file__).parent)} + run = subprocess.run([sys.executable, "-m", "pytest", "--pyargs", "pyarrow.tests", + "-q", "-p", "no:cacheprovider", "-p", "add_types_to_stubs", + "-n", "auto"], + cwd=directory, env=env, capture_output=True, text=True) + if run.returncode > 1: # failing tests are fine, not collecting is not + log.warning("pytest exited with %d:\n%s", run.returncode, + "\n".join(run.stdout.splitlines()[-15:])) + + +def load_traces(directory): + """Per module: traced annotations keyed by (class, function), plus the + imports they need, keeping only names from TRACE_IMPORTS modules.""" + stores = [] + for db in sorted(directory.glob("*.sqlite3")): + os.environ["MT_DB_PATH"] = str(db) + stores.append(CONFIG.trace_store()) + result = {} + for module in sorted({m for store in stores for m in store.list_modules()}): + log.info("Reading traces of %s", module) + traces = [] + # The default limit of 2000 drops traces of the big modules, and + # which ones depends on the order the tests ran in. + for thunk in (t for store in stores + for t in store.filter(module, limit=sys.maxsize)): + try: + traces.append(thunk.to_trace()) + except Exception: # types local to a test function; the CLI skips too + pass + stub = build_module_stubs_from_traces( + traces, CONFIG.max_typed_dict_size).get(module) + if stub is None: + continue + tree = Modernize().visit(ast.parse(stub.render())) + imports = [n for n in tree.body if isinstance(n, ast.ImportFrom) + and n.module.split(".")[0] in TRACE_IMPORTS] + for node in imports: # Python 3.13 defines Path in pathlib._local + public = importlib.import_module(node.module.split("._")[0]) + for alias in node.names: + alias.name = Modernize.PORTABLE.get(alias.name, alias.name) + if all(hasattr(public, a.name) for a in node.names): + node.module = public.__name__ + allowed = {a.asname or a.name for n in imports for a in n.names} | set( + dir(builtins)) + + def usable(node): + names = {n.id for n in ast.walk(node) if isinstance(n, ast.Name)} + return names <= allowed and ast.unparse(node) not in ("Any", "None") + + functions = {} + scopes = [(None, tree.body)] + scopes += [(n.name, n.body) for n in tree.body if isinstance(n, ast.ClassDef)] + for owner, body in scopes: + for n in body: + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)): + args = n.args.posonlyargs + n.args.args + n.args.kwonlyargs + params = {a.arg: a.annotation for a in args + if a.annotation and usable(a.annotation)} + returns = n.returns if n.returns and usable(n.returns) else None + functions[(owner, n.name)] = (params, returns) + result[module] = {"functions": functions, + "imports": [ast.unparse(n) for n in imports]} + return result + + +# --- formatting and linting ------------------------------------------------ + +def lint(): + """Apply the fixes ruff has for the stubs (unused and duplicate imports, + typing spellings, flake8-pyi conventions), format them, and report what + it still finds. The pre-commit hooks run the same configuration.""" + log.info("Formatting and linting the stubs") + ruff = pathlib.Path(sys.executable).parent / "ruff" + subprocess.run([ruff, "check", "--quiet", "--fix", "--unsafe-fixes", STUBS], + stdout=subprocess.DEVNULL) + subprocess.run([ruff, "format", "--quiet", STUBS]) + report = subprocess.run([ruff, "check", "--quiet", "--output-format", "concise", + STUBS], capture_output=True, text=True).stdout + print(f"ruff findings: {len(report.splitlines())}") + print(report, end="") + + +# --- completeness score ---------------------------------------------------- + +def verify_types(): + """Score the stubs with pyright as if they shipped inside the package.""" + log.info("Scoring type completeness with pyright") + package = pathlib.Path(pyarrow.__file__).parent + copied = [package / "py.typed"] + copied[0].touch() + for path in STUBS.rglob("*.pyi"): + target = package / path.relative_to(STUBS / "pyarrow") + if not target.exists(): + shutil.copy(path, target) + copied.append(target) + try: + # pyright finds the package through `python` + binaries = pathlib.Path(sys.executable).parent + env = {**os.environ, "PATH": f"{binaries}{os.pathsep}{os.environ['PATH']}"} + command = [binaries / "pyright", "--verifytypes", "pyarrow", "--ignoreexternal"] + report = subprocess.run(command, capture_output=True, text=True, env=env).stdout + finally: + for path in copied: + path.unlink() + summary = r"Symbols exported|Other symbols| With|Type completeness" + print(*(line for line in report.splitlines() if re.match(summary, line)), sep="\n") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--no-traces", action="store_true", + help="skip tracing the test suite") + args = parser.parse_args() + logging.basicConfig(format="%(asctime)s %(message)s", level=logging.INFO) + for sub in SUBMODULES: # make them attributes of pyarrow for name resolution + importlib.import_module(f"pyarrow.{sub}") + traced = {} + if not args.no_traces: + with tempfile.TemporaryDirectory() as tmp: + trace_tests(pathlib.Path(tmp)) + traced = load_traces(pathlib.Path(tmp)) + log.info("Reading return types from the Cython sources") + sourced = cython_return_types.return_types(sorted(SOURCES.glob("*.pyx"))) + for path in sorted(STUBS.rglob("*.pyi")): + name = ".".join(path.relative_to(STUBS).with_suffix( + "").parts).removesuffix(".__init__") + try: + module = importlib.import_module(name) + except ImportError: + log.info("Skipping %s, not importable", name) + continue + log.info("Annotating %s", path) + rewrite(path, module, sourced.get(name, {}), traced.get(name, {})) + print("traced modules:", len(traced)) + for key, value in sorted(stats.items()): + print(f"{key}: {value}") + print("unmapped docstring types:", * + (f"{n}x {t!r}" for t, n in unresolved.most_common(20)), sep="\n ") + print("undefined names made Incomplete:", * + (f"{n}x {t}" for t, n in undefined.most_common()), sep="\n ") + lint() + verify_types() + + +if __name__ == "__main__": + main() diff --git a/python/scripts/cython_return_types.py b/python/scripts/cython_return_types.py new file mode 100644 index 000000000000..315912c4f80e --- /dev/null +++ b/python/scripts/cython_return_types.py @@ -0,0 +1,210 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Derive return types of Cython ``def`` functions from their sources. + +stubgen-pyx carries declared parameter types into the stubs, but Cython +functions rarely declare what they return. Their return statements are +regular enough to read: wrapper calls such as ``pyarrow_wrap_table(...)``, +constructors and ``cls(...)``, ``self``, literals, builtins, and locals that +were declared ``cdef Table result`` or assigned from one of those. A function +without a return value returns None, unless its body raises unconditionally, +which marks an abstract method whose overrides return something else. + +Types are reported by the names used in the sources; the caller resolves +them against the stub they are written into. +""" + +import re + +from Cython.Compiler import ExprNodes, Nodes +from Cython.Compiler.Visitor import TreeVisitor +from stubgen_pyx.parsing.parser import parse_pyx + +WRAPPERS = {"array": "Array", "chunked_array": "ChunkedArray", "table": "Table", + "batch": "RecordBatch", "schema": "Schema", "field": "Field", + "data_type": "DataType", "buffer": "Buffer", + "resizable_buffer": "ResizableBuffer", "tensor": "Tensor", + "scalar": "Scalar", "metadata": "KeyValueMetadata", + "sparse_coo_tensor": "SparseCOOTensor", + "sparse_csr_matrix": "SparseCSRMatrix", + "sparse_csc_matrix": "SparseCSCMatrix", + "sparse_csf_tensor": "SparseCSFTensor"} +HELPERS = {"frombytes": "str", "tobytes": "bytes", "primitive_type": "DataType", + "isinstance": "bool", "callable": "bool", "len": "int", "str": "str", + "repr": "str", "int": "int", "float": "float", "bool": "bool", + "bytes": "bytes", "list": "list", "sorted": "list", "dict": "dict", + "tuple": "tuple", "set": "set", "frozenset": "frozenset"} +LITERALS = {ExprNodes.NoneNode: "None", ExprNodes.BoolNode: "bool", + ExprNodes.IntNode: "int", ExprNodes.FloatNode: "float", + ExprNodes.UnicodeNode: "str", ExprNodes.JoinedStrNode: "str", + ExprNodes.BytesNode: "bytes", ExprNodes.ListNode: "list", + ExprNodes.DictNode: "dict", ExprNodes.TupleNode: "tuple", + ExprNodes.SetNode: "set", ExprNodes.PrimaryCmpNode: "bool", + ExprNodes.NotNode: "bool"} +C_SCALARS = {"bint": "bool", "double": "float", "float": "float", + "c_string": "bytes", "unicode": "str"} +C_INTEGER = re.compile( + r"^(u?int(8|16|32|64)?_t|unsigned .*|int|long|size_t|Py_ssize_t)$") +SKIPPED = ("__init__", "__cinit__", "__dealloc__") + + +def python_type(c_type, classes): + """The Python type a C type converts to, or None.""" + if c_type in C_SCALARS: + return C_SCALARS[c_type] + if C_INTEGER.match(c_type): + return "int" + return c_type if c_type in classes else None + + +class Returns(TreeVisitor): + """Collect the return types of def functions in one module.""" + + def __init__(self, classes): + super().__init__() + self.classes = classes + self.found = {} + self.owner = None + self.function = None + + def visit_Node(self, node): + self.visitchildren(node) + + def visit_CClassDefNode(self, node): + self.owner = node.class_name + self.visitchildren(node) + self.owner = None + + def visit_PyClassDefNode(self, node): + self.owner = node.name + self.visitchildren(node) + self.owner = None + + def visit_DefNode(self, node): + if self.function is not None or node.name in SKIPPED: + return # nested function, or a constructor + self.function = (self.owner, node.name) + self.locals = {} + self.returns, self.yields = [], [] + self.visitchildren(node) + statements = getattr(node.body, "stats", [node.body]) + raises = any(isinstance(s, Nodes.RaiseStatNode) for s in statements) + self.found[self.function] = self.result(raises) + self.function = None + + def result(self, raises): + if self.yields: + inner = self.union(self.yields) or "Any" + return f"Iterator[{inner}]" + if not self.returns: + return None if raises else "None" + return self.union(self.returns) + + @staticmethod + def union(types): + return " | ".join(dict.fromkeys(types)) if None not in types else None + + def visit_CVarDefNode(self, node): # cdef Table result + if self.function and isinstance(node.base_type, Nodes.CSimpleBaseTypeNode): + declared = python_type(node.base_type.name, self.classes) + for declarator in node.declarators: + if declared and isinstance(declarator, Nodes.CNameDeclaratorNode): + self.locals[declarator.name] = declared + self.visitchildren(node) + + def visit_SingleAssignmentNode(self, node): # result = pyarrow_wrap_table(...) + if self.function and isinstance(node.lhs, ExprNodes.NameNode): + assigned = self.classify(node.rhs) + if assigned and node.lhs.name not in self.locals: + self.locals[node.lhs.name] = assigned + self.visitchildren(node) + + def visit_ReturnStatNode(self, node): + if self.function: + value = "None" if node.value is None else self.classify(node.value) + self.returns.append(value) + + def visit_YieldExprNode(self, node): + if self.function: + self.yields.append(self.classify(node.arg) if node.arg else None) + self.visitchildren(node) + + def classify(self, node): + """The type of an expression, or None when it cannot be told.""" + for kind, name in LITERALS.items(): + if isinstance(node, kind): + return name + if isinstance(node, ExprNodes.ComprehensionNode): + return getattr(node.type, "name", None) + if isinstance(node, ExprNodes.NameNode): + if node.name == "self": + return self.function[0] + if node.name == "cls": + return "Self" + return self.locals.get(node.name) + if isinstance(node, ExprNodes.TypecastNode): + if isinstance(node.base_type, Nodes.CSimpleBaseTypeNode): + return python_type(node.base_type.name, self.classes) + if isinstance(node, ExprNodes.CondExprNode): + return self.union([self.classify(node.true_val), + self.classify(node.false_val)]) + if isinstance(node, (ExprNodes.SimpleCallNode, ExprNodes.GeneralCallNode)): + return self.classify_call(node.function) + return None + + def classify_call(self, function): + if isinstance(function, ExprNodes.NameNode): + name = function.name + if name.startswith("pyarrow_wrap_"): + return WRAPPERS.get(name[len("pyarrow_wrap_"):]) + if name == "cls": + return "Self" + return HELPERS.get(name) or (name if name in self.classes else None) + if isinstance(function, ExprNodes.AttributeNode): + owner, method = function.obj, function.attribute + if isinstance(owner, ExprNodes.UnicodeNode) and method == "join": + return "str" + if not isinstance(owner, ExprNodes.NameNode): + return None + constructs = (method in ("wrap", "_wrap", "__new__") + or method.startswith("from_")) + if owner.name == "cls" and constructs: + return "Self" + if owner.name in self.classes and constructs: + return owner.name + return None + + +def module_classes(parsed): + body = getattr(parsed.source_ast.body, "stats", []) + return ({n.class_name for n in body if isinstance(n, Nodes.CClassDefNode)} + | {n.name for n in body if isinstance(n, Nodes.PyClassDefNode)}) + + +def return_types(paths): + """Map ``pyarrow.`` to ``{(class, function): type}`` for the + functions in the given .pyx files whose return type can be read.""" + parsed = {path: parse_pyx(path.read_text(), pyx_path=path) for path in paths} + classes = set().union(*(module_classes(p) for p in parsed.values())) + result = {} + for path, source in parsed.items(): + visitor = Returns(classes) + visitor.visit(source.source_ast) + result[f"pyarrow.{path.stem}"] = { + key: value for key, value in visitor.found.items() if value} + return result diff --git a/python/scripts/generate_stubs.sh b/python/scripts/generate_stubs.sh new file mode 100755 index 000000000000..e77e673b9407 --- /dev/null +++ b/python/scripts/generate_stubs.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generates python/pyarrow-stubs from the Cython and Python sources and checks +# the result against the nightly PyArrow wheel with mypy.stubtest. +# Run from the repository root. Requires uv, git and GNU sed. +set -euo pipefail + +venv=python/.venv-stubgen +stubs=python/pyarrow-stubs + +# MonkeyType does not support Python 3.14 yet. +[[ -d $venv ]] || uv venv --no-project --python 3.13 "$venv" +uv pip install --python "$venv/bin/python" --upgrade --prerelease allow \ + --default-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \ + pyarrow +uv pip install --python "$venv/bin/python" --upgrade \ + 'git+https://github.com/astral-sh/docstring-adder' stubgen-pyx 'mypy>=2.1' \ + 'numpy>=2' cffi +# For scripts/add_types_to_stubs.py: tracing the test suite, scoring, +# formatting and linting. +uv pip install --python "$venv/bin/python" --upgrade \ + -r python/requirements-test.txt monkeytype pytest-xdist pyright ruff +# shellcheck source=/dev/null +source "$venv/bin/activate" + +rm -rf "$stubs" +stubgen-pyx python/pyarrow --file '*.pyx' --include-private \ + --exclude-docstrings --output-dir "$stubs/pyarrow" + +# Make the Cython stubs match what Python sees at runtime: drop special +# methods and C typedefs that have no Python counterpart, add the `mod` +# argument and reflected methods Cython gives operator slots, annotate +# unresolved names (private class attributes included, enum members excluded +# so they stay members), declare the `__slots__` attribute stubgen-pyx skips, +# and keep its provenance comment within the line length. +sed -E -i \ + -e '/^\s*def __(dealloc|getbuffer|releasebuffer|richcmp)__\(/d' \ + -e '/^\w+: TypeAlias = /d' \ + -e 's/^(\s*def __pow__\(self[^,)]*, [^,)]*)\)/\1, mod: None = None)/' \ + -e 's/^(\s*)def __(add|sub|mul|matmul|truediv|floordiv|mod|divmod|pow|lshift|rshift|and|or|xor)__\((self[^)]*)\)(.*)$/&\n\1def __r\2__(\3)\4/' \ + -e 's/^(\w+|\s+_\w*) = \.\.\.$/\1: Incomplete/' \ + -e '1a from _typeshed import Incomplete' \ + -e '/^class TracingServerMiddleware\(/a\ trace_context: Incomplete' \ + -e 's/^# This file was generated by stubgen-pyx v[0-9.]+ from /# Generated by stubgen-pyx from /' \ + "$stubs"/pyarrow/*.pyi +# Symbols stubgen-pyx cannot derive from the sources: chained and try/except +# guarded module assignments. +cat >> "$stubs/pyarrow/lib.pyi" <<'PYI' + +_NULL: NullScalar +NA: NullScalar +PYI +cat >> "$stubs/pyarrow/_dataset_parquet.pyi" <<'PYI' + +parquet_encryption_enabled: bool +PYI +cat >> "$stubs/pyarrow/_substrait.pyi" <<'PYI' + +from types import ModuleType +py_substrait: ModuleType | None +PYI + +# Python modules are parsed, not imported, so optional dependencies are not +# needed. pyarrow.compute builds its functions at import time and is inspected. +stubgen --no-import --include-private --search-path python --output "$stubs" \ + -m pyarrow -m pyarrow._compute_docstrings -m pyarrow.acero \ + -m pyarrow.benchmark -m pyarrow.cffi -m pyarrow.csv -m pyarrow.cuda \ + -m pyarrow.dataset -m pyarrow.feather -m pyarrow.flight -m pyarrow.fs \ + -m pyarrow.ipc -m pyarrow.json -m pyarrow.jvm -m pyarrow.orc \ + -m pyarrow.pandas_compat -m pyarrow.substrait -m pyarrow.types \ + -m pyarrow.util -m pyarrow.interchange -m pyarrow.interchange.buffer \ + -m pyarrow.interchange.column -m pyarrow.interchange.dataframe \ + -m pyarrow.interchange.from_dataframe -m pyarrow.parquet \ + -m pyarrow.parquet.core -m pyarrow.parquet.encryption +stubgen --inspect-mode --module pyarrow.compute --output "$stubs" +# mypy stubgen drops imports of private names and variables outside __all__. +echo 'from pyarrow.lib import _NULL as NULL' >> "$stubs/pyarrow/__init__.pyi" +echo 'EXCLUDED_PARQUET_PATHS: set[str]' >> "$stubs/pyarrow/parquet/core.pyi" +#add-docstrings --packages "$stubs/pyarrow" + +# Reuse this script's license header. +header=$(sed -n '/^# Licensed to the Apache/,/^# under the License\.$/p' "$0") +for stub in "$stubs"/pyarrow/*.pyi "$stubs"/pyarrow/*/*.pyi; do + { echo "$header"; echo; cat "$stub"; } > "$stub.tmp" + mv "$stub.tmp" "$stub" +done + +# Mismatches the stubs cannot express, as regexes over dotted runtime paths: +# attributes Cython adds to every module and class; cdef-class __init__ and +# __call__, which Cython exposes as (*args, **kwargs); the shipped test suite, +# vendored modules (mypy stubgen refuses them), bundled shared libraries and +# extension modules missing from the nightly wheel; names that are not +# identifiers or depend on optional imports; stubgen quirks. Entries go unused +# when the wheel includes more optional modules, which is fine. +allowlist=$(mktemp) +cat > "$allowlist" <<'TXT' +.*\.__(reduce_cython|setstate_cython|test|pyx_capi)__ +pyarrow\.(lib|gandiva|_\w+)\.\w+\.__(init|call)__ +pyarrow\.(tests|vendored)(\..*)? +pyarrow\.(conftest|_?cuda|gandiva) +pyarrow\.libarrow_python(_\w+)? +pyarrow\.compute\.(and|or) +pyarrow\.compute\._OptionsClassDoc(\..*)? +pyarrow\.parse_git +pyarrow\.acero\.(ds|DatasetModuleStub) +pyarrow\._compute\.SUPPORTED_INPUT_ARR_TYPES +pyarrow\.types\._INTEGER_TYPES +pyarrow\._dataset\.TaggedRecordBatch@base1 +pyarrow\.parquet\.core\.__all__ +pyarrow\._orc\.WriteOptions +TXT +# Check runtime coverage only; mypy build errors in the generated stubs would +# otherwise abort stubtest. +mypy_config=$(mktemp) +cat > "$mypy_config" <<'INI' +[mypy] + +[mypy-pyarrow.*] +ignore_errors = True +INI + +MYPYPATH=$PWD/$stubs python -m mypy.stubtest pyarrow --concise \ + --ignore-positional-only --ignore-disjoint-bases \ + --allowlist "$allowlist" --ignore-unused-allowlist \ + --mypy-config-file "$mypy_config" + +# Report how many public runtime symbols (module attributes and class members +# defined in each module) the stubs declare, and list the ones they miss. +python - "$stubs" <<'PY' +import ast, importlib, pkgutil, sys, types +import pyarrow + +stubs = sys.argv[1] + +def declared(node): + names = set() + for child in node.body: + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.add(child.name) + elif isinstance(child, ast.AnnAssign): + names.add(child.target.id) + elif isinstance(child, ast.Assign): + names.update(t.id for t in child.targets if isinstance(t, ast.Name)) + elif isinstance(child, ast.ImportFrom): + names.update(a.asname or a.name for a in child.names) + if isinstance(child, ast.ClassDef): + names.update(f"{child.name}.{n}" for n in declared(child)) + return names + +def public(namespace, module_name): + return {n for n, v in vars(namespace).items() + if not n.startswith("_") and not isinstance(v, types.ModuleType) + and getattr(v, "__module__", module_name) == module_name} + +total = covered = 0 +modules = ["pyarrow"] + [m.name for m in pkgutil.walk_packages(pyarrow.__path__, "pyarrow.")] +for name in modules: + if name.startswith(("pyarrow.tests", "pyarrow.vendored")) or name.endswith("conftest"): + continue + try: + module = importlib.import_module(name) + except ImportError: + continue + symbols = set() + for attr in public(module, name): + symbols.add(attr) + obj = getattr(module, attr) + if isinstance(obj, type): + symbols.update(f"{attr}.{m}" for m in vars(obj) if not m.startswith("_")) + path = f"{stubs}/{name.replace('.', '/')}" + path += "/__init__.pyi" if hasattr(module, "__path__") else ".pyi" + try: + stub = declared(ast.parse(open(path).read())) + except FileNotFoundError: + stub = set() + missing = sorted(symbols - stub) + total += len(symbols) + covered += len(symbols) - len(missing) + print(f"{name}: {len(symbols) - len(missing)}/{len(symbols)}", *missing) +print(f"Stub coverage: {covered}/{total} public symbols ({100 * covered // max(total, 1)}%)") +PY