From 8060fb2acba83387396191225865ddcd46a4a76b Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Wed, 7 Oct 2026 21:35:17 +0200 Subject: [PATCH 1/8] [Python] Add stub generation and type extraction tooling for pyarrow Generate pyarrow-stubs from the Cython and Python sources, validate them against the nightly wheel with mypy.stubtest, and report public symbol coverage. Annotate the stubs from Cython return statements, numpydoc docstrings, defaults, the runtime values of module and class attributes, and MonkeyType traces of the test suite; fix, format and lint them with ruff as configured in pyproject.toml; and score them with pyright. Type-check the test suite against the stubs. --- python/pyproject.toml | 31 +- python/scripts/add_types_to_stubs.py | 717 ++++++++++++++++++++++++++ python/scripts/cython_return_types.py | 210 ++++++++ python/scripts/generate_stubs.sh | 197 +++++++ 4 files changed, 1147 insertions(+), 8 deletions(-) create mode 100755 python/scripts/add_types_to_stubs.py create mode 100644 python/scripts/cython_return_types.py create mode 100755 python/scripts/generate_stubs.sh diff --git a/python/pyproject.toml b/python/pyproject.toml index 68c1a807dd51..7c73679e3c55 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -112,23 +112,24 @@ git_describe_command = 'git describe --dirty --tags --long --match "apache-arrow fallback_version = '26.0.0a0' # TODO: Enable type checking once stubs are merged +# The test suite is checked against the stubs: it calls the API with real +# values, so it validates the annotations. Test bodies are unannotated, hence +# check_untyped_defs. The library sources themselves are not checked yet. [tool.mypy] -files = ["pyarrow-stubs"] +files = ["pyarrow-stubs", "pyarrow/tests"] mypy_path = "$MYPY_CONFIG_FILE_DIR/pyarrow-stubs" +check_untyped_defs = true exclude = [ - "^pyarrow/", "^benchmarks/", "^examples/", "^scripts/", ] -# TODO: Enable type checking once stubs are merged [tool.pyright] pythonPlatform = "All" pythonVersion = "3.11" -include = ["pyarrow-stubs"] +include = ["pyarrow-stubs", "pyarrow/tests"] exclude = [ - "pyarrow", "benchmarks", "examples", "scripts", @@ -136,16 +137,30 @@ exclude = [ ] stubPath = "pyarrow-stubs" typeCheckingMode = "basic" +# The tests import optional dependencies as `x = None` on failure and then +# use them unguarded; every report of that idiom would be noise here. +reportOptionalMemberAccess = false -# TODO: Enable type checking once stubs are merged [tool.ty.environment] extra-paths = ["pyarrow-stubs"] [tool.ty.src] -include = ["pyarrow-stubs"] +include = ["pyarrow-stubs", "pyarrow/tests"] exclude = [ - "pyarrow", "benchmarks", "examples", "scripts", ] + +# ruff formats and lints the generated stubs only; the pre-commit hooks and +# scripts/add_types_to_stubs.py run it. The stubs follow the typeshed +# conventions its flake8-pyi rules describe. +[tool.ruff] +include = ["pyarrow-stubs/**/*.pyi"] +line-length = 88 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "W", "PYI", "UP006", "UP007", "UP035", "UP037", "UP045"] +# Star imports mirror the sources; in a stub they re-export. +ignore = ["F403", "F405"] diff --git a/python/scripts/add_types_to_stubs.py b/python/scripts/add_types_to_stubs.py new file mode 100755 index 000000000000..7d48a17b5405 --- /dev/null +++ b/python/scripts/add_types_to_stubs.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Add type annotations to python/pyarrow-stubs. + +Existing annotations are kept. Missing ones are filled from, in order: + +1. what the Cython sources return, read from their return statements + (see cython_return_types); +2. the numpydoc "Parameters" and "Returns" sections of docstrings, which + cover the Cython modules as well; +3. default values of parameters; +4. MonkeyType traces of the test suite, which cover the pure Python modules. + +Leaked Cython names are resolved first, since an unknown base class makes a +whole class unknown to type checkers. Module attributes are declared with the +type of their runtime value, and namedtuples as NamedTuple classes. The stubs +are then fixed, formatted and linted with ruff as configured in pyproject.toml, +which the pre-commit hooks run too. The script prints what it annotated, the +docstring type phrases it could not map, what ruff still reports, and pyright's +type completeness score for the result. + +Run from the repository root with python/.venv-stubgen/bin/python after +scripts/generate_stubs.sh. Pass --no-traces to redo everything but the slow +test suite tracing, for instance after changing this script. +""" + +import argparse +import ast +import builtins +import collections +import enum +import functools +import importlib +import inspect +import logging +import os +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +import types +import typing + +import monkeytype +import pyarrow +from monkeytype.config import DefaultConfig +from monkeytype.stubs import build_module_stubs_from_traces +from pyarrow.vendored.docscrape import NumpyDocString + +import cython_return_types + +log = logging.getLogger("add_types_to_stubs") + +STUBS = pathlib.Path("python/pyarrow-stubs") +SOURCES = pathlib.Path("python/pyarrow") +SUBMODULES = ("compute", "dataset", "fs", "parquet", + "flight", "csv", "json", "ipc", "orc") +BUILTINS = {"int": "int", "integer": "int", "bool": "bool", "boolean": "bool", + "str": "str", "string": "str", "float": "float", "bytes": "bytes", + "dict": "dict", "list": "list", "tuple": "tuple", "set": "set", + "object": "object", "none": "None", "any": "Any", "self": "Self", + "callable": "Callable[..., Any]", "function": "Callable[..., Any]", + "iterable": "Iterable[Any]", "iterator": "Iterator[Any]", + "sequence": "Sequence[Any]", "mapping": "Mapping[Any, Any]", + "integers": "int", "strings": "str", "file-like": "IO[Any]", + "file-like object": "IO[Any]", "file-like python object": "IO[Any]", + "path-like": "os.PathLike[str]"} +CONTAINERS = {"list of": "list[{}]", "sequence of": "Sequence[{}]", + "iterable of": "Iterable[{}]", "iterator of": "Iterator[{}]", + "tuple of": "tuple[{}, ...]", "set of": "set[{}]"} +GENERICS = {"list": "list", "tuple": "tuple", "dict": "dict", "set": "set", + "sequence": "Sequence", "iterable": "Iterable", "iterator": "Iterator"} +# Modules whose names traced annotations may refer to; others would make the +# stubs depend on optional packages such as pandas. +TRACE_IMPORTS = ("pyarrow", "typing", "collections", + "datetime", "decimal", "pathlib", "os", "io") +TYPING_IMPORT = ("from typing import Any, Callable, IO, Iterable, Iterator, " + "Literal, Mapping, NamedTuple, Self, Sequence") +# Cython C types that leak into the stubs through declared parameter and +# attribute types, and what Python passes for them; `Type` and `TimeUnit` are +# C enums. +CYTHON_TYPES = {"c_bool": "bool", "c_string": "bytes", "std_vector": "list", + "Type": "int", "TimeUnit": "int"} +# Annotations for module attributes, by the type of their runtime value. +VALUE_TYPES = {int: "int", float: "float", str: "str", bytes: "bytes", + bool: "bool", list: "list[Incomplete]", set: "set[Incomplete]", + frozenset: "frozenset[Incomplete]", tuple: "tuple[Incomplete, ...]", + dict: "dict[Incomplete, Incomplete]", re.Pattern: "re.Pattern[str]"} + +stats = collections.Counter() +unresolved = collections.Counter() +undefined = collections.Counter() + + +class TraceConfig(DefaultConfig): + """MonkeyType configuration: trace the installed package, not its tests.""" + + def code_filter(self): + return lambda code: ("/pyarrow/" in code.co_filename + and "/tests/" not in code.co_filename) + + +CONFIG = TraceConfig() + + +# This module doubles as a pytest plugin so that every pytest-xdist worker +# traces the tests it runs, each into its own database under MT_DB_DIR. + +def pytest_configure(config): + worker = os.environ.get("PYTEST_XDIST_WORKER", "main") + database = pathlib.Path(os.environ["MT_DB_DIR"]) / f"{worker}.sqlite3" + os.environ["MT_DB_PATH"] = str(database) + config.monkeytype = monkeytype.trace(CONFIG) + config.monkeytype.__enter__() + + +def pytest_unconfigure(config): + if hasattr(config, "monkeytype"): + config.monkeytype.__exit__(None, None, None) + + +# --- docstring types ------------------------------------------------------- + +def resolve(text, local, imports): + """Map one numpydoc type phrase to an annotation, or None if unknown.""" + text = re.sub(r"\s*\(.*?\)", "", text).strip(" `.") + if text.lower() in BUILTINS: + return BUILTINS[text.lower()] + for prefix, template in CONTAINERS.items(): + if text.lower().startswith(prefix + " "): + inner = resolve(text[len(prefix) + 1:], local, imports) + return template.format(inner) if inner else None + literal = re.fullmatch( + r"\{(.*)\}", text) or re.fullmatch(r"('[^']*'|\"[^\"]*\")", text) + if literal: + items = [i.strip() for i in literal.group(1).split(",")] + if all(re.fullmatch(r"'[^']*'|\"[^\"]*\"", i) for i in items): + return f"Literal[{', '.join(items)}]" + generic = re.fullmatch(r"(\w+)\[(.*)\]", text) + if generic and generic.group(1).lower() in GENERICS: + args = [resolve(a, local, imports) for a in generic.group(2).split(",")] + outer = GENERICS[generic.group(1).lower()] + return f"{outer}[{', '.join(args)}]" if all(args) else None + name = re.sub(r"^(pyarrow|pa)\.(lib\.)?", "", text) + if name in local: + return name + if re.fullmatch(r"[\w.]+", name): + for prefix in ("", *(f"{sub}." for sub in SUBMODULES)): + obj = pyarrow + for part in f"{prefix}{name}".split("."): + obj = getattr(obj, part, None) + if isinstance(obj, type): + if "." in prefix + name: + imports.add("pyarrow." + (prefix + name).rsplit(".", 1)[0]) + return f"pyarrow.{prefix}{name}" + unresolved[text] += 1 + return None + + +def annotation(text, local, imports): + """Map a numpydoc type string such as "int or str, optional".""" + match = re.search(r",?\s+((optional|default)\b.*)$", text) + body, tail = (text[:match.start()], match.group(1)) if match else (text, "") + parts = [resolve(p, local, imports) + for p in re.split(r",?\s+or\s+|,\s*|\s*\|\s*", body) if p.strip()] + if not parts or None in parts: + return None + result = " | ".join(dict.fromkeys(parts)) + if re.match(r"optional|default\s+`?None`?$", tail) and "None" not in parts: + result += " | None" + return result + + +def from_default(node): + if isinstance(node, ast.Constant): + if type(node.value) in (bool, int, float, str, bytes): + return type(node.value).__name__ + return None + + +# --- stub rewriting -------------------------------------------------------- + +class Cleanup(ast.NodeTransformer): + """Drop cimports of .pxd declarations, and make imports from pyarrow + explicit re-exports where the runtime module has the name, as + `from pyarrow.lib import Table` in a stub is not one.""" + + def __init__(self, module): + self.module = module + + def visit_ImportFrom(self, node): + origin = node.module or "" + if origin.startswith("pyarrow.includes"): + return None + if origin.startswith("pyarrow"): + for alias in node.names: + if alias.asname is None and hasattr(self.module, alias.name): + alias.asname = alias.name + return node + + +def is_namedtuple(node): + return (isinstance(node, ast.Call) and len(node.args) == 2 + and ast.unparse(node.func).endswith("namedtuple") + and isinstance(node.args[1], (ast.Tuple, ast.List))) + + +def namedtuple_fields(node): + return [ast.AnnAssign(ast.Name(f.value), ast.Name("Incomplete"), None, simple=1) + for f in node.args[1].elts] + + +def dedupe_imports(nodes): + """Keep the last import of each name: the stub's own import rather than + one added before it.""" + seen = set() + for node in reversed(nodes): + if isinstance(node, ast.ImportFrom) and node.names[0].name != "*": + node.names = [a for a in node.names if (a.asname or a.name) not in seen] + seen |= {a.asname or a.name for a in node.names} + return [n for n in nodes if not isinstance(n, ast.ImportFrom) or n.names] + + +def order_assignments(body): + """Move assignments after the definitions their values refer to; only + annotations may refer forward in a stub.""" + position = {} + for i, node in enumerate(body): + for name in defined_names(ast.Module([node], [])): + position[name] = i + ordered, waiting = [], collections.defaultdict(list) + for i, node in enumerate(body): + if isinstance(node, (ast.Assign, ast.AnnAssign)) and node.value is not None: + later = [position[n.id] for n in ast.walk(node.value) + if isinstance(n, ast.Name) and position.get(n.id, -1) > i] + if later: + waiting[max(later)].append(node) + continue + ordered.append(node) + ordered.extend(waiting.pop(i, [])) + return ordered + + +class Names(ast.NodeTransformer): + """Spell the names an annotation uses so that the stub knows them.""" + + def __init__(self, tidy): + self.tidy = tidy + + def visit_Name(self, node): + if node.id in self.tidy.known: + return node + if node.id in CYTHON_TYPES: + stats["Cython types spelled"] += 1 + return ast.Name(CYTHON_TYPES[node.id]) + if qualified := self.tidy.lookup(node.id): + stats["annotation names qualified"] += 1 + return ast.parse(qualified, mode="eval").body + undefined[node.id] += 1 + return ast.Name("Incomplete") + + +class Tidy(ast.NodeTransformer): + """Say what the stub can about attributes, using only names it knows. + + Module assignments become declarations typed from the runtime value: + namedtuples become NamedTuple classes, type aliases stay, and the rest + get the value's type or Incomplete; class constants likewise. Leaked + Cython types get their Python spelling, classes other modules define get + qualified, and other names an annotation does not know become + Incomplete. Imports come first and once, assignments move after the + names they alias, and `__enter__` and its kind return Self.""" + + SELF_RETURNING = {"__enter__", "__aenter__", "__new__", + "__iter__", "__aiter__"} + + def __init__(self, module, known): + self.module, self.known = module, known | set(dir(builtins)) + self.owner, self.owners, self.imports = None, [module], set() + + def lookup(self, name): + """A class of that name in pyarrow or a submodule, qualified.""" + for prefix in ("", *(f"{sub}." for sub in SUBMODULES)): + obj = pyarrow + for part in f"{prefix}{name}".split("."): + obj = getattr(obj, part, None) + if isinstance(obj, type): + if prefix: + self.imports.add(f"pyarrow.{prefix[:-1]}") + return f"pyarrow.{prefix}{name}" + return None + + def visit_Module(self, node): + body = order_assignments([self.declare(n) for n in node.body]) + imports = [n for n in body if isinstance(n, (ast.Import, ast.ImportFrom))] + node.body = dedupe_imports(imports) + [n for n in body if n not in imports] + self.generic_visit(node) + return node + + def declare(self, node): + if not (isinstance(node, ast.Assign) and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name)): + return node + name = node.targets[0].id + if name.startswith("__") and name.endswith("__"): + return node + owner = self.owners[-1] + if owner is not self.module: # class constants; members and aliases stay + constant = isinstance(node.value, (ast.Constant, ast.BinOp, ast.UnaryOp)) + member = isinstance(owner, type) and issubclass(owner, enum.Enum) + if not constant or member: + return node + value = getattr(owner, name, None) + if is_namedtuple(node.value): + stats["namedtuples declared"] += 1 + return ast.ClassDef(name=name, bases=[ast.Name("NamedTuple")], keywords=[], + body=namedtuple_fields(node.value) or [ast.Pass()], + decorator_list=[], type_params=[]) + if (isinstance(value, (type, types.UnionType, typing.TypeVar)) + or value is typing.Any or typing.get_origin(value) is not None): + return node + typed = from_default(node.value) or VALUE_TYPES.get(type(value)) + cls = type(value) + if not typed and cls.__name__ in self.known \ + and getattr(self.module, cls.__name__, None) is cls: + typed = cls.__name__ + stats["attributes typed from runtime" if typed + else "attributes left Incomplete"] += 1 + return ast.AnnAssign(ast.Name(name), ast.parse( + typed or "Incomplete", mode="eval").body, None, simple=1) + + def visit_ClassDef(self, node): + for i, base in enumerate(node.bases): + if is_namedtuple(base): + node.bases[i] = ast.Name("NamedTuple") + node.body[:0] = namedtuple_fields(base) + stats["namedtuples declared"] += 1 + self.owners.append(getattr(self.owners[-1], node.name, None)) + node.body = order_assignments([self.declare(n) for n in node.body]) + self.owner, outer = node.name, self.owner + self.generic_visit(node) + self.owner = outer + self.owners.pop() + return node + + def visit_FunctionDef(self, node): + self.generic_visit(node) + node.returns = self.annotation(node.returns) + returns = node.returns + if (self.owner and node.name in self.SELF_RETURNING + and isinstance(returns, ast.Name) and returns.id == self.owner): + node.returns = ast.Name("Self") + stats["returns changed to Self"] += 1 + return node + + visit_AsyncFunctionDef = visit_FunctionDef + + def visit_arg(self, node): + node.annotation = self.annotation(node.annotation) + return node + + def visit_AnnAssign(self, node): + node.annotation = self.annotation(node.annotation) + return node + + def annotation(self, node): + return Names(self).visit(node) if node else None + + +class Annotator(ast.NodeTransformer): + def __init__(self, module, local, sourced, traced): + self.module, self.local = module, local + self.sourced, self.traced = sourced, traced + self.imports, self.owners = set(), [(module, None)] + + def visit_ClassDef(self, node): + self.owners.append((getattr(self.owners[-1][0], node.name, None), node.name)) + self.generic_visit(node) + self.owners.pop() + return node + + def visit_FunctionDef(self, node): + runtime, owner = self.owners[-1] + try: + doc = NumpyDocString(inspect.getdoc( + getattr(runtime, node.name, None)) or "") + except Exception: + doc = {"Parameters": [], "Returns": []} + params = { + n.strip("* "): p.type for p in doc["Parameters"] for n in p.name.split(",")} + traced_params, traced_return = self.traced.get((owner, node.name), ({}, None)) + positional = node.args.posonlyargs + node.args.args + defaults = dict(zip(reversed(positional), reversed(node.args.defaults))) + defaults.update((a, d) for a, d in zip( + node.args.kwonlyargs, node.args.kw_defaults) if d) + for arg in positional + node.args.kwonlyargs: + if arg.arg in ("self", "cls"): + continue + stats["parameters"] += 1 + if arg.annotation: + stats["parameters already typed"] += 1 + continue + default = defaults.get(arg) + typed = annotation(params[arg.arg], self.local, + self.imports) if params.get(arg.arg) else None + source = "parameters typed from docstring" + if not typed and from_default(default): + typed, source = from_default(default), "parameters typed from default" + # Traces only show what the tests passed; a default they never + # saw, such as another class, means the observed type is too narrow. + unknown_default = default is not None and not ( + isinstance(default, ast.Constant) and default.value is not ...) + if not typed and arg.arg in traced_params and not unknown_default: + typed, source = ast.unparse( + traced_params[arg.arg]), "parameters typed from traces" + if typed: + if isinstance(default, ast.Constant) and default.value is None: + if "None" not in typed: + typed += " | None" + default_type = from_default(default) + if default_type and default_type not in typed.split(" | "): + typed += " | " + default_type + arg.annotation = ast.parse(typed, mode="eval").body + stats[source] += 1 + if node.name != "__init__": + stats["returns"] += 1 + if node.returns: + stats["returns already typed"] += 1 + elif (typed := annotation(self.sourced.get((owner, node.name), ""), + self.local, self.imports)): + node.returns = ast.parse(typed, mode="eval").body + stats["returns typed from Cython sources"] += 1 + elif len(doc["Returns"]) == 1 and (typed := annotation( + doc["Returns"][0].type or doc["Returns"][0].name, + self.local, self.imports)): + node.returns = ast.parse(typed, mode="eval").body + stats["returns typed from docstring"] += 1 + elif traced_return is not None: + node.returns = traced_return + stats["returns typed from traces"] += 1 + return node + + visit_AsyncFunctionDef = visit_FunctionDef + + +def defined_names(tree): + names = {n.name for n in tree.body if isinstance( + n, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef))} + names |= {a.asname or a.name for n in tree.body if isinstance( + n, ast.ImportFrom) for a in n.names} + names |= {a.asname or a.name.split(".")[0] for n in tree.body if isinstance( + n, ast.Import) for a in n.names} + names |= {t.id for n in tree.body if isinstance( + n, ast.Assign) for t in n.targets if isinstance(t, ast.Name)} + names |= {n.target.id for n in tree.body if isinstance( + n, ast.AnnAssign) and isinstance(n.target, ast.Name)} + return names + + +def order_classes(tree): + """Define classes after their bases; pyright treats a base defined later + in a stub as unknown. Everything else keeps its order.""" + classes = {n.name: n for n in tree.body if isinstance(n, ast.ClassDef)} + ordered, done = [], set() + + def emit(node): + if node.name in done: + return + done.add(node.name) + for base in node.bases: + if isinstance(base, ast.Name) and base.id in classes: + emit(classes[base.id]) + ordered.append(node) + + for node in classes.values(): + emit(node) + tree.body = [n for n in tree.body if not isinstance(n, ast.ClassDef)] + ordered + return tree + + +def rewrite(path, module, sourced, traced): + source = path.read_text() + header = "".join(line for line in source.splitlines( + keepends=True)[:20] if line.startswith("#")) + tree = ast.parse(source) + defined = defined_names(tree) + # `from pyarrow.lib import *` skips private names such as the base class + # of most Cython classes. + missing = {n.id for n in ast.walk(tree) + if isinstance(n, ast.Name) and n.id.startswith("_") + and n.id not in defined and hasattr(pyarrow.lib, n.id)} + local = {n for n in defined if isinstance(getattr(module, n, None), type)} + annotator = Annotator(module, local, sourced, traced.get("functions", {})) + tree = order_classes(annotator.visit(tree)) + imports = ["import os", "import re", "import pyarrow", TYPING_IMPORT, + "from _typeshed import Incomplete"] + imports += sorted(f"import {m}" for m in annotator.imports) + imports += traced.get("imports", []) + if missing and module is not pyarrow.lib: + imports.append(f"from pyarrow.lib import {', '.join(sorted(missing))}") + # Names the stub imports already stay with that import; ruff drops the + # unused ones afterwards. + for node in reversed([ast.parse(line).body[0] for line in imports]): + node.names = [a for a in node.names + if (a.asname or a.name.split(".")[0]) not in defined] + if node.names: + tree.body.insert(0, node) + tree = Cleanup(module).visit(tree) + known = defined_names(tree) + for node in tree.body: # star imports bring in the runtime module's names + if isinstance(node, ast.ImportFrom) and node.names[0].name == "*": + known |= set(dir(importlib.import_module( + "." * node.level + (node.module or ""), module.__name__))) + tidy = Tidy(module, known) + tree = tidy.visit(tree) + tree.body[:0] = [ast.parse(f"import {m}").body[0] for m in sorted(tidy.imports)] + path.write_text(header + "\n" + ast.unparse(tree) + "\n") + + +# --- MonkeyType traces ----------------------------------------------------- + +class Modernize(ast.NodeTransformer): + """Spell the typing module forms MonkeyType renders the way the rest of + the stubs do: `X | None`, `A | B` and `list[X]`, and the platform's path + class as `Path`.""" + + BUILTIN = {"List": "list", "Dict": "dict", "Tuple": "tuple", "Set": "set", + "FrozenSet": "frozenset", "Type": "type"} + PORTABLE = {"PosixPath": "Path", "WindowsPath": "Path"} + + def visit_Name(self, node): + node.id = self.PORTABLE.get(node.id, node.id) + return node + + def visit_Subscript(self, node): + self.generic_visit(node) + if not isinstance(node.value, ast.Name): + return node + if node.value.id == "Optional": + return ast.BinOp(node.slice, ast.BitOr(), ast.Constant(None)) + if node.value.id == "Union": + parts = node.slice.elts if isinstance( + node.slice, ast.Tuple) else [node.slice] + return functools.reduce(lambda a, b: ast.BinOp(a, ast.BitOr(), b), parts) + node.value.id = self.BUILTIN.get(node.value.id, node.value.id) + return node + + +def trace_tests(directory): + """Run the shipped test suite in parallel under MonkeyType, recording + into one database per worker in directory.""" + log.info("Tracing the test suite with MonkeyType") + env = {**os.environ, "MT_DB_DIR": str(directory), + "PYTHONPATH": str(pathlib.Path(__file__).parent)} + run = subprocess.run([sys.executable, "-m", "pytest", "--pyargs", "pyarrow.tests", + "-q", "-p", "no:cacheprovider", "-p", "add_types_to_stubs", + "-n", "auto"], + cwd=directory, env=env, capture_output=True, text=True) + if run.returncode > 1: # failing tests are fine, not collecting is not + log.warning("pytest exited with %d:\n%s", run.returncode, + "\n".join(run.stdout.splitlines()[-15:])) + + +def load_traces(directory): + """Per module: traced annotations keyed by (class, function), plus the + imports they need, keeping only names from TRACE_IMPORTS modules.""" + stores = [] + for db in sorted(directory.glob("*.sqlite3")): + os.environ["MT_DB_PATH"] = str(db) + stores.append(CONFIG.trace_store()) + result = {} + for module in sorted({m for store in stores for m in store.list_modules()}): + log.info("Reading traces of %s", module) + traces = [] + # The default limit of 2000 drops traces of the big modules, and + # which ones depends on the order the tests ran in. + for thunk in (t for store in stores + for t in store.filter(module, limit=sys.maxsize)): + try: + traces.append(thunk.to_trace()) + except Exception: # types local to a test function; the CLI skips too + pass + stub = build_module_stubs_from_traces( + traces, CONFIG.max_typed_dict_size).get(module) + if stub is None: + continue + tree = Modernize().visit(ast.parse(stub.render())) + imports = [n for n in tree.body if isinstance(n, ast.ImportFrom) + and n.module.split(".")[0] in TRACE_IMPORTS] + for node in imports: # Python 3.13 defines Path in pathlib._local + public = importlib.import_module(node.module.split("._")[0]) + for alias in node.names: + alias.name = Modernize.PORTABLE.get(alias.name, alias.name) + if all(hasattr(public, a.name) for a in node.names): + node.module = public.__name__ + allowed = {a.asname or a.name for n in imports for a in n.names} | set( + dir(builtins)) + + def usable(node): + names = {n.id for n in ast.walk(node) if isinstance(n, ast.Name)} + return names <= allowed and ast.unparse(node) not in ("Any", "None") + + functions = {} + scopes = [(None, tree.body)] + scopes += [(n.name, n.body) for n in tree.body if isinstance(n, ast.ClassDef)] + for owner, body in scopes: + for n in body: + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)): + args = n.args.posonlyargs + n.args.args + n.args.kwonlyargs + params = {a.arg: a.annotation for a in args + if a.annotation and usable(a.annotation)} + returns = n.returns if n.returns and usable(n.returns) else None + functions[(owner, n.name)] = (params, returns) + result[module] = {"functions": functions, + "imports": [ast.unparse(n) for n in imports]} + return result + + +# --- formatting and linting ------------------------------------------------ + +def lint(): + """Apply the fixes ruff has for the stubs (unused and duplicate imports, + typing spellings, flake8-pyi conventions), format them, and report what + it still finds. The pre-commit hooks run the same configuration.""" + log.info("Formatting and linting the stubs") + ruff = pathlib.Path(sys.executable).parent / "ruff" + subprocess.run([ruff, "check", "--quiet", "--fix", "--unsafe-fixes", STUBS], + stdout=subprocess.DEVNULL) + subprocess.run([ruff, "format", "--quiet", STUBS]) + report = subprocess.run([ruff, "check", "--quiet", "--output-format", "concise", + STUBS], capture_output=True, text=True).stdout + print(f"ruff findings: {len(report.splitlines())}") + print(report, end="") + + +# --- completeness score ---------------------------------------------------- + +def verify_types(): + """Score the stubs with pyright as if they shipped inside the package.""" + log.info("Scoring type completeness with pyright") + package = pathlib.Path(pyarrow.__file__).parent + copied = [package / "py.typed"] + copied[0].touch() + for path in STUBS.rglob("*.pyi"): + target = package / path.relative_to(STUBS / "pyarrow") + if not target.exists(): + shutil.copy(path, target) + copied.append(target) + try: + # pyright finds the package through `python` + binaries = pathlib.Path(sys.executable).parent + env = {**os.environ, "PATH": f"{binaries}{os.pathsep}{os.environ['PATH']}"} + command = [binaries / "pyright", "--verifytypes", "pyarrow", "--ignoreexternal"] + report = subprocess.run(command, capture_output=True, text=True, env=env).stdout + finally: + for path in copied: + path.unlink() + summary = r"Symbols exported|Other symbols| With|Type completeness" + print(*(line for line in report.splitlines() if re.match(summary, line)), sep="\n") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--no-traces", action="store_true", + help="skip tracing the test suite") + args = parser.parse_args() + logging.basicConfig(format="%(asctime)s %(message)s", level=logging.INFO) + for sub in SUBMODULES: # make them attributes of pyarrow for name resolution + importlib.import_module(f"pyarrow.{sub}") + traced = {} + if not args.no_traces: + with tempfile.TemporaryDirectory() as tmp: + trace_tests(pathlib.Path(tmp)) + traced = load_traces(pathlib.Path(tmp)) + log.info("Reading return types from the Cython sources") + sourced = cython_return_types.return_types(sorted(SOURCES.glob("*.pyx"))) + for path in sorted(STUBS.rglob("*.pyi")): + name = ".".join(path.relative_to(STUBS).with_suffix( + "").parts).removesuffix(".__init__") + try: + module = importlib.import_module(name) + except ImportError: + log.info("Skipping %s, not importable", name) + continue + log.info("Annotating %s", path) + rewrite(path, module, sourced.get(name, {}), traced.get(name, {})) + print("traced modules:", len(traced)) + for key, value in sorted(stats.items()): + print(f"{key}: {value}") + print("unmapped docstring types:", * + (f"{n}x {t!r}" for t, n in unresolved.most_common(20)), sep="\n ") + print("undefined names made Incomplete:", * + (f"{n}x {t}" for t, n in undefined.most_common()), sep="\n ") + lint() + verify_types() + + +if __name__ == "__main__": + main() diff --git a/python/scripts/cython_return_types.py b/python/scripts/cython_return_types.py new file mode 100644 index 000000000000..315912c4f80e --- /dev/null +++ b/python/scripts/cython_return_types.py @@ -0,0 +1,210 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Derive return types of Cython ``def`` functions from their sources. + +stubgen-pyx carries declared parameter types into the stubs, but Cython +functions rarely declare what they return. Their return statements are +regular enough to read: wrapper calls such as ``pyarrow_wrap_table(...)``, +constructors and ``cls(...)``, ``self``, literals, builtins, and locals that +were declared ``cdef Table result`` or assigned from one of those. A function +without a return value returns None, unless its body raises unconditionally, +which marks an abstract method whose overrides return something else. + +Types are reported by the names used in the sources; the caller resolves +them against the stub they are written into. +""" + +import re + +from Cython.Compiler import ExprNodes, Nodes +from Cython.Compiler.Visitor import TreeVisitor +from stubgen_pyx.parsing.parser import parse_pyx + +WRAPPERS = {"array": "Array", "chunked_array": "ChunkedArray", "table": "Table", + "batch": "RecordBatch", "schema": "Schema", "field": "Field", + "data_type": "DataType", "buffer": "Buffer", + "resizable_buffer": "ResizableBuffer", "tensor": "Tensor", + "scalar": "Scalar", "metadata": "KeyValueMetadata", + "sparse_coo_tensor": "SparseCOOTensor", + "sparse_csr_matrix": "SparseCSRMatrix", + "sparse_csc_matrix": "SparseCSCMatrix", + "sparse_csf_tensor": "SparseCSFTensor"} +HELPERS = {"frombytes": "str", "tobytes": "bytes", "primitive_type": "DataType", + "isinstance": "bool", "callable": "bool", "len": "int", "str": "str", + "repr": "str", "int": "int", "float": "float", "bool": "bool", + "bytes": "bytes", "list": "list", "sorted": "list", "dict": "dict", + "tuple": "tuple", "set": "set", "frozenset": "frozenset"} +LITERALS = {ExprNodes.NoneNode: "None", ExprNodes.BoolNode: "bool", + ExprNodes.IntNode: "int", ExprNodes.FloatNode: "float", + ExprNodes.UnicodeNode: "str", ExprNodes.JoinedStrNode: "str", + ExprNodes.BytesNode: "bytes", ExprNodes.ListNode: "list", + ExprNodes.DictNode: "dict", ExprNodes.TupleNode: "tuple", + ExprNodes.SetNode: "set", ExprNodes.PrimaryCmpNode: "bool", + ExprNodes.NotNode: "bool"} +C_SCALARS = {"bint": "bool", "double": "float", "float": "float", + "c_string": "bytes", "unicode": "str"} +C_INTEGER = re.compile( + r"^(u?int(8|16|32|64)?_t|unsigned .*|int|long|size_t|Py_ssize_t)$") +SKIPPED = ("__init__", "__cinit__", "__dealloc__") + + +def python_type(c_type, classes): + """The Python type a C type converts to, or None.""" + if c_type in C_SCALARS: + return C_SCALARS[c_type] + if C_INTEGER.match(c_type): + return "int" + return c_type if c_type in classes else None + + +class Returns(TreeVisitor): + """Collect the return types of def functions in one module.""" + + def __init__(self, classes): + super().__init__() + self.classes = classes + self.found = {} + self.owner = None + self.function = None + + def visit_Node(self, node): + self.visitchildren(node) + + def visit_CClassDefNode(self, node): + self.owner = node.class_name + self.visitchildren(node) + self.owner = None + + def visit_PyClassDefNode(self, node): + self.owner = node.name + self.visitchildren(node) + self.owner = None + + def visit_DefNode(self, node): + if self.function is not None or node.name in SKIPPED: + return # nested function, or a constructor + self.function = (self.owner, node.name) + self.locals = {} + self.returns, self.yields = [], [] + self.visitchildren(node) + statements = getattr(node.body, "stats", [node.body]) + raises = any(isinstance(s, Nodes.RaiseStatNode) for s in statements) + self.found[self.function] = self.result(raises) + self.function = None + + def result(self, raises): + if self.yields: + inner = self.union(self.yields) or "Any" + return f"Iterator[{inner}]" + if not self.returns: + return None if raises else "None" + return self.union(self.returns) + + @staticmethod + def union(types): + return " | ".join(dict.fromkeys(types)) if None not in types else None + + def visit_CVarDefNode(self, node): # cdef Table result + if self.function and isinstance(node.base_type, Nodes.CSimpleBaseTypeNode): + declared = python_type(node.base_type.name, self.classes) + for declarator in node.declarators: + if declared and isinstance(declarator, Nodes.CNameDeclaratorNode): + self.locals[declarator.name] = declared + self.visitchildren(node) + + def visit_SingleAssignmentNode(self, node): # result = pyarrow_wrap_table(...) + if self.function and isinstance(node.lhs, ExprNodes.NameNode): + assigned = self.classify(node.rhs) + if assigned and node.lhs.name not in self.locals: + self.locals[node.lhs.name] = assigned + self.visitchildren(node) + + def visit_ReturnStatNode(self, node): + if self.function: + value = "None" if node.value is None else self.classify(node.value) + self.returns.append(value) + + def visit_YieldExprNode(self, node): + if self.function: + self.yields.append(self.classify(node.arg) if node.arg else None) + self.visitchildren(node) + + def classify(self, node): + """The type of an expression, or None when it cannot be told.""" + for kind, name in LITERALS.items(): + if isinstance(node, kind): + return name + if isinstance(node, ExprNodes.ComprehensionNode): + return getattr(node.type, "name", None) + if isinstance(node, ExprNodes.NameNode): + if node.name == "self": + return self.function[0] + if node.name == "cls": + return "Self" + return self.locals.get(node.name) + if isinstance(node, ExprNodes.TypecastNode): + if isinstance(node.base_type, Nodes.CSimpleBaseTypeNode): + return python_type(node.base_type.name, self.classes) + if isinstance(node, ExprNodes.CondExprNode): + return self.union([self.classify(node.true_val), + self.classify(node.false_val)]) + if isinstance(node, (ExprNodes.SimpleCallNode, ExprNodes.GeneralCallNode)): + return self.classify_call(node.function) + return None + + def classify_call(self, function): + if isinstance(function, ExprNodes.NameNode): + name = function.name + if name.startswith("pyarrow_wrap_"): + return WRAPPERS.get(name[len("pyarrow_wrap_"):]) + if name == "cls": + return "Self" + return HELPERS.get(name) or (name if name in self.classes else None) + if isinstance(function, ExprNodes.AttributeNode): + owner, method = function.obj, function.attribute + if isinstance(owner, ExprNodes.UnicodeNode) and method == "join": + return "str" + if not isinstance(owner, ExprNodes.NameNode): + return None + constructs = (method in ("wrap", "_wrap", "__new__") + or method.startswith("from_")) + if owner.name == "cls" and constructs: + return "Self" + if owner.name in self.classes and constructs: + return owner.name + return None + + +def module_classes(parsed): + body = getattr(parsed.source_ast.body, "stats", []) + return ({n.class_name for n in body if isinstance(n, Nodes.CClassDefNode)} + | {n.name for n in body if isinstance(n, Nodes.PyClassDefNode)}) + + +def return_types(paths): + """Map ``pyarrow.`` to ``{(class, function): type}`` for the + functions in the given .pyx files whose return type can be read.""" + parsed = {path: parse_pyx(path.read_text(), pyx_path=path) for path in paths} + classes = set().union(*(module_classes(p) for p in parsed.values())) + result = {} + for path, source in parsed.items(): + visitor = Returns(classes) + visitor.visit(source.source_ast) + result[f"pyarrow.{path.stem}"] = { + key: value for key, value in visitor.found.items() if value} + return result diff --git a/python/scripts/generate_stubs.sh b/python/scripts/generate_stubs.sh new file mode 100755 index 000000000000..e77e673b9407 --- /dev/null +++ b/python/scripts/generate_stubs.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generates python/pyarrow-stubs from the Cython and Python sources and checks +# the result against the nightly PyArrow wheel with mypy.stubtest. +# Run from the repository root. Requires uv, git and GNU sed. +set -euo pipefail + +venv=python/.venv-stubgen +stubs=python/pyarrow-stubs + +# MonkeyType does not support Python 3.14 yet. +[[ -d $venv ]] || uv venv --no-project --python 3.13 "$venv" +uv pip install --python "$venv/bin/python" --upgrade --prerelease allow \ + --default-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \ + pyarrow +uv pip install --python "$venv/bin/python" --upgrade \ + 'git+https://github.com/astral-sh/docstring-adder' stubgen-pyx 'mypy>=2.1' \ + 'numpy>=2' cffi +# For scripts/add_types_to_stubs.py: tracing the test suite, scoring, +# formatting and linting. +uv pip install --python "$venv/bin/python" --upgrade \ + -r python/requirements-test.txt monkeytype pytest-xdist pyright ruff +# shellcheck source=/dev/null +source "$venv/bin/activate" + +rm -rf "$stubs" +stubgen-pyx python/pyarrow --file '*.pyx' --include-private \ + --exclude-docstrings --output-dir "$stubs/pyarrow" + +# Make the Cython stubs match what Python sees at runtime: drop special +# methods and C typedefs that have no Python counterpart, add the `mod` +# argument and reflected methods Cython gives operator slots, annotate +# unresolved names (private class attributes included, enum members excluded +# so they stay members), declare the `__slots__` attribute stubgen-pyx skips, +# and keep its provenance comment within the line length. +sed -E -i \ + -e '/^\s*def __(dealloc|getbuffer|releasebuffer|richcmp)__\(/d' \ + -e '/^\w+: TypeAlias = /d' \ + -e 's/^(\s*def __pow__\(self[^,)]*, [^,)]*)\)/\1, mod: None = None)/' \ + -e 's/^(\s*)def __(add|sub|mul|matmul|truediv|floordiv|mod|divmod|pow|lshift|rshift|and|or|xor)__\((self[^)]*)\)(.*)$/&\n\1def __r\2__(\3)\4/' \ + -e 's/^(\w+|\s+_\w*) = \.\.\.$/\1: Incomplete/' \ + -e '1a from _typeshed import Incomplete' \ + -e '/^class TracingServerMiddleware\(/a\ trace_context: Incomplete' \ + -e 's/^# This file was generated by stubgen-pyx v[0-9.]+ from /# Generated by stubgen-pyx from /' \ + "$stubs"/pyarrow/*.pyi +# Symbols stubgen-pyx cannot derive from the sources: chained and try/except +# guarded module assignments. +cat >> "$stubs/pyarrow/lib.pyi" <<'PYI' + +_NULL: NullScalar +NA: NullScalar +PYI +cat >> "$stubs/pyarrow/_dataset_parquet.pyi" <<'PYI' + +parquet_encryption_enabled: bool +PYI +cat >> "$stubs/pyarrow/_substrait.pyi" <<'PYI' + +from types import ModuleType +py_substrait: ModuleType | None +PYI + +# Python modules are parsed, not imported, so optional dependencies are not +# needed. pyarrow.compute builds its functions at import time and is inspected. +stubgen --no-import --include-private --search-path python --output "$stubs" \ + -m pyarrow -m pyarrow._compute_docstrings -m pyarrow.acero \ + -m pyarrow.benchmark -m pyarrow.cffi -m pyarrow.csv -m pyarrow.cuda \ + -m pyarrow.dataset -m pyarrow.feather -m pyarrow.flight -m pyarrow.fs \ + -m pyarrow.ipc -m pyarrow.json -m pyarrow.jvm -m pyarrow.orc \ + -m pyarrow.pandas_compat -m pyarrow.substrait -m pyarrow.types \ + -m pyarrow.util -m pyarrow.interchange -m pyarrow.interchange.buffer \ + -m pyarrow.interchange.column -m pyarrow.interchange.dataframe \ + -m pyarrow.interchange.from_dataframe -m pyarrow.parquet \ + -m pyarrow.parquet.core -m pyarrow.parquet.encryption +stubgen --inspect-mode --module pyarrow.compute --output "$stubs" +# mypy stubgen drops imports of private names and variables outside __all__. +echo 'from pyarrow.lib import _NULL as NULL' >> "$stubs/pyarrow/__init__.pyi" +echo 'EXCLUDED_PARQUET_PATHS: set[str]' >> "$stubs/pyarrow/parquet/core.pyi" +#add-docstrings --packages "$stubs/pyarrow" + +# Reuse this script's license header. +header=$(sed -n '/^# Licensed to the Apache/,/^# under the License\.$/p' "$0") +for stub in "$stubs"/pyarrow/*.pyi "$stubs"/pyarrow/*/*.pyi; do + { echo "$header"; echo; cat "$stub"; } > "$stub.tmp" + mv "$stub.tmp" "$stub" +done + +# Mismatches the stubs cannot express, as regexes over dotted runtime paths: +# attributes Cython adds to every module and class; cdef-class __init__ and +# __call__, which Cython exposes as (*args, **kwargs); the shipped test suite, +# vendored modules (mypy stubgen refuses them), bundled shared libraries and +# extension modules missing from the nightly wheel; names that are not +# identifiers or depend on optional imports; stubgen quirks. Entries go unused +# when the wheel includes more optional modules, which is fine. +allowlist=$(mktemp) +cat > "$allowlist" <<'TXT' +.*\.__(reduce_cython|setstate_cython|test|pyx_capi)__ +pyarrow\.(lib|gandiva|_\w+)\.\w+\.__(init|call)__ +pyarrow\.(tests|vendored)(\..*)? +pyarrow\.(conftest|_?cuda|gandiva) +pyarrow\.libarrow_python(_\w+)? +pyarrow\.compute\.(and|or) +pyarrow\.compute\._OptionsClassDoc(\..*)? +pyarrow\.parse_git +pyarrow\.acero\.(ds|DatasetModuleStub) +pyarrow\._compute\.SUPPORTED_INPUT_ARR_TYPES +pyarrow\.types\._INTEGER_TYPES +pyarrow\._dataset\.TaggedRecordBatch@base1 +pyarrow\.parquet\.core\.__all__ +pyarrow\._orc\.WriteOptions +TXT +# Check runtime coverage only; mypy build errors in the generated stubs would +# otherwise abort stubtest. +mypy_config=$(mktemp) +cat > "$mypy_config" <<'INI' +[mypy] + +[mypy-pyarrow.*] +ignore_errors = True +INI + +MYPYPATH=$PWD/$stubs python -m mypy.stubtest pyarrow --concise \ + --ignore-positional-only --ignore-disjoint-bases \ + --allowlist "$allowlist" --ignore-unused-allowlist \ + --mypy-config-file "$mypy_config" + +# Report how many public runtime symbols (module attributes and class members +# defined in each module) the stubs declare, and list the ones they miss. +python - "$stubs" <<'PY' +import ast, importlib, pkgutil, sys, types +import pyarrow + +stubs = sys.argv[1] + +def declared(node): + names = set() + for child in node.body: + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.add(child.name) + elif isinstance(child, ast.AnnAssign): + names.add(child.target.id) + elif isinstance(child, ast.Assign): + names.update(t.id for t in child.targets if isinstance(t, ast.Name)) + elif isinstance(child, ast.ImportFrom): + names.update(a.asname or a.name for a in child.names) + if isinstance(child, ast.ClassDef): + names.update(f"{child.name}.{n}" for n in declared(child)) + return names + +def public(namespace, module_name): + return {n for n, v in vars(namespace).items() + if not n.startswith("_") and not isinstance(v, types.ModuleType) + and getattr(v, "__module__", module_name) == module_name} + +total = covered = 0 +modules = ["pyarrow"] + [m.name for m in pkgutil.walk_packages(pyarrow.__path__, "pyarrow.")] +for name in modules: + if name.startswith(("pyarrow.tests", "pyarrow.vendored")) or name.endswith("conftest"): + continue + try: + module = importlib.import_module(name) + except ImportError: + continue + symbols = set() + for attr in public(module, name): + symbols.add(attr) + obj = getattr(module, attr) + if isinstance(obj, type): + symbols.update(f"{attr}.{m}" for m in vars(obj) if not m.startswith("_")) + path = f"{stubs}/{name.replace('.', '/')}" + path += "/__init__.pyi" if hasattr(module, "__path__") else ".pyi" + try: + stub = declared(ast.parse(open(path).read())) + except FileNotFoundError: + stub = set() + missing = sorted(symbols - stub) + total += len(symbols) + covered += len(symbols) - len(missing) + print(f"{name}: {len(symbols) - len(missing)}/{len(symbols)}", *missing) +print(f"Stub coverage: {covered}/{total} public symbols ({100 * covered // max(total, 1)}%)") +PY From be64e46a001e2d6da43ff9bdd5a2e6e1445a2a31 Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Fri, 9 Oct 2026 18:13:47 +0100 Subject: [PATCH 2/8] [Python] Generate stubs for the Cython modules with stubgen-pyx Output of stubgen-pyx over python/pyarrow/*.pyx, private names included and docstrings excluded, as scripts/generate_stubs.sh runs it. --- python/pyarrow-stubs/pyarrow/__init__.pyi | 29 - python/pyarrow-stubs/pyarrow/_acero.pyi | 69 + python/pyarrow-stubs/pyarrow/_azurefs.pyi | 11 + python/pyarrow-stubs/pyarrow/_compute.pyi | 542 +++++ python/pyarrow-stubs/pyarrow/_csv.pyi | 219 ++ python/pyarrow-stubs/pyarrow/_cuda.pyi | 80 + python/pyarrow-stubs/pyarrow/_dataset.pyi | 327 +++ python/pyarrow-stubs/pyarrow/_dataset_orc.pyi | 14 + .../pyarrow/_dataset_parquet.pyi | 159 ++ .../pyarrow/_dataset_parquet_encryption.pyi | 21 + python/pyarrow-stubs/pyarrow/_feather.pyi | 19 + python/pyarrow-stubs/pyarrow/_flight.pyi | 362 +++ python/pyarrow-stubs/pyarrow/_fs.pyi | 139 ++ python/pyarrow-stubs/pyarrow/_gcsfs.pyi | 18 + python/pyarrow-stubs/pyarrow/_hdfs.pyi | 17 + python/pyarrow-stubs/pyarrow/_json.pyi | 55 + python/pyarrow-stubs/pyarrow/_orc.pyi | 59 + python/pyarrow-stubs/pyarrow/_parquet.pyi | 277 +++ .../pyarrow/_parquet_encryption.pyi | 128 ++ .../pyarrow/_pyarrow_cpp_tests.pyi | 20 + python/pyarrow-stubs/pyarrow/_s3fs.pyi | 39 + python/pyarrow-stubs/pyarrow/_substrait.pyi | 28 + python/pyarrow-stubs/pyarrow/gandiva.pyi | 75 + python/pyarrow-stubs/pyarrow/lib.pyi | 2020 +++++++++++++++++ 24 files changed, 4698 insertions(+), 29 deletions(-) delete mode 100644 python/pyarrow-stubs/pyarrow/__init__.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_acero.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_azurefs.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_compute.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_csv.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_cuda.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_dataset.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_dataset_orc.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_feather.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_flight.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_fs.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_gcsfs.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_hdfs.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_json.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_orc.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_parquet.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_s3fs.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_substrait.pyi create mode 100644 python/pyarrow-stubs/pyarrow/gandiva.pyi create mode 100644 python/pyarrow-stubs/pyarrow/lib.pyi diff --git a/python/pyarrow-stubs/pyarrow/__init__.pyi b/python/pyarrow-stubs/pyarrow/__init__.pyi deleted file mode 100644 index ccec8d5abc07..000000000000 --- a/python/pyarrow-stubs/pyarrow/__init__.pyi +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Type stubs for PyArrow. - -This is a placeholder stub file. -Complete type annotations will be added in subsequent PRs. -""" - -from typing import Any - -# TODO(GH-48970): remove __getattr__ before release as this -# will annotate non-existing attributes as Any. -# https://github.com/apache/arrow/issues/48970 -def __getattr__(name: str) -> Any: ... diff --git a/python/pyarrow-stubs/pyarrow/_acero.pyi b/python/pyarrow-stubs/pyarrow/_acero.pyi new file mode 100644 index 000000000000..9a45895ff7be --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_acero.pyi @@ -0,0 +1,69 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_acero.pyx + +from pyarrow._compute import Expression +from pyarrow.includes.common import * +from pyarrow.includes.libarrow import * +from pyarrow.includes.libarrow_acero import * +from pyarrow.lib import * +from pyarrow.lib import RecordBatchReader, Table + + +class ExecNodeOptions(_Weakrefable): + __slots__ = () + +class _TableSourceNodeOptions(ExecNodeOptions): + def _set_options(self, table: Table): ... + +class TableSourceNodeOptions(_TableSourceNodeOptions): + def __init__(self, table: Table): ... + +class _RecordBatchReaderSourceNodeOptions(ExecNodeOptions): + def _set_options(self, reader: RecordBatchReader): ... + +class RecordBatchReaderSourceNodeOptions(_RecordBatchReaderSourceNodeOptions): + def __init__(self, reader: RecordBatchReader): ... + +class _FilterNodeOptions(ExecNodeOptions): + def _set_options(self, filter_expression: Expression): ... + +class FilterNodeOptions(_FilterNodeOptions): + def __init__(self, filter_expression: Expression): ... + +class _ProjectNodeOptions(ExecNodeOptions): + def _set_options(self, expressions, names=None): ... + +class ProjectNodeOptions(_ProjectNodeOptions): + def __init__(self, expressions, names=None): ... + +class _AggregateNodeOptions(ExecNodeOptions): + def _set_options(self, aggregates, keys=None): ... + +class AggregateNodeOptions(_AggregateNodeOptions): + def __init__(self, aggregates, keys=None): ... + +class _OrderByNodeOptions(ExecNodeOptions): + def _set_options(self, sort_keys, null_placement): ... + +class OrderByNodeOptions(_OrderByNodeOptions): + def __init__(self, sort_keys=(), *, null_placement=None): ... + +class _HashJoinNodeOptions(ExecNodeOptions): + def _set_options(self, join_type, left_keys, right_keys, left_output=None, right_output=None, output_suffix_for_left='', output_suffix_for_right='', filter_expression: Expression | None=None): ... + +class HashJoinNodeOptions(_HashJoinNodeOptions): + def __init__(self, join_type, left_keys, right_keys, left_output=None, right_output=None, output_suffix_for_left='', output_suffix_for_right='', filter_expression=None): ... + +class _AsofJoinNodeOptions(ExecNodeOptions): + def _set_options(self, left_on, left_by, right_on, right_by, tolerance): ... + +class AsofJoinNodeOptions(_AsofJoinNodeOptions): + def __init__(self, left_on, left_by, right_on, right_by, tolerance): ... + +class Declaration(_Weakrefable): + def __init__(self, factory_name, options: ExecNodeOptions, inputs=None): ... + @staticmethod + def from_sequence(decls): ... + def __str__(self): ... + def __repr__(self): ... + def to_table(self, use_threads: bool=True): ... + def to_reader(self, use_threads: bool=True): ... diff --git a/python/pyarrow-stubs/pyarrow/_azurefs.pyi b/python/pyarrow-stubs/pyarrow/_azurefs.pyi new file mode 100644 index 000000000000..8c5529dfa843 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_azurefs.pyi @@ -0,0 +1,11 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_azurefs.pyx + +from pyarrow._fs import FileSystem +from pyarrow.includes.libarrow_fs import * + + +class AzureFileSystem(FileSystem): + def __init__(self, account_name, *, account_key=None, blob_storage_authority=None, blob_storage_scheme=None, client_id=None, client_secret=None, dfs_storage_authority=None, dfs_storage_scheme=None, sas_token=None, tenant_id=None): ... + @staticmethod + def _reconstruct(kwargs): ... + def __reduce__(self): ... diff --git a/python/pyarrow-stubs/pyarrow/_compute.pyi b/python/pyarrow-stubs/pyarrow/_compute.pyi new file mode 100644 index 000000000000..7ce2cbe9460a --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_compute.pyi @@ -0,0 +1,542 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_compute.pyx + +from collections import namedtuple +from typing import Any, Callable + +import pyarrow.lib as lib +from pyarrow.includes.common import * +from pyarrow.includes.libarrow import * +from pyarrow.lib import * +from typing_extensions import TypeAlias + +__pas = None +_substrait_msg = 'The pyarrow installation is not built with support for Substrait.' +SUPPORTED_INPUT_ARR_TYPES = (list, tuple) +FunctionDoc = namedtuple('FunctionDoc', ('summary', 'description', 'arg_names', 'options_class', 'options_required')) +_deserialize = Expression._deserialize +CRegisterUdf: TypeAlias = Callable[[PyObject, function[CallbackUdf], CUdfOptions, CFunctionRegistry], CStatus] + +class Kernel(_Weakrefable): + def __init__(self): ... + +class ScalarKernel(Kernel): + def __repr__(self): ... + +class VectorKernel(Kernel): + def __repr__(self): ... + +class ScalarAggregateKernel(Kernel): + def __repr__(self): ... + +class HashAggregateKernel(Kernel): + def __repr__(self): ... + +class Function(_Weakrefable): + _kind_map = ... + + def __init__(self): ... + def __repr__(self): ... + def __reduce__(self): ... + @property + def name(self): ... + @property + def arity(self): ... + @property + def kind(self): ... + @property + def _doc(self): ... + @property + def num_kernels(self): ... + def call(self, args, options: FunctionOptions | None=None, memory_pool: MemoryPool | None=None, length=None): ... + +class ScalarFunction(Function): + @property + def kernels(self): ... + +class VectorFunction(Function): + @property + def kernels(self): ... + +class ScalarAggregateFunction(Function): + @property + def kernels(self): ... + +class HashAggregateFunction(Function): + @property + def kernels(self): ... + +class MetaFunction(Function): + @property + def kernels(self): ... + +class FunctionRegistry(_Weakrefable): + def __init__(self): ... + def list_functions(self): ... + def get_function(self, name): ... + +class FunctionOptions(_Weakrefable): + __slots__ = () + + def serialize(self): ... + @staticmethod + def deserialize(buf: Buffer): ... + def __repr__(self): ... + def __eq__(self, other: FunctionOptions): ... + +class _CastOptions(FunctionOptions): + def _set_options(self, target_type: DataType, allow_int_overflow, allow_time_truncate, allow_time_overflow, allow_decimal_truncate, allow_float_truncate, allow_invalid_utf8): ... + def _set_type(self, target_type=None): ... + def _set_safe(self): ... + def _set_unsafe(self): ... + def is_safe(self): ... + @property + def allow_int_overflow(self): ... + @allow_int_overflow.setter + def allow_int_overflow(self, flag: c_bool): ... + @property + def allow_time_truncate(self): ... + @allow_time_truncate.setter + def allow_time_truncate(self, flag: c_bool): ... + @property + def allow_time_overflow(self): ... + @allow_time_overflow.setter + def allow_time_overflow(self, flag: c_bool): ... + @property + def allow_decimal_truncate(self): ... + @allow_decimal_truncate.setter + def allow_decimal_truncate(self, flag: c_bool): ... + @property + def allow_float_truncate(self): ... + @allow_float_truncate.setter + def allow_float_truncate(self, flag: c_bool): ... + @property + def allow_invalid_utf8(self): ... + @allow_invalid_utf8.setter + def allow_invalid_utf8(self, flag: c_bool): ... + +class CastOptions(_CastOptions): + def __init__(self, target_type=None, *, allow_int_overflow=None, allow_time_truncate=None, allow_time_overflow=None, allow_decimal_truncate=None, allow_float_truncate=None, allow_invalid_utf8=None): ... + @staticmethod + def safe(target_type=None): ... + @staticmethod + def unsafe(target_type=None): ... + +class _ElementWiseAggregateOptions(FunctionOptions): + def _set_options(self, skip_nulls): ... + +class ElementWiseAggregateOptions(_ElementWiseAggregateOptions): + def __init__(self, *, skip_nulls=True): ... + +class _RoundOptions(FunctionOptions): + def _set_options(self, ndigits, round_mode): ... + +class RoundOptions(_RoundOptions): + def __init__(self, ndigits=0, round_mode='half_to_even'): ... + +class _RoundBinaryOptions(FunctionOptions): + def _set_options(self, round_mode): ... + +class RoundBinaryOptions(_RoundBinaryOptions): + def __init__(self, round_mode='half_to_even'): ... + +class _RoundTemporalOptions(FunctionOptions): + def _set_options(self, multiple, unit, week_starts_monday, ceil_is_strictly_greater, calendar_based_origin): ... + +class RoundTemporalOptions(_RoundTemporalOptions): + def __init__(self, multiple=1, unit='day', *, week_starts_monday=True, ceil_is_strictly_greater=False, calendar_based_origin=False): ... + +class _RoundToMultipleOptions(FunctionOptions): + def _set_options(self, multiple, round_mode): ... + +class RoundToMultipleOptions(_RoundToMultipleOptions): + def __init__(self, multiple=1.0, round_mode='half_to_even'): ... + +class _JoinOptions(FunctionOptions): + _null_handling_map = ... + + def _set_options(self, null_handling, null_replacement): ... + +class JoinOptions(_JoinOptions): + def __init__(self, null_handling='emit_null', null_replacement=''): ... + +class _MatchSubstringOptions(FunctionOptions): + def _set_options(self, pattern, ignore_case): ... + +class MatchSubstringOptions(_MatchSubstringOptions): + def __init__(self, pattern, *, ignore_case=False): ... + +class _PadOptions(FunctionOptions): + def _set_options(self, width, padding, lean_left_on_odd_padding): ... + +class PadOptions(_PadOptions): + def __init__(self, width, padding=' ', lean_left_on_odd_padding=True): ... + +class _ZeroFillOptions(FunctionOptions): + def _set_options(self, width, padding): ... + +class ZeroFillOptions(_ZeroFillOptions): + def __init__(self, width, padding='0'): ... + +class _TrimOptions(FunctionOptions): + def _set_options(self, characters): ... + +class TrimOptions(_TrimOptions): + def __init__(self, characters): ... + +class _ReplaceSubstringOptions(FunctionOptions): + def _set_options(self, pattern, replacement, max_replacements): ... + +class ReplaceSubstringOptions(_ReplaceSubstringOptions): + def __init__(self, pattern, replacement, *, max_replacements=None): ... + +class _ExtractRegexOptions(FunctionOptions): + def _set_options(self, pattern): ... + +class ExtractRegexOptions(_ExtractRegexOptions): + def __init__(self, pattern): ... + +class _ExtractRegexSpanOptions(FunctionOptions): + def _set_options(self, pattern): ... + +class ExtractRegexSpanOptions(_ExtractRegexSpanOptions): + def __init__(self, pattern): ... + +class _SliceOptions(FunctionOptions): + def _set_options(self, start, stop, step): ... + +class SliceOptions(_SliceOptions): + def __init__(self, start, stop=None, step=1): ... + +class _ListSliceOptions(FunctionOptions): + def _set_options(self, start, stop=None, step=1, return_fixed_size_list=None): ... + +class ListSliceOptions(_ListSliceOptions): + def __init__(self, start, stop=None, step=1, return_fixed_size_list=None): ... + +class _ReplaceSliceOptions(FunctionOptions): + def _set_options(self, start, stop, replacement): ... + +class ReplaceSliceOptions(_ReplaceSliceOptions): + def __init__(self, start, stop, replacement): ... + +class _FilterOptions(FunctionOptions): + _null_selection_map = ... + + def _set_options(self, null_selection_behavior): ... + +class FilterOptions(_FilterOptions): + def __init__(self, null_selection_behavior='drop'): ... + +class _DictionaryEncodeOptions(FunctionOptions): + _null_encoding_map = ... + + def _set_options(self, null_encoding): ... + +class DictionaryEncodeOptions(_DictionaryEncodeOptions): + def __init__(self, null_encoding='mask'): ... + +class _RunEndEncodeOptions(FunctionOptions): + def _set_options(self, run_end_type): ... + +class RunEndEncodeOptions(_RunEndEncodeOptions): + def __init__(self, run_end_type=lib.int32()): ... + +class _InversePermutationOptions(FunctionOptions): + def _set_options(self, max_index=-1, output_type=None): ... + +class InversePermutationOptions(_InversePermutationOptions): + def __init__(self, max_index=-1, output_type=None): ... + +class _ScatterOptions(FunctionOptions): + def _set_options(self, max_index): ... + +class ScatterOptions(_ScatterOptions): + def __init__(self, max_index=-1): ... + +class _TakeOptions(FunctionOptions): + def _set_options(self, boundscheck): ... + +class TakeOptions(_TakeOptions): + def __init__(self, *, boundscheck=True): ... + +class _MakeStructOptions(FunctionOptions): + def _set_options(self, field_names, field_nullability, field_metadata): ... + +class MakeStructOptions(_MakeStructOptions): + def __init__(self, field_names=(), *, field_nullability=None, field_metadata=None): ... + +class _StructFieldOptions(FunctionOptions): + def _set_options(self, indices): ... + +class StructFieldOptions(_StructFieldOptions): + def __init__(self, indices): ... + +class _ScalarAggregateOptions(FunctionOptions): + def _set_options(self, skip_nulls, min_count): ... + +class ScalarAggregateOptions(_ScalarAggregateOptions): + def __init__(self, *, skip_nulls=True, min_count=1): ... + +class _CountOptions(FunctionOptions): + _mode_map = ... + + def _set_options(self, mode): ... + +class CountOptions(_CountOptions): + def __init__(self, mode='only_valid'): ... + +class _IndexOptions(FunctionOptions): + def _set_options(self, scalar): ... + +class IndexOptions(_IndexOptions): + def __init__(self, value): ... + +class _MapLookupOptions(FunctionOptions): + _occurrence_map = ... + + def _set_options(self, query_key, occurrence): ... + +class MapLookupOptions(_MapLookupOptions): + def __init__(self, query_key, occurrence): ... + +class _ModeOptions(FunctionOptions): + def _set_options(self, n, skip_nulls, min_count): ... + +class ModeOptions(_ModeOptions): + def __init__(self, n=1, *, skip_nulls=True, min_count=0): ... + +class _SetLookupOptions(FunctionOptions): + def _set_options(self, value_set, skip_nulls: c_bool): ... + +class SetLookupOptions(_SetLookupOptions): + def __init__(self, value_set, *, skip_nulls=False): ... + +class _StrptimeOptions(FunctionOptions): + _unit_map = ... + + def _set_options(self, format, unit, error_is_null): ... + +class StrptimeOptions(_StrptimeOptions): + def __init__(self, format, unit, error_is_null=False): ... + +class _StrftimeOptions(FunctionOptions): + def _set_options(self, format, locale): ... + +class StrftimeOptions(_StrftimeOptions): + def __init__(self, format='%Y-%m-%dT%H:%M:%S', locale='C'): ... + +class _DayOfWeekOptions(FunctionOptions): + def _set_options(self, count_from_zero, week_start): ... + +class DayOfWeekOptions(_DayOfWeekOptions): + def __init__(self, *, count_from_zero=True, week_start=1): ... + +class _WeekOptions(FunctionOptions): + def _set_options(self, week_starts_monday, count_from_zero, first_week_is_fully_in_year): ... + +class WeekOptions(_WeekOptions): + def __init__(self, *, week_starts_monday=True, count_from_zero=False, first_week_is_fully_in_year=False): ... + +class _AssumeTimezoneOptions(FunctionOptions): + _ambiguous_map = ... + _nonexistent_map = ... + + def _set_options(self, timezone, ambiguous, nonexistent): ... + +class AssumeTimezoneOptions(_AssumeTimezoneOptions): + def __init__(self, timezone, *, ambiguous='raise', nonexistent='raise'): ... + +class _NullOptions(FunctionOptions): + def _set_options(self, nan_is_null): ... + +class NullOptions(_NullOptions): + def __init__(self, *, nan_is_null=False): ... + +class _VarianceOptions(FunctionOptions): + def _set_options(self, ddof, skip_nulls, min_count): ... + +class VarianceOptions(_VarianceOptions): + def __init__(self, *, ddof=0, skip_nulls=True, min_count=0): ... + +class _SkewOptions(FunctionOptions): + def _set_options(self, skip_nulls, biased, min_count): ... + +class SkewOptions(_SkewOptions): + def __init__(self, *, skip_nulls=True, biased=True, min_count=0): ... + +class _SplitOptions(FunctionOptions): + def _set_options(self, max_splits, reverse): ... + +class SplitOptions(_SplitOptions): + def __init__(self, *, max_splits=None, reverse=False): ... + +class _SplitPatternOptions(FunctionOptions): + def _set_options(self, pattern, max_splits, reverse): ... + +class SplitPatternOptions(_SplitPatternOptions): + def __init__(self, pattern, *, max_splits=None, reverse=False): ... + +class _PartitionNthOptions(FunctionOptions): + def _set_options(self, pivot, null_placement): ... + +class PartitionNthOptions(_PartitionNthOptions): + def __init__(self, pivot, *, null_placement='at_end'): ... + +class _WinsorizeOptions(FunctionOptions): + def _set_options(self, lower_limit, upper_limit): ... + +class WinsorizeOptions(_WinsorizeOptions): + def __init__(self, lower_limit, upper_limit): ... + +class _CumulativeOptions(FunctionOptions): + def _set_options(self, start, skip_nulls): ... + +class CumulativeOptions(_CumulativeOptions): + def __init__(self, start=None, *, skip_nulls=False): ... + +class CumulativeSumOptions(_CumulativeOptions): + def __init__(self, start=None, *, skip_nulls=False): ... + +class _PairwiseOptions(FunctionOptions): + def _set_options(self, period): ... + +class PairwiseOptions(_PairwiseOptions): + def __init__(self, period=1): ... + +class _ListFlattenOptions(FunctionOptions): + def _set_options(self, recursive): ... + +class ListFlattenOptions(_ListFlattenOptions): + def __init__(self, recursive=False): ... + +class _ArraySortOptions(FunctionOptions): + def _set_options(self, order, null_placement): ... + +class ArraySortOptions(_ArraySortOptions): + def __init__(self, order='ascending', *, null_placement='at_end'): ... + +class _SearchSortedOptions(FunctionOptions): + def _set_options(self, side): ... + +class SearchSortedOptions(_SearchSortedOptions): + def __init__(self, side='left'): ... + +class _SortOptions(FunctionOptions): + def _set_options(self, sort_keys, null_placement): ... + +class SortOptions(_SortOptions): + def __init__(self, sort_keys=(), *, null_placement=None): ... + +class _SelectKOptions(FunctionOptions): + def _set_options(self, k, sort_keys): ... + +class SelectKOptions(_SelectKOptions): + def __init__(self, k, sort_keys): ... + +class _QuantileOptions(FunctionOptions): + _interp_map = ... + + def _set_options(self, quantiles, interp, skip_nulls, min_count): ... + +class QuantileOptions(_QuantileOptions): + def __init__(self, q=0.5, *, interpolation='linear', skip_nulls=True, min_count=0): ... + +class _TDigestOptions(FunctionOptions): + def _set_options(self, quantiles, delta, buffer_size, skip_nulls, min_count): ... + +class TDigestOptions(_TDigestOptions): + def __init__(self, q=0.5, *, delta=100, buffer_size=500, skip_nulls=True, min_count=0): ... + +class _Utf8NormalizeOptions(FunctionOptions): + _form_map = ... + + def _set_options(self, form): ... + +class Utf8NormalizeOptions(_Utf8NormalizeOptions): + def __init__(self, form): ... + +class _RandomOptions(FunctionOptions): + def _set_options(self, initializer): ... + +class RandomOptions(_RandomOptions): + def __init__(self, *, initializer='system'): ... + +class _RankOptions(FunctionOptions): + _tiebreaker_map = ... + + def _set_options(self, sort_keys, null_placement, tiebreaker): ... + +class RankOptions(_RankOptions): + def __init__(self, sort_keys='ascending', *, null_placement=None, tiebreaker='first'): ... + +class _RankQuantileOptions(FunctionOptions): + def _set_options(self, sort_keys, null_placement): ... + +class RankQuantileOptions(_RankQuantileOptions): + def __init__(self, sort_keys='ascending', *, null_placement=None): ... + +class _PivotWiderOptions(FunctionOptions): + def _set_options(self, key_names, unexpected_key_behavior): ... + +class PivotWiderOptions(_PivotWiderOptions): + def __init__(self, key_names, *, unexpected_key_behavior='ignore'): ... + +class Expression(_Weakrefable): + def __init__(self): ... + def equals(self, other: Expression): ... + def __str__(self): ... + def __repr__(self): ... + @staticmethod + def from_substrait(message: object): ... + def to_substrait(self, schema: Schema, allow_arrow_extensions: c_bool=False): ... + @staticmethod + def _deserialize(buffer: Buffer): ... + def __reduce__(self): ... + @staticmethod + def _call(function_name: str, arguments: list[Any], options: FunctionOptions | None=None): ... + def __richcmp__(self, other, op: int): ... + def __bool__(self): ... + def __invert__(self): ... + def __and__(self: Expression, other): ... + def __or__(self: Expression, other): ... + def __add__(self: Expression, other): ... + def __mul__(self: Expression, other): ... + def __sub__(self: Expression, other): ... + def __truediv__(self: Expression, other): ... + def is_valid(self): ... + def is_null(self, nan_is_null: bool=False): ... + def is_nan(self): ... + def cast(self, type=None, safe=None, options=None): ... + def isin(self, values): ... + @staticmethod + def _field(name_or_idx): ... + @staticmethod + def _nested_field(names: tuple[Any, ...]): ... + @staticmethod + def _scalar(value): ... + +class UdfContext(_Weakrefable): + def __init__(self): ... + @property + def batch_length(self): ... + @property + def memory_pool(self): ... + +class RegisterUdf(_Weakrefable): ... + +def _pas(): ... +def _forbid_instantiation(klass, subclasses_instead=True): ... +def function_registry(): ... +def get_function(name): ... +def list_functions(): ... +def call_function(name, args, options=None, memory_pool=None, length=None): ... +def _raise_invalid_function_option(value, description, *, exception_class=ValueError): ... +def _skip_nulls_doc(): ... +def _min_count_doc(*, default): ... +def _get_udf_context(memory_pool, batch_length): ... +def register_scalar_function(func, function_name, function_doc, in_types, out_type, func_registry=None): ... +def register_vector_function(func, function_name, function_doc, in_types, out_type, func_registry=None): ... +def register_aggregate_function(func, function_name, function_doc, in_types, out_type, func_registry=None): ... +def register_tabular_function(func, function_name, function_doc, in_types, out_type, func_registry=None): ... +def _register_user_defined_function(register_func, func, function_name, function_doc, in_types, out_type, func_registry=None): ... +def call_tabular_function(function_name, args=None, func_registry=None): ... diff --git a/python/pyarrow-stubs/pyarrow/_csv.pyi b/python/pyarrow-stubs/pyarrow/_csv.pyi new file mode 100644 index 000000000000..80d81b7ff330 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_csv.pyi @@ -0,0 +1,219 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_csv.pyx + +from collections import namedtuple + +from pyarrow.includes.common import * +from pyarrow.includes.libarrow import * +from pyarrow.includes.libarrow_python import * +from pyarrow.lib import (MemoryPool, RecordBatchReader, Schema, + _CRecordBatchWriter, _Weakrefable) + +_InvalidRow = namedtuple('_InvalidRow', ('expected_columns', 'actual_columns', 'number', 'text'), module=__name__) +ISO8601 = _ISO8601() + +class InvalidRow(_InvalidRow): + __slots__ = () + +class ReadOptions(_Weakrefable): + __slots__ = () + encoding: object + + def __init__(self, *, use_threads=None, block_size=None, skip_rows=None, skip_rows_after_names=None, column_names=None, autogenerate_column_names=None, encoding='utf8'): ... + @property + def use_threads(self): ... + @use_threads.setter + def use_threads(self, value): ... + @property + def block_size(self): ... + @block_size.setter + def block_size(self, value): ... + @property + def skip_rows(self): ... + @skip_rows.setter + def skip_rows(self, value): ... + @property + def skip_rows_after_names(self): ... + @skip_rows_after_names.setter + def skip_rows_after_names(self, value): ... + @property + def column_names(self): ... + @column_names.setter + def column_names(self, value): ... + @property + def autogenerate_column_names(self): ... + @autogenerate_column_names.setter + def autogenerate_column_names(self, value): ... + def validate(self): ... + def equals(self, other: ReadOptions): ... + def __getstate__(self): ... + def __setstate__(self, state): ... + def __eq__(self, other): ... + def _repr_base(self): ... + def __repr__(self): ... + def __str__(self): ... + +class ParseOptions(_Weakrefable): + __slots__ = () + + def __init__(self, *, delimiter=None, quote_char=None, double_quote=None, escape_char=None, newlines_in_values=None, ignore_empty_lines=None, invalid_row_handler=None): ... + @property + def delimiter(self): ... + @delimiter.setter + def delimiter(self, value): ... + @property + def quote_char(self): ... + @quote_char.setter + def quote_char(self, value): ... + @property + def double_quote(self): ... + @double_quote.setter + def double_quote(self, value): ... + @property + def escape_char(self): ... + @escape_char.setter + def escape_char(self, value): ... + @property + def newlines_in_values(self): ... + @newlines_in_values.setter + def newlines_in_values(self, value): ... + @property + def ignore_empty_lines(self): ... + @property + def invalid_row_handler(self): ... + @invalid_row_handler.setter + def invalid_row_handler(self, value): ... + @ignore_empty_lines.setter + def ignore_empty_lines(self, value): ... + def validate(self): ... + def equals(self, other: ParseOptions): ... + def __getstate__(self): ... + def __setstate__(self, state): ... + def __eq__(self, other): ... + def _repr_base(self): ... + def __repr__(self): ... + def __str__(self): ... + +class _ISO8601(_Weakrefable): + __slots__ = () + + def __str__(self): ... + def __eq__(self, other): ... + +class ConvertOptions(_Weakrefable): + __slots__ = () + + def __init__(self, *, check_utf8=None, column_types=None, default_column_type=None, null_values=None, true_values=None, false_values=None, decimal_point=None, strings_can_be_null=None, quoted_strings_can_be_null=None, include_columns=None, include_missing_columns=None, auto_dict_encode=None, auto_dict_max_cardinality=None, timestamp_parsers=None): ... + @property + def check_utf8(self): ... + @check_utf8.setter + def check_utf8(self, value): ... + @property + def strings_can_be_null(self): ... + @strings_can_be_null.setter + def strings_can_be_null(self, value): ... + @property + def quoted_strings_can_be_null(self): ... + @quoted_strings_can_be_null.setter + def quoted_strings_can_be_null(self, value): ... + @property + def column_types(self): ... + @column_types.setter + def column_types(self, value): ... + @property + def default_column_type(self): ... + @default_column_type.setter + def default_column_type(self, value): ... + @property + def null_values(self): ... + @null_values.setter + def null_values(self, value): ... + @property + def true_values(self): ... + @true_values.setter + def true_values(self, value): ... + @property + def false_values(self): ... + @false_values.setter + def false_values(self, value): ... + @property + def decimal_point(self): ... + @decimal_point.setter + def decimal_point(self, value): ... + @property + def auto_dict_encode(self): ... + @auto_dict_encode.setter + def auto_dict_encode(self, value): ... + @property + def auto_dict_max_cardinality(self): ... + @auto_dict_max_cardinality.setter + def auto_dict_max_cardinality(self, value): ... + @property + def include_columns(self): ... + @include_columns.setter + def include_columns(self, value): ... + @property + def include_missing_columns(self): ... + @include_missing_columns.setter + def include_missing_columns(self, value): ... + @property + def timestamp_parsers(self): ... + @timestamp_parsers.setter + def timestamp_parsers(self, value): ... + def validate(self): ... + def equals(self, other: ConvertOptions): ... + def __getstate__(self): ... + def __setstate__(self, state): ... + def __eq__(self, other): ... + def _repr_base(self): ... + def __repr__(self): ... + def __str__(self): ... + +class CSVStreamingReader(RecordBatchReader): + schema: Schema + + def __init__(self): ... + +class WriteOptions(_Weakrefable): + __slots__ = () + + def __init__(self, *, include_header=None, batch_size=None, delimiter=None, eol=None, null_string=None, quoting_style=None, quoting_header=None): ... + @property + def include_header(self): ... + @include_header.setter + def include_header(self, value): ... + @property + def batch_size(self): ... + @batch_size.setter + def batch_size(self, value): ... + @property + def delimiter(self): ... + @delimiter.setter + def delimiter(self, value): ... + @property + def eol(self): ... + @eol.setter + def eol(self, value): ... + @property + def null_string(self): ... + @null_string.setter + def null_string(self, value): ... + @property + def quoting_style(self): ... + @quoting_style.setter + def quoting_style(self, value): ... + @property + def quoting_header(self): ... + @quoting_header.setter + def quoting_header(self, value): ... + def validate(self): ... + def _repr_base(self): ... + def __repr__(self): ... + def __str__(self): ... + +class CSVWriter(_CRecordBatchWriter): + def __init__(self, sink, schema: Schema, *, write_options: WriteOptions | None=None, memory_pool: MemoryPool | None=None): ... + +def read_csv(input_file, read_options=None, parse_options=None, convert_options=None, memory_pool: MemoryPool | None=None): ... +def open_csv(input_file, read_options=None, parse_options=None, convert_options=None, memory_pool: MemoryPool | None=None): ... +def _raise_invalid_function_option(value, description, *, exception_class=ValueError): ... +def write_csv(data, output_file, write_options=None, memory_pool: MemoryPool | None=None): ... diff --git a/python/pyarrow-stubs/pyarrow/_cuda.pyi b/python/pyarrow-stubs/pyarrow/_cuda.pyi new file mode 100644 index 000000000000..ee268f28786a --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_cuda.pyi @@ -0,0 +1,80 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_cuda.pyx + +from pyarrow.includes.common import * +from pyarrow.includes.libarrow import * +from pyarrow.includes.libarrow_cuda import * +from pyarrow.lib import * + + +class Context(_Weakrefable): + def __init__(self, *args, **kwargs): ... + @staticmethod + def from_numba(context=None): ... + def to_numba(self): ... + @staticmethod + def get_num_devices(): ... + @property + def device_number(self): ... + @property + def handle(self): ... + def synchronize(self): ... + @property + def bytes_allocated(self): ... + def get_device_address(self, address: int): ... + def new_buffer(self, nbytes: int): ... + @property + def memory_manager(self): ... + @property + def device(self): ... + def foreign_buffer(self, address, size, base=None): ... + def open_ipc_buffer(self, ipc_handle): ... + def buffer_from_data(self, data: object, offset: int=0, size: int=-1): ... + def buffer_from_object(self, obj): ... + +class IpcMemHandle(_Weakrefable): + @staticmethod + def from_buffer(opaque_handle: Buffer): ... + def serialize(self, pool=None): ... + +class CudaBuffer(Buffer): + def __init__(self): ... + @staticmethod + def from_buffer(buf): ... + @staticmethod + def from_numba(mem): ... + def to_numba(self): ... + def copy_to_host(self, position: int=0, nbytes: int=-1, buf: Buffer | None=None, memory_pool: MemoryPool | None=None, resizable: c_bool=False): ... + def copy_from_host(self, data, position: int=0, nbytes: int=-1): ... + def copy_from_device(self, buf, position: int=0, nbytes: int=-1): ... + def export_for_ipc(self): ... + @property + def context(self): ... + def slice(self, offset=0, length=None): ... + def to_pybytes(self): ... + def __getbuffer__(self, buffer: cp.Py_buffer, flags: int): ... + +class HostBuffer(Buffer): + def __init__(self): ... + @property + def size(self): ... + +class BufferReader(NativeFile): + def __init__(self, obj: CudaBuffer): ... + def read_buffer(self, nbytes=None): ... + +class BufferWriter(NativeFile): + def __init__(self, buffer: CudaBuffer): ... + def writeat(self, position: int, data: object): ... + def flush(self): ... + def seek(self, position: int, whence: int=0): ... + @property + def buffer_size(self): ... + @buffer_size.setter + def buffer_size(self, buffer_size: int): ... + @property + def num_bytes_buffered(self): ... + +def new_host_buffer(size: int, device: int=0): ... +def serialize_record_batch(batch: object, ctx: object): ... +def read_message(source: object, pool=None): ... +def read_record_batch(buffer: object, schema: object, *, dictionary_memo: DictionaryMemo | None=None, pool=None): ... diff --git a/python/pyarrow-stubs/pyarrow/_dataset.pyi b/python/pyarrow-stubs/pyarrow/_dataset.pyi new file mode 100644 index 000000000000..7b1e38cd8dd4 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_dataset.pyi @@ -0,0 +1,327 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_dataset.pyx + +import collections +from typing import Any + +from pyarrow._acero import ExecNodeOptions +from pyarrow._compute import Expression +from pyarrow._csv import (ConvertOptions, ParseOptions, ReadOptions, + WriteOptions) +from pyarrow._fs import FileSystem +from pyarrow._json import ParseOptions as JsonParseOptions +from pyarrow._json import ReadOptions as JsonReadOptions +from pyarrow.includes.common import * +from pyarrow.includes.libarrow_dataset import * +from pyarrow.lib import * + +_DEFAULT_BATCH_SIZE = 2 ** 17 +_DEFAULT_BATCH_READAHEAD = 16 +_DEFAULT_FRAGMENT_READAHEAD = 4 +_orc_fileformat = None +_orc_imported = False +_dataset_pq = False + +class Dataset(_Weakrefable): + _scan_options: dict[Any, Any] + + def __init__(self): ... + @property + def partition_expression(self): ... + def replace_schema(self, schema: Schema): ... + def get_fragments(self, filter: Expression | None=None): ... + def _get_fragments(self, filter: Expression): ... + def _scanner_options(self, options): ... + def scanner(self, columns: object | None=None, filter: object | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + def to_batches(self, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + def to_table(self, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + def take(self, indices: object, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + def head(self, num_rows: int, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + def count_rows(self, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + @property + def schema(self): ... + def filter(self, expression): ... + def sort_by(self, sorting, **kwargs): ... + def join(self, right_dataset, keys, right_keys=None, join_type='left outer', left_suffix=None, right_suffix=None, coalesce_keys=True, use_threads=True): ... + def join_asof(self, right_dataset, on, by, tolerance, right_on=None, right_by=None): ... + +class InMemoryDataset(Dataset): + def __init__(self, source, schema: Schema | None=None): ... + +class UnionDataset(Dataset): + def __init__(self, schema: Schema, children): ... + def __reduce__(self): ... + @property + def children(self): ... + +class FileSystemDataset(Dataset): + def __init__(self, fragments, schema: Schema, format: FileFormat, filesystem: FileSystem | None=None, root_partition=None): ... + @property + def filesystem(self): ... + @property + def partitioning(self): ... + def __reduce__(self): ... + @classmethod + def from_paths(cls, paths, schema=None, format=None, filesystem=None, partitions=None, root_partition=None): ... + @property + def files(self): ... + @property + def format(self): ... + +class FileWriteOptions(_Weakrefable): + def __init__(self): ... + @property + def format(self): ... + +class FileFormat(_Weakrefable): + def __init__(self): ... + def inspect(self, file, filesystem=None): ... + def make_fragment(self, file, filesystem=None, partition_expression: Expression | None=None, *, file_size=None): ... + def make_write_options(self): ... + @property + def default_extname(self): ... + @property + def default_fragment_scan_options(self): ... + @default_fragment_scan_options.setter + def default_fragment_scan_options(self, options: FragmentScanOptions): ... + def __eq__(self, other): ... + +class Fragment(_Weakrefable): + def __init__(self): ... + @property + def physical_schema(self): ... + @property + def partition_expression(self): ... + def scanner(self, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + def to_batches(self, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + def to_table(self, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + def take(self, indices: object, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + def head(self, num_rows: int, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + def count_rows(self, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + +class FileFragment(Fragment): + def __repr__(self): ... + def __reduce__(self): ... + def open(self): ... + @property + def path(self): ... + @property + def filesystem(self): ... + @property + def buffer(self): ... + @property + def format(self): ... + +class FragmentScanOptions(_Weakrefable): + def __init__(self): ... + @property + def type_name(self): ... + def __eq__(self, other): ... + +class IpcFileWriteOptions(FileWriteOptions): + def __init__(self): ... + @property + def write_options(self): ... + @write_options.setter + def write_options(self, write_options: IpcWriteOptions): ... + +class IpcFileFormat(FileFormat): + def __init__(self): ... + def equals(self, other: IpcFileFormat): ... + def make_write_options(self, **kwargs): ... + @property + def default_extname(self): ... + def __reduce__(self): ... + +class FeatherFileFormat(IpcFileFormat): + @property + def default_extname(self): ... + +class CsvFileFormat(FileFormat): + __slots__ = () + _read_options_py: ReadOptions + + def __init__(self, parse_options: ParseOptions | None=None, default_fragment_scan_options=None, convert_options: ConvertOptions | None=None, read_options: ReadOptions | None=None): ... + def make_write_options(self, **kwargs): ... + @property + def parse_options(self): ... + @parse_options.setter + def parse_options(self, parse_options: ParseOptions): ... + def equals(self, other: CsvFileFormat): ... + def __reduce__(self): ... + def __repr__(self): ... + +class CsvFragmentScanOptions(FragmentScanOptions): + __slots__ = () + + def __init__(self, convert_options: ConvertOptions | None=None, read_options: ReadOptions | None=None): ... + @property + def convert_options(self): ... + @convert_options.setter + def convert_options(self, convert_options: ConvertOptions): ... + @property + def read_options(self): ... + @read_options.setter + def read_options(self, read_options: ReadOptions): ... + def equals(self, other: CsvFragmentScanOptions): ... + def __reduce__(self): ... + +class CsvFileWriteOptions(FileWriteOptions): + def __init__(self): ... + @property + def write_options(self): ... + @write_options.setter + def write_options(self, write_options: WriteOptions): ... + +class JsonFileFormat(FileFormat): + __slots__ = () + + def __init__(self, default_fragment_scan_options=None, parse_options: JsonParseOptions | None=None, read_options: JsonReadOptions | None=None): ... + def equals(self, other: JsonFileFormat): ... + def __reduce__(self): ... + def __repr__(self): ... + +class JsonFragmentScanOptions(FragmentScanOptions): + __slots__ = () + + def __init__(self, parse_options: JsonParseOptions | None=None, read_options: JsonReadOptions | None=None): ... + @property + def parse_options(self): ... + @parse_options.setter + def parse_options(self, parse_options: JsonParseOptions): ... + @property + def read_options(self): ... + @read_options.setter + def read_options(self, read_options: JsonReadOptions): ... + def equals(self, other: JsonFragmentScanOptions): ... + def __reduce__(self): ... + +class Partitioning(_Weakrefable): + def __init__(self): ... + def __eq__(self, other): ... + def parse(self, path): ... + def format(self, expr): ... + @property + def schema(self): ... + +class PartitioningFactory(_Weakrefable): + def __init__(self): ... + def __reduce__(self): ... + @property + def type_name(self): ... + +class KeyValuePartitioning(Partitioning): + def __init__(self): ... + def __reduce__(self): ... + @property + def dictionaries(self): ... + +class DirectoryPartitioning(KeyValuePartitioning): + def __init__(self, schema: Schema, dictionaries=None, segment_encoding='uri'): ... + @staticmethod + def discover(field_names=None, infer_dictionary=False, max_partition_dictionary_size=0, schema=None, segment_encoding='uri'): ... + +class HivePartitioning(KeyValuePartitioning): + def __init__(self, schema: Schema, dictionaries=None, null_fallback='__HIVE_DEFAULT_PARTITION__', segment_encoding='uri'): ... + def __reduce__(self): ... + @staticmethod + def discover(infer_dictionary=False, max_partition_dictionary_size=0, null_fallback='__HIVE_DEFAULT_PARTITION__', schema=None, segment_encoding='uri'): ... + +class FilenamePartitioning(KeyValuePartitioning): + def __init__(self, schema: Schema, dictionaries=None, segment_encoding='uri'): ... + @staticmethod + def discover(field_names=None, infer_dictionary=False, schema=None, segment_encoding='uri'): ... + +class DatasetFactory(_Weakrefable): + def __init__(self): ... + @property + def root_partition(self): ... + @root_partition.setter + def root_partition(self, expr: Expression): ... + def inspect_schemas(self): ... + def inspect(self, *, promote_options='default', fragments=None): ... + def finish(self, schema: Schema | None=None): ... + +class FileSystemFactoryOptions(_Weakrefable): + __slots__ = () + + def __init__(self, partition_base_dir=None, partitioning=None, exclude_invalid_files=None, selector_ignore_prefixes: list[Any] | None=None): ... + @property + def partitioning(self): ... + @partitioning.setter + def partitioning(self, value: Partitioning): ... + @property + def partitioning_factory(self): ... + @partitioning_factory.setter + def partitioning_factory(self, value: PartitioningFactory): ... + @property + def partition_base_dir(self): ... + @partition_base_dir.setter + def partition_base_dir(self, value): ... + @property + def exclude_invalid_files(self): ... + @exclude_invalid_files.setter + def exclude_invalid_files(self, value: bool): ... + @property + def selector_ignore_prefixes(self): ... + @selector_ignore_prefixes.setter + def selector_ignore_prefixes(self, values): ... + +class FileSystemDatasetFactory(DatasetFactory): + def __init__(self, filesystem: FileSystem, paths_or_selector, format: FileFormat, options: FileSystemFactoryOptions | None=None): ... + +class UnionDatasetFactory(DatasetFactory): + def __init__(self, factories: list[Any]): ... + +class RecordBatchIterator(_Weakrefable): + def __init__(self): ... + def __iter__(self): ... + def __next__(self): ... + +class TaggedRecordBatch(collections.namedtuple('TaggedRecordBatch', ['record_batch', 'fragment'])): ... + +class TaggedRecordBatchIterator(_Weakrefable): + def __init__(self): ... + def __iter__(self): ... + def __next__(self): ... + +class Scanner(_Weakrefable): + def __init__(self): ... + @staticmethod + def from_dataset(dataset: Dataset, *, columns: object | None=None, filter: object | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + @staticmethod + def from_fragment(fragment: Fragment, *, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + @staticmethod + def from_batches(source, *, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + @property + def dataset_schema(self): ... + @property + def projected_schema(self): ... + def to_batches(self): ... + def scan_batches(self): ... + def to_table(self): ... + def take(self, indices: object): ... + def head(self, num_rows: int): ... + def count_rows(self): ... + def to_reader(self): ... + +class WrittenFile(_Weakrefable): + path: str + metadata: object + size: int + + def __init__(self, path, metadata, size): ... + +class _ScanNodeOptions(ExecNodeOptions): + def _set_options(self, dataset: Dataset, scan_options: dict[Any, Any]): ... + +class ScanNodeOptions(_ScanNodeOptions): + def __init__(self, dataset: Dataset, **kwargs): ... + +def _get_orc_fileformat(): ... +def _get_parquet_classes(): ... +def _get_parquet_symbol(name): ... +def _constructor_directory_partitioning_factory(*args): ... +def _constructor_hive_partitioning_factory(*args): ... +def _constructor_filename_partitioning_factory(*args): ... +def get_partition_keys(partition_expression: Expression): ... +def _filesystemdataset_write(data: Scanner, base_dir: object, basename_template: str, filesystem: FileSystem, partitioning: Partitioning, preserve_order: bool, file_options: FileWriteOptions, max_partitions: int, file_visitor: object, existing_data_behavior: str, max_open_files: int, max_rows_per_file: int, min_rows_per_group: int, max_rows_per_group: int, create_dir: bool): ... diff --git a/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi b/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi new file mode 100644 index 000000000000..e64547c8a965 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi @@ -0,0 +1,14 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_dataset_orc.pyx + +from pyarrow._dataset import FileFormat +from pyarrow.includes.libarrow import * +from pyarrow.includes.libarrow_dataset import * +from pyarrow.lib import * + + +class OrcFileFormat(FileFormat): + def __init__(self): ... + def equals(self, other: OrcFileFormat): ... + @property + def default_extname(self): ... + def __reduce__(self): ... diff --git a/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi b/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi new file mode 100644 index 000000000000..cb84f890be0f --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi @@ -0,0 +1,159 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_dataset_parquet.pyx + +from typing import Any + +from pyarrow._compute import Expression +from pyarrow._dataset import (CacheOptions, DatasetFactory, FileFormat, + FileFragment, FileWriteOptions, + FragmentScanOptions, Partitioning, + PartitioningFactory) +from pyarrow._fs import FileSystem +from pyarrow.includes.libarrow import * +from pyarrow.includes.libarrow_dataset import * +from pyarrow.includes.libarrow_dataset_parquet import * +from pyarrow.lib import * +from typing_extensions import TypeAlias + +_CParquetFileWriterPtr: TypeAlias = CParquetFileWriter + +class ParquetFileFormat(FileFormat): + def __init__(self, read_options=None, default_fragment_scan_options=None, **kwargs): ... + @property + def read_options(self): ... + def make_write_options(self, **kwargs): ... + def equals(self, other: ParquetFileFormat): ... + @property + def default_extname(self): ... + def __reduce__(self): ... + def __repr__(self): ... + def make_fragment(self, file, filesystem=None, partition_expression: Expression | None=None, row_groups=None, *, file_size=None): ... + +class RowGroupInfo: + def __init__(self, id, metadata, schema): ... + @property + def num_rows(self): ... + @property + def total_byte_size(self): ... + @property + def statistics(self): ... + def __repr__(self): ... + def __eq__(self, other): ... + +class ParquetFileFragment(FileFragment): + def __reduce__(self): ... + def ensure_complete_metadata(self): ... + @property + def row_groups(self): ... + @property + def metadata(self): ... + @property + def num_row_groups(self): ... + def split_by_row_group(self, filter: Expression | None=None, schema: Schema | None=None): ... + def subset(self, filter: Expression | None=None, schema: Schema | None=None, row_group_ids: object | None=None): ... + +class ParquetReadOptions(_Weakrefable): + dictionary_columns: set[Any] + _coerce_int96_timestamp_unit: TimeUnit + _binary_type: Type + _list_type: Type + + def __init__(self, dictionary_columns=None, coerce_int96_timestamp_unit=None, binary_type=None, list_type=None): ... + @property + def binary_type(self): ... + @binary_type.setter + def binary_type(self, ty): ... + @property + def list_type(self): ... + @list_type.setter + def list_type(self, ty): ... + @property + def coerce_int96_timestamp_unit(self): ... + @coerce_int96_timestamp_unit.setter + def coerce_int96_timestamp_unit(self, unit): ... + def equals(self, other: ParquetReadOptions): ... + def __eq__(self, other): ... + def __repr__(self): ... + +class ParquetFileWriteOptions(FileWriteOptions): + def update(self, **kwargs): ... + def _set_properties(self): ... + def _set_arrow_properties(self): ... + def _set_encryption_config(self): ... + def __repr__(self): ... + +class ParquetFragmentScanOptions(FragmentScanOptions): + __slots__ = () + + def __init__(self, *, use_buffered_stream: bool=False, buffer_size=8192, pre_buffer: bool=True, cache_options=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, decryption_config=None, decryption_properties=None, page_checksum_verification: bool=False, arrow_extensions_enabled: bool=False): ... + @property + def use_buffered_stream(self): ... + @use_buffered_stream.setter + def use_buffered_stream(self, use_buffered_stream: bool): ... + @property + def buffer_size(self): ... + @buffer_size.setter + def buffer_size(self, buffer_size): ... + @property + def pre_buffer(self): ... + @pre_buffer.setter + def pre_buffer(self, pre_buffer: bool): ... + @property + def cache_options(self): ... + @cache_options.setter + def cache_options(self, options: CacheOptions): ... + @property + def thrift_string_size_limit(self): ... + @thrift_string_size_limit.setter + def thrift_string_size_limit(self, size): ... + @property + def thrift_container_size_limit(self): ... + @thrift_container_size_limit.setter + def thrift_container_size_limit(self, size): ... + @property + def schema_depth_limit(self): ... + @schema_depth_limit.setter + def schema_depth_limit(self, limit): ... + @property + def decryption_properties(self): ... + @decryption_properties.setter + def decryption_properties(self, config): ... + @property + def parquet_decryption_config(self): ... + @parquet_decryption_config.setter + def parquet_decryption_config(self, config): ... + @property + def page_checksum_verification(self): ... + @page_checksum_verification.setter + def page_checksum_verification(self, page_checksum_verification: bool): ... + @property + def arrow_extensions_enabled(self): ... + @arrow_extensions_enabled.setter + def arrow_extensions_enabled(self, arrow_extensions_enabled: bool): ... + def equals(self, other: ParquetFragmentScanOptions): ... + @staticmethod + def _reconstruct(kwargs): ... + def __reduce__(self): ... + +class ParquetFactoryOptions(_Weakrefable): + __slots__ = () + + def __init__(self, partition_base_dir=None, partitioning=None, validate_column_chunk_paths=False): ... + @property + def partitioning(self): ... + @partitioning.setter + def partitioning(self, value: Partitioning): ... + @property + def partitioning_factory(self): ... + @partitioning_factory.setter + def partitioning_factory(self, value: PartitioningFactory): ... + @property + def partition_base_dir(self): ... + @partition_base_dir.setter + def partition_base_dir(self, value): ... + @property + def validate_column_chunk_paths(self): ... + @validate_column_chunk_paths.setter + def validate_column_chunk_paths(self, value): ... + +class ParquetDatasetFactory(DatasetFactory): + def __init__(self, metadata_path, filesystem: FileSystem, format: FileFormat, options: ParquetFactoryOptions | None=None): ... diff --git a/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi b/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi new file mode 100644 index 000000000000..5f863e9deb3c --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi @@ -0,0 +1,21 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_dataset_parquet_encryption.pyx + +from pyarrow._dataset_parquet import (ParquetFileWriteOptions, + ParquetFragmentScanOptions) +from pyarrow._parquet_encryption import * +from pyarrow.includes.libarrow_dataset_parquet import * + + +class ParquetEncryptionConfig(_Weakrefable): + __slots__ = () + + def __init__(self, crypto_factory: CryptoFactory, kms_connection_config: KmsConnectionConfig, encryption_config: EncryptionConfiguration): ... + +class ParquetDecryptionConfig(_Weakrefable): + __slots__ = () + + def __init__(self, crypto_factory: CryptoFactory, kms_connection_config: KmsConnectionConfig, decryption_config: DecryptionConfiguration): ... + +def set_encryption_config(opts: ParquetFileWriteOptions, config: ParquetEncryptionConfig): ... +def set_decryption_properties(opts: ParquetFragmentScanOptions, config: FileDecryptionProperties): ... +def set_decryption_config(opts: ParquetFragmentScanOptions, config: ParquetDecryptionConfig): ... diff --git a/python/pyarrow-stubs/pyarrow/_feather.pyi b/python/pyarrow-stubs/pyarrow/_feather.pyi new file mode 100644 index 000000000000..24cbe0124462 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_feather.pyi @@ -0,0 +1,19 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_feather.pyx + +from pyarrow.includes.common import * +from pyarrow.includes.libarrow import * +from pyarrow.includes.libarrow_feather import * +from pyarrow.lib import Table, _Weakrefable + + +class FeatherError(Exception): ... + +class FeatherReader(_Weakrefable): + def __init__(self, source, use_memory_map: c_bool, use_threads: c_bool): ... + @property + def version(self): ... + def read(self): ... + def read_indices(self, indices): ... + def read_names(self, names): ... + +def write_feather(table: Table, dest: object, compression=None, compression_level=None, chunksize=None, version=2): ... diff --git a/python/pyarrow-stubs/pyarrow/_flight.pyi b/python/pyarrow-stubs/pyarrow/_flight.pyi new file mode 100644 index 000000000000..6415d2305353 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_flight.pyi @@ -0,0 +1,362 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_flight.pyx + +import collections +import enum +import re +from typing import Any + +from pyarrow.includes.libarrow_flight import * +from pyarrow.ipc import _ReadPandasMixin +from pyarrow.lib import * +from pyarrow.lib import ArrowCancelled, ArrowException, ArrowInvalid + +_FLIGHT_SERVER_ERROR_REGEX = re.compile('Flight RPC failed with message: (.*). Detail: Python exception: (.*)', re.DOTALL) +_CertKeyPair = collections.namedtuple('_CertKeyPair', ['cert', 'key']) +_ActionType = collections.namedtuple('_ActionType', ['type', 'description']) +_CallInfo = collections.namedtuple('_CallInfo', ['method']) + +class FlightCallOptions(_Weakrefable): + def __init__(self, timeout=None, write_options=None, headers=None, read_options: IpcReadOptions | None=None): ... + @property + def timeout(self): ... + @property + def headers(self): ... + @property + def read_options(self): ... + @property + def write_options(self): ... + def __repr__(self): ... + +class CertKeyPair(_CertKeyPair): ... + +class FlightError(Exception): + def __init__(self, message='', extra_info=b''): ... + +class FlightInternalError(FlightError, ArrowException): ... + +class FlightTimedOutError(FlightError, ArrowException): ... + +class FlightCancelledError(FlightError, ArrowCancelled): ... + +class FlightServerError(FlightError, ArrowException): ... + +class FlightUnauthenticatedError(FlightError, ArrowException): ... + +class FlightUnauthorizedError(FlightError, ArrowException): ... + +class FlightUnavailableError(FlightError, ArrowException): ... + +class FlightWriteSizeExceededError(ArrowInvalid): + def __init__(self, message, limit, actual): ... + +class Action(_Weakrefable): + def __init__(self, action_type, buf): ... + @property + def type(self): ... + @property + def body(self): ... + def serialize(self): ... + @classmethod + def deserialize(cls, serialized): ... + def __eq__(self, other: Action): ... + def __repr__(self): ... + +class ActionType(_ActionType): + def make_action(self, buf): ... + +class Result(_Weakrefable): + def __init__(self, buf): ... + @property + def body(self): ... + def serialize(self): ... + @classmethod + def deserialize(cls, serialized): ... + def __eq__(self, other: Result): ... + def __repr__(self): ... + +class BasicAuth(_Weakrefable): + def __init__(self, username=None, password=None): ... + @property + def username(self): ... + @property + def password(self): ... + @staticmethod + def deserialize(serialized): ... + def serialize(self): ... + def __eq__(self, other: BasicAuth): ... + def __repr__(self): ... + +class DescriptorType(enum.Enum): + UNKNOWN = 0 + PATH = 1 + CMD = 2 + +class FlightMethod(enum.Enum): + INVALID = 0 + HANDSHAKE = 1 + LIST_FLIGHTS = 2 + GET_FLIGHT_INFO = 3 + GET_SCHEMA = 4 + DO_GET = 5 + DO_PUT = 6 + DO_ACTION = 7 + LIST_ACTIONS = 8 + DO_EXCHANGE = 9 + +class FlightDescriptor(_Weakrefable): + def __init__(self): ... + @staticmethod + def for_path(*path): ... + @staticmethod + def for_command(command): ... + @property + def descriptor_type(self): ... + @property + def command(self): ... + @property + def path(self): ... + def __repr__(self): ... + def serialize(self): ... + @classmethod + def deserialize(cls, serialized): ... + def __eq__(self, other: FlightDescriptor): ... + +class Ticket(_Weakrefable): + def __init__(self, ticket): ... + @property + def ticket(self): ... + def serialize(self): ... + @classmethod + def deserialize(cls, serialized): ... + def __eq__(self, other: Ticket): ... + def __repr__(self): ... + +class Location(_Weakrefable): + def __init__(self, uri): ... + def __repr__(self): ... + @property + def uri(self): ... + def equals(self, other: Location): ... + def __eq__(self, other): ... + @staticmethod + def for_grpc_tcp(host, port): ... + @staticmethod + def for_grpc_tls(host, port): ... + @staticmethod + def for_grpc_unix(path): ... + +class FlightEndpoint(_Weakrefable): + def __init__(self, ticket, locations, expiration_time=None, app_metadata=''): ... + @property + def ticket(self): ... + @property + def locations(self): ... + @property + def expiration_time(self): ... + @property + def app_metadata(self): ... + def serialize(self): ... + @classmethod + def deserialize(cls, serialized): ... + def __repr__(self): ... + def __eq__(self, other: FlightEndpoint): ... + +class SchemaResult(_Weakrefable): + def __init__(self, schema: Schema): ... + @property + def schema(self): ... + def serialize(self): ... + @classmethod + def deserialize(cls, serialized): ... + def __eq__(self, other: SchemaResult): ... + def __repr__(self): ... + +class FlightInfo(_Weakrefable): + def __init__(self, schema: Schema, descriptor: FlightDescriptor, endpoints, total_records=None, total_bytes=None, ordered=False, app_metadata=''): ... + @property + def total_records(self): ... + @property + def total_bytes(self): ... + @property + def ordered(self): ... + @property + def app_metadata(self): ... + @property + def schema(self): ... + @property + def descriptor(self): ... + @property + def endpoints(self): ... + def serialize(self): ... + @classmethod + def deserialize(cls, serialized): ... + def __eq__(self, other: FlightInfo): ... + def __repr__(self): ... + +class FlightStreamChunk(_Weakrefable): + @property + def data(self): ... + @property + def app_metadata(self): ... + def __iter__(self): ... + def __repr__(self): ... + +class _MetadataRecordBatchReader(_Weakrefable, _ReadPandasMixin): + def __iter__(self): ... + def __next__(self): ... + @property + def schema(self): ... + def read_all(self): ... + def read_chunk(self): ... + def to_reader(self): ... + @property + def stats(self): ... + +class MetadataRecordBatchReader(_MetadataRecordBatchReader): ... + +class FlightStreamReader(MetadataRecordBatchReader): + def cancel(self): ... + def read_all(self): ... + +class MetadataRecordBatchWriter(_CRecordBatchWriter): + def begin(self, schema: Schema, options=None): ... + def write_metadata(self, buf): ... + def write_batch(self, batch: RecordBatch): ... + def write_table(self, table: Table, max_chunksize=None, **kwargs): ... + def close(self): ... + def write_with_metadata(self, batch: RecordBatch, buf): ... + +class FlightStreamWriter(MetadataRecordBatchWriter): + def done_writing(self): ... + +class FlightMetadataReader(_Weakrefable): + def read(self): ... + +class FlightMetadataWriter(_Weakrefable): + def write(self, message): ... + +class AsyncioCall: + def __init__(self) -> None: ... + def as_awaitable(self) -> object: ... + def wakeup(self, result_or_exception) -> None: ... + +class AsyncioFlightClient: + def __init__(self, client: FlightClient) -> None: ... + async def get_flight_info(self, descriptor: FlightDescriptor, *, options: FlightCallOptions | None=None): ... + +class FlightClient(_Weakrefable): + def __init__(self, location, *, tls_root_certs=None, cert_chain=None, private_key=None, override_hostname=None, middleware=None, write_size_limit_bytes=None, disable_server_verification=None, generic_options=None): ... + @property + def supports_async(self): ... + def as_async(self) -> None: ... + def wait_for_available(self, timeout=5): ... + @classmethod + def connect(cls, location, tls_root_certs=None, cert_chain=None, private_key=None, override_hostname=None, disable_server_verification=None): ... + def authenticate(self, auth_handler, options: FlightCallOptions | None=None): ... + def authenticate_basic_token(self, username, password, options: FlightCallOptions | None=None): ... + def list_actions(self, options: FlightCallOptions | None=None): ... + def do_action(self, action, options: FlightCallOptions | None=None): ... + def list_flights(self, criteria: bytes | None=None, options: FlightCallOptions | None=None): ... + def get_flight_info(self, descriptor: FlightDescriptor, options: FlightCallOptions | None=None): ... + def get_schema(self, descriptor: FlightDescriptor, options: FlightCallOptions | None=None): ... + def do_get(self, ticket: Ticket, options: FlightCallOptions | None=None): ... + def do_put(self, descriptor: FlightDescriptor, schema: Schema, options: FlightCallOptions | None=None): ... + def do_exchange(self, descriptor: FlightDescriptor, options: FlightCallOptions | None=None): ... + def close(self): ... + def __del__(self): ... + def __enter__(self): ... + def __exit__(self, exc_type, exc_value, traceback): ... + +class FlightDataStream(_Weakrefable): ... + +class RecordBatchStream(FlightDataStream): + def __init__(self, data_source, options=None): ... + +class GeneratorStream(FlightDataStream): + def __init__(self, schema, generator, options=None): ... + +class ServerCallContext(_Weakrefable): + def peer_identity(self): ... + def peer(self): ... + def is_cancelled(self): ... + def add_header(self, key, value): ... + def add_trailer(self, key, value): ... + def get_middleware(self, key): ... + +class ServerAuthReader(_Weakrefable): + def read(self): ... + +class ServerAuthSender(_Weakrefable): + def write(self, message): ... + +class ClientAuthReader(_Weakrefable): + def read(self): ... + +class ClientAuthSender(_Weakrefable): + def write(self, message): ... + +class ServerAuthHandler(_Weakrefable): + def authenticate(self, outgoing, incoming): ... + def is_valid(self, token): ... + +class ClientAuthHandler(_Weakrefable): + def authenticate(self, outgoing, incoming): ... + def get_token(self): ... + +class CallInfo(_CallInfo): ... + +class ClientMiddlewareFactory(_Weakrefable): + def start_call(self, info): ... + +class ClientMiddleware(_Weakrefable): + def sending_headers(self): ... + def received_headers(self, headers): ... + def call_completed(self, exception): ... + +class ServerMiddlewareFactory(_Weakrefable): + def start_call(self, info, headers): ... + +class TracingServerMiddlewareFactory(ServerMiddlewareFactory): ... + +class ServerMiddleware(_Weakrefable): + def sending_headers(self): ... + def call_completed(self, exception): ... + +class TracingServerMiddleware(ServerMiddleware): + __slots__ = ['trace_context'] + + def __init__(self, trace_context): ... + +class _ServerMiddlewareFactoryWrapper(ServerMiddlewareFactory): + def __init__(self, factories: dict[Any, Any]): ... + def start_call(self, info, headers): ... + +class _ServerMiddlewareWrapper(ServerMiddleware): + def __init__(self, middleware: dict[Any, Any]): ... + def sending_headers(self): ... + def call_completed(self, exception): ... + +class _FlightServerFinalizer(_Weakrefable): + def finalize(self): ... + +class FlightServerBase(_Weakrefable): + def __init__(self, location=None, auth_handler=None, tls_certificates=None, verify_client=None, root_certificates=None, middleware=None): ... + @property + def port(self): ... + def list_flights(self, context, criteria): ... + def get_flight_info(self, context, descriptor): ... + def get_schema(self, context, descriptor): ... + def do_put(self, context, descriptor, reader: MetadataRecordBatchReader, writer: FlightMetadataWriter): ... + def do_get(self, context, ticket): ... + def do_exchange(self, context, descriptor, reader, writer): ... + def list_actions(self, context): ... + def do_action(self, context, action): ... + def serve(self): ... + def run(self): ... + def shutdown(self): ... + def wait(self): ... + def __enter__(self): ... + def __exit__(self, exc_type, exc_value, traceback): ... + +def _munge_grpc_python_error(message): ... +def connect(location, **kwargs): ... diff --git a/python/pyarrow-stubs/pyarrow/_fs.pyi b/python/pyarrow-stubs/pyarrow/_fs.pyi new file mode 100644 index 000000000000..c5e539a5c3f4 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_fs.pyi @@ -0,0 +1,139 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_fs.pyx + +from abc import ABC, abstractmethod +from datetime import datetime +from enum import IntEnum + +from pyarrow.includes.common import * +from pyarrow.includes.libarrow_fs import * +from pyarrow.lib import * + + +class FileType(IntEnum): + NotFound = ... + Unknown = ... + File = ... + Directory = ... + +class FileInfo(_Weakrefable): + def __init__(self, path, type: FileType=FileType.Unknown, *, mtime=None, mtime_ns=None, size=None): ... + def __repr__(self): ... + @property + def type(self): ... + @property + def is_file(self): ... + @property + def path(self): ... + @property + def base_name(self): ... + @property + def size(self): ... + @property + def extension(self): ... + @property + def mtime(self): ... + @property + def mtime_ns(self): ... + +class FileSelector(_Weakrefable): + def __init__(self, base_dir, allow_not_found: bool=False, recursive: bool=False): ... + @property + def base_dir(self): ... + @base_dir.setter + def base_dir(self, base_dir): ... + @property + def allow_not_found(self): ... + @allow_not_found.setter + def allow_not_found(self, allow_not_found: bool): ... + @property + def recursive(self): ... + @recursive.setter + def recursive(self, recursive: bool): ... + def __repr__(self): ... + +class FileSystem(_Weakrefable): + def __init__(self): ... + @staticmethod + def _from_uri(uri): ... + @staticmethod + def _fsspec_from_uri(uri): ... + @staticmethod + def _native_from_uri(uri): ... + @staticmethod + def from_uri(uri): ... + def equals(self, other: FileSystem): ... + def __eq__(self, other): ... + @property + def type_name(self): ... + def get_file_info(self, paths_or_selector): ... + def create_dir(self, path, *, recursive: bool=True): ... + def delete_dir(self, path): ... + def delete_dir_contents(self, path, *, accept_root_dir: bool=False, missing_dir_ok: bool=False): ... + def move(self, src, dest): ... + def copy_file(self, src, dest): ... + def delete_file(self, path): ... + def _wrap_input_stream(self, stream, path, compression, buffer_size): ... + def _wrap_output_stream(self, stream, path, compression, buffer_size): ... + def open_input_file(self, path): ... + def open_input_stream(self, path, compression='detect', buffer_size=None): ... + def open_output_stream(self, path, compression='detect', buffer_size=None, metadata=None): ... + def open_append_stream(self, path, compression='detect', buffer_size=None, metadata=None): ... + def normalize_path(self, path): ... + +class LocalFileSystem(FileSystem): + def __init__(self, *, use_mmap=False): ... + def __reduce__(self): ... + +class SubTreeFileSystem(FileSystem): + def __init__(self, base_path, base_fs: FileSystem): ... + def __repr__(self): ... + def __reduce__(self): ... + @property + def base_path(self): ... + @property + def base_fs(self): ... + +class _MockFileSystem(FileSystem): + def __init__(self, current_time: datetime | None=None): ... + +class PyFileSystem(FileSystem): + def __init__(self, handler): ... + @property + def handler(self): ... + def __reduce__(self): ... + +class FileSystemHandler(ABC): + @abstractmethod + def get_type_name(self): ... + @abstractmethod + def get_file_info(self, paths): ... + @abstractmethod + def get_file_info_selector(self, selector): ... + @abstractmethod + def create_dir(self, path, recursive): ... + @abstractmethod + def delete_dir(self, path): ... + @abstractmethod + def delete_dir_contents(self, path, missing_dir_ok=False): ... + @abstractmethod + def delete_root_dir_contents(self): ... + @abstractmethod + def delete_file(self, path): ... + @abstractmethod + def move(self, src, dest): ... + @abstractmethod + def copy_file(self, src, dest): ... + @abstractmethod + def open_input_stream(self, path): ... + @abstractmethod + def open_input_file(self, path): ... + @abstractmethod + def open_output_stream(self, path, metadata): ... + @abstractmethod + def open_append_stream(self, path, metadata): ... + @abstractmethod + def normalize_path(self, path): ... + +def _file_type_to_string(ty): ... +def _copy_files(source_fs: FileSystem, source_path: str, destination_fs: FileSystem, destination_path: str, chunk_size: int, use_threads: c_bool): ... +def _copy_files_selector(source_fs: FileSystem, source_sel: FileSelector, destination_fs: FileSystem, destination_base_dir: str, chunk_size: int, use_threads: c_bool): ... diff --git a/python/pyarrow-stubs/pyarrow/_gcsfs.pyi b/python/pyarrow-stubs/pyarrow/_gcsfs.pyi new file mode 100644 index 000000000000..53423286ed97 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_gcsfs.pyi @@ -0,0 +1,18 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_gcsfs.pyx + +from pyarrow._fs import FileSystem +from pyarrow.includes.common import * +from pyarrow.includes.libarrow import * +from pyarrow.includes.libarrow_fs import * + + +class GcsFileSystem(FileSystem): + def __init__(self, *, anonymous: bool=False, access_token=None, target_service_account=None, credential_token_expiration=None, default_bucket_location='US', scheme=None, endpoint_override=None, default_metadata=None, retry_time_limit=None, project_id=None): ... + def _expiration_datetime_from_options(self): ... + @staticmethod + def _reconstruct(kwargs): ... + def __reduce__(self): ... + @property + def default_bucket_location(self): ... + @property + def project_id(self): ... diff --git a/python/pyarrow-stubs/pyarrow/_hdfs.pyi b/python/pyarrow-stubs/pyarrow/_hdfs.pyi new file mode 100644 index 000000000000..0aba678d6025 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_hdfs.pyi @@ -0,0 +1,17 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_hdfs.pyx + +from pyarrow._fs import FileSystem +from pyarrow.includes.common import * +from pyarrow.includes.libarrow import * +from pyarrow.includes.libarrow_fs import * + + +class HadoopFileSystem(FileSystem): + def __init__(self, host: str, port: int=8020, *, user: str | None=None, replication: int=3, buffer_size: int=0, default_block_size=None, kerb_ticket=None, extra_conf=None): ... + @staticmethod + def from_uri(uri): ... + @staticmethod + def _reconstruct(kwargs): ... + def __reduce__(self): ... + +def _have_libhdfs(): ... diff --git a/python/pyarrow-stubs/pyarrow/_json.pyi b/python/pyarrow-stubs/pyarrow/_json.pyi new file mode 100644 index 000000000000..4a00403ed74c --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_json.pyi @@ -0,0 +1,55 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_json.pyx + +from pyarrow.includes.common import * +from pyarrow.includes.libarrow import * +from pyarrow.lib import MemoryPool, RecordBatchReader, Schema, _Weakrefable + + +class ReadOptions(_Weakrefable): + __slots__ = () + + def __init__(self, use_threads=None, block_size=None): ... + @property + def use_threads(self): ... + @use_threads.setter + def use_threads(self, value): ... + @property + def block_size(self): ... + @block_size.setter + def block_size(self, value): ... + def __reduce__(self): ... + def equals(self, other: ReadOptions): ... + def __eq__(self, other): ... + def __repr__(self): ... + def __str__(self): ... + +class ParseOptions(_Weakrefable): + __slots__ = () + + def __init__(self, explicit_schema=None, newlines_in_values=None, unexpected_field_behavior=None): ... + def __reduce__(self): ... + @property + def explicit_schema(self): ... + @explicit_schema.setter + def explicit_schema(self, value): ... + @property + def newlines_in_values(self): ... + @newlines_in_values.setter + def newlines_in_values(self, value): ... + @property + def unexpected_field_behavior(self): ... + @unexpected_field_behavior.setter + def unexpected_field_behavior(self, value): ... + def equals(self, other: ParseOptions): ... + def __eq__(self, other): ... + def _repr_base(self): ... + def __repr__(self): ... + def __str__(self): ... + +class JSONStreamingReader(RecordBatchReader): + schema: Schema + + def __init__(self): ... + +def read_json(input_file, read_options=None, parse_options=None, memory_pool: MemoryPool | None=None): ... +def open_json(input_file, read_options=None, parse_options=None, memory_pool: MemoryPool | None=None): ... diff --git a/python/pyarrow-stubs/pyarrow/_orc.pyi b/python/pyarrow-stubs/pyarrow/_orc.pyi new file mode 100644 index 000000000000..91fba12ca14d --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_orc.pyi @@ -0,0 +1,59 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_orc.pyx + +from typing import TypedDict + +from _typeshed import Incomplete +from pyarrow.includes.common import * +from pyarrow.includes.libarrow import * +from pyarrow.includes.libarrow import CCompressionType +from pyarrow.lib import MemoryPool, Table, _Weakrefable +from typing_extensions import TypeAlias + +CompressionStrategy: TypeAlias = int +WriterId: TypeAlias = int +WriterVersion: TypeAlias = int +FileVersion: TypeAlias = Incomplete +ORCFileReader: TypeAlias = Incomplete +ORCFileWriter: TypeAlias = Incomplete + +class ORCReader(_Weakrefable): + def __init__(self, memory_pool: MemoryPool | None=None): ... + def open(self, source: object, use_memory_map: c_bool=True): ... + def metadata(self): ... + def schema(self): ... + def nrows(self): ... + def nstripes(self): ... + def file_version(self): ... + def software_version(self): ... + def compression(self): ... + def compression_size(self): ... + def row_index_stride(self): ... + def writer(self): ... + def writer_version(self): ... + def nstripe_statistics(self): ... + def content_length(self): ... + def stripe_statistics_length(self): ... + def file_footer_length(self): ... + def file_postscript_length(self): ... + def file_length(self): ... + def serialized_file_tail(self): ... + def read_stripe(self, n, columns=None): ... + def read(self, columns=None): ... + +class ORCWriter(_Weakrefable): + def open(self, where: object, *, file_version=None, batch_size=None, stripe_size=None, compression=None, compression_block_size=None, compression_strategy=None, row_index_stride=None, padding_tolerance=None, dictionary_key_size_threshold=None, bloom_filter_columns=None, bloom_filter_fpp=None): ... + def write(self, table: Table): ... + def close(self): ... + +class WriteOptions(TypedDict): + batch_size: int + file_version: FileVersion + stripe_size: int + compression: CCompressionType + compression_block_size: int + compression_strategy: CompressionStrategy + row_index_stride: int + padding_tolerance: float + dictionary_key_size_threshold: float + bloom_filter_columns: std_vector[int] + bloom_filter_fpp: float diff --git a/python/pyarrow-stubs/pyarrow/_parquet.pyi b/python/pyarrow-stubs/pyarrow/_parquet.pyi new file mode 100644 index 000000000000..b34fe8bd6df4 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_parquet.pyi @@ -0,0 +1,277 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_parquet.pyx + +from typing import Any + +from pyarrow.includes.common import * +from pyarrow.includes.libarrow import * +from pyarrow.includes.libarrow_python import * +from pyarrow.includes.libparquet import * +from pyarrow.lib import Buffer, MemoryPool, Schema, Table, _Weakrefable + +_DEFAULT_ROW_GROUP_SIZE = 1024 * 1024 +_MAX_ROW_GROUP_SIZE = 64 * 1024 * 1024 + +class Statistics(_Weakrefable): + def __init__(self): ... + def __repr__(self): ... + def to_dict(self): ... + def __eq__(self, other): ... + def equals(self, other: Statistics): ... + @property + def has_min_max(self): ... + @property + def has_null_count(self): ... + @property + def has_distinct_count(self): ... + @property + def min_raw(self): ... + @property + def max_raw(self): ... + @property + def min(self): ... + @property + def max(self): ... + @property + def null_count(self): ... + @property + def distinct_count(self): ... + @property + def num_values(self): ... + @property + def physical_type(self): ... + @property + def logical_type(self): ... + @property + def converted_type(self): ... + +class ParquetLogicalType(_Weakrefable): + def __init__(self): ... + def __repr__(self): ... + def __str__(self): ... + def to_json(self): ... + @property + def type(self): ... + +class GeoStatistics(_Weakrefable): + def __init__(self): ... + def __repr__(self): ... + def to_dict(self): ... + @property + def geospatial_types(self): ... + @property + def xmin(self): ... + @property + def xmax(self): ... + @property + def ymin(self): ... + @property + def ymax(self): ... + @property + def zmin(self): ... + @property + def zmax(self): ... + @property + def mmin(self): ... + @property + def mmax(self): ... + def _x_valid(self): ... + def _y_valid(self): ... + def _z_valid(self): ... + def _m_valid(self): ... + +class ColumnChunkMetaData(_Weakrefable): + def __init__(self): ... + def __repr__(self): ... + def to_dict(self): ... + def __eq__(self, other): ... + def equals(self, other: ColumnChunkMetaData): ... + @property + def file_offset(self): ... + @property + def file_path(self): ... + @property + def physical_type(self): ... + @property + def num_values(self): ... + @property + def path_in_schema(self): ... + @property + def is_stats_set(self): ... + @property + def statistics(self): ... + @property + def is_geo_stats_set(self): ... + @property + def geo_statistics(self): ... + @property + def compression(self): ... + @property + def encodings(self): ... + @property + def has_dictionary_page(self): ... + @property + def dictionary_page_offset(self): ... + @property + def data_page_offset(self): ... + @property + def has_index_page(self): ... + @property + def index_page_offset(self): ... + @property + def total_compressed_size(self): ... + @property + def total_uncompressed_size(self): ... + @property + def bloom_filter_offset(self): ... + @property + def bloom_filter_length(self): ... + @property + def has_offset_index(self): ... + @property + def has_column_index(self): ... + @property + def metadata(self): ... + +class SortingColumn: + def __init__(self, column_index: int, descending: c_bool=False, nulls_first: c_bool=False): ... + @classmethod + def from_ordering(cls, schema: Schema, sort_keys, null_placement='at_end'): ... + @staticmethod + def to_ordering(schema: Schema, sorting_columns): ... + def __repr__(self): ... + def __eq__(self, other: SortingColumn): ... + def __hash__(self): ... + @property + def column_index(self): ... + @property + def descending(self): ... + @property + def nulls_first(self): ... + def to_dict(self): ... + +class RowGroupMetaData(_Weakrefable): + def __init__(self): ... + def __reduce__(self): ... + def __eq__(self, other): ... + def equals(self, other: RowGroupMetaData): ... + def column(self, i: int): ... + def __repr__(self): ... + def to_dict(self): ... + @property + def num_columns(self): ... + @property + def num_rows(self): ... + @property + def total_byte_size(self): ... + @property + def sorting_columns(self): ... + +class FileMetaData(_Weakrefable): + def __init__(self): ... + def __reduce__(self): ... + def __hash__(self): ... + def __repr__(self): ... + def to_dict(self): ... + def __eq__(self, other): ... + def equals(self, other: FileMetaData): ... + @property + def schema(self): ... + @property + def serialized_size(self): ... + @property + def num_columns(self): ... + @property + def num_rows(self): ... + @property + def num_row_groups(self): ... + @property + def format_version(self): ... + @property + def created_by(self): ... + @property + def metadata(self): ... + def row_group(self, i: int): ... + def set_file_path(self, path): ... + def append_row_groups(self, other: FileMetaData): ... + def write_metadata_file(self, where): ... + +class ParquetSchema(_Weakrefable): + def __init__(self, container: FileMetaData): ... + def __repr__(self): ... + def __reduce__(self): ... + def __len__(self): ... + def __getitem__(self, i): ... + def __hash__(self): ... + @property + def names(self): ... + def to_arrow_schema(self): ... + def __eq__(self, other): ... + def equals(self, other: ParquetSchema): ... + def column(self, i): ... + +class ColumnSchema(_Weakrefable): + def __init__(self, schema: ParquetSchema, index: int): ... + def __eq__(self, other): ... + def __reduce__(self): ... + def equals(self, other: ColumnSchema): ... + def __repr__(self): ... + @property + def name(self): ... + @property + def path(self): ... + @property + def max_definition_level(self): ... + @property + def max_repetition_level(self): ... + @property + def physical_type(self): ... + @property + def logical_type(self): ... + @property + def converted_type(self): ... + @property + def length(self): ... + @property + def precision(self): ... + @property + def scale(self): ... + +class ParquetReader(_Weakrefable): + _column_idx_map: Any + + def __init__(self, memory_pool: MemoryPool | None=None): ... + def open(self, source: object, *, use_memory_map: bool=False, read_dictionary=None, binary_type=None, list_type=None, metadata: FileMetaData | None=None, buffer_size: int=0, pre_buffer: bool=True, coerce_int96_timestamp_unit=None, decryption_properties: FileDecryptionProperties | None=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, page_checksum_verification=False, arrow_extensions_enabled=False): ... + @property + def column_paths(self): ... + @property + def metadata(self): ... + @property + def schema_arrow(self): ... + @property + def num_row_groups(self): ... + def set_use_threads(self, use_threads: bool): ... + def set_batch_size(self, batch_size: int): ... + def iter_batches(self, batch_size: int, row_groups, column_indices=None, use_threads: bool=True): ... + def read_row_group(self, i: int, column_indices=None, use_threads: bool=True): ... + def read_row_groups(self, row_groups, column_indices=None, use_threads: bool=True): ... + def read_all(self, column_indices=None, use_threads: bool=True): ... + def scan_contents(self, column_indices=None, batch_size=65536): ... + def column_name_idx(self, column_name): ... + def read_column(self, column_index: int): ... + def close(self): ... + @property + def closed(self): ... + +class ParquetWriter(_Weakrefable): + def __init__(self, where, schema: Schema, use_dictionary=None, compression=None, version=None, write_statistics=None, memory_pool: MemoryPool | None=None, use_deprecated_int96_timestamps=False, coerce_timestamps=None, data_page_size=None, max_rows_per_page=None, allow_truncated_timestamps=False, compression_level=None, use_byte_stream_split=False, column_encoding=None, writer_engine_version=None, data_page_version=None, use_compliant_nested_type=True, encryption_properties=None, write_batch_size=None, dictionary_pagesize_limit=None, store_schema=True, write_page_index=False, write_page_checksum=False, sorting_columns=None, store_decimal_as_integer=False, use_content_defined_chunking=False, write_time_adjusted_to_utc=False, bloom_filter_options=None): ... + def close(self): ... + def write_table(self, table: Table, row_group_size=None): ... + def add_key_value_metadata(self, key_value_metadata): ... + @property + def metadata(self): ... + +class FileEncryptionProperties: ... + +class FileDecryptionProperties: ... + +def _reconstruct_filemetadata(serialized: Buffer): ... diff --git a/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi b/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi new file mode 100644 index 000000000000..0eabec387f63 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi @@ -0,0 +1,128 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_parquet_encryption.pyx + +from typing import Any + +from pyarrow._fs import FileSystem +from pyarrow.includes.common import * +from pyarrow.includes.libarrow import * +from pyarrow.includes.libparquet_encryption import * +from pyarrow.lib import _Weakrefable + + +class EncryptionConfiguration(_Weakrefable): + __slots__ = () + + def __init__(self, footer_key, *, column_keys=None, uniform_encryption=None, encryption_algorithm=None, plaintext_footer=None, double_wrapping=None, cache_lifetime=None, internal_key_material=None, data_key_length_bits=None): ... + @property + def footer_key(self): ... + @property + def column_keys(self): ... + @column_keys.setter + def column_keys(self, value: dict[Any, Any]): ... + @property + def uniform_encryption(self): ... + @uniform_encryption.setter + def uniform_encryption(self, value): ... + @property + def encryption_algorithm(self): ... + @encryption_algorithm.setter + def encryption_algorithm(self, value): ... + @property + def plaintext_footer(self): ... + @plaintext_footer.setter + def plaintext_footer(self, value): ... + @property + def double_wrapping(self): ... + @double_wrapping.setter + def double_wrapping(self, value): ... + @property + def cache_lifetime(self): ... + @cache_lifetime.setter + def cache_lifetime(self, value): ... + @property + def internal_key_material(self): ... + @internal_key_material.setter + def internal_key_material(self, value): ... + @property + def data_key_length_bits(self): ... + @data_key_length_bits.setter + def data_key_length_bits(self, value): ... + +class DecryptionConfiguration(_Weakrefable): + __slots__ = () + + def __init__(self, *, cache_lifetime=None, read_kms_url=None): ... + @property + def cache_lifetime(self): ... + @cache_lifetime.setter + def cache_lifetime(self, value): ... + @property + def read_kms_url(self): ... + @read_kms_url.setter + def read_kms_url(self, value): ... + +class KmsConnectionConfig(_Weakrefable): + __slots__ = () + + def __init__(self, *, kms_instance_id=None, kms_instance_url=None, key_access_token=None, custom_kms_conf=None): ... + @property + def kms_instance_id(self): ... + @kms_instance_id.setter + def kms_instance_id(self, value): ... + @property + def kms_instance_url(self): ... + @kms_instance_url.setter + def kms_instance_url(self, value): ... + @property + def key_access_token(self): ... + @key_access_token.setter + def key_access_token(self, value): ... + @property + def custom_kms_conf(self): ... + @custom_kms_conf.setter + def custom_kms_conf(self, value: dict[Any, Any]): ... + def refresh_key_access_token(self, value): ... + +class KmsClient(_Weakrefable): + def __init__(self): ... + def wrap_key(self, key_bytes, master_key_identifier): ... + def unwrap_key(self, wrapped_key, master_key_identifier): ... + +class CryptoFactory(_Weakrefable): + __slots__ = () + + def __init__(self, kms_client_factory): ... + def file_encryption_properties(self, kms_connection_config: KmsConnectionConfig, encryption_config: EncryptionConfiguration, parquet_file_path=None, filesystem: FileSystem | None=None): ... + def file_decryption_properties(self, kms_connection_config: KmsConnectionConfig, decryption_config: DecryptionConfiguration | None=None, parquet_file_path=None, filesystem: FileSystem | None=None): ... + def remove_cache_entries_for_token(self, access_token): ... + def remove_cache_entries_for_all_tokens(self): ... + def rotate_master_keys(self, kms_connection_config: KmsConnectionConfig, parquet_file_path, filesystem: FileSystem | None=None, double_wrapping=True, cache_lifetime_seconds=600, read_kms_url=False): ... + +class KeyMaterial(_Weakrefable): + @property + def is_footer_key(self): ... + @property + def is_double_wrapped(self): ... + @property + def master_key_id(self): ... + @property + def wrapped_dek(self): ... + @property + def kek_id(self): ... + @property + def wrapped_kek(self): ... + @property + def kms_instance_id(self): ... + @property + def kms_instance_url(self): ... + @staticmethod + def parse(key_material_string: c_string): ... + +class FileSystemKeyMaterialStore(_Weakrefable): + def get_key_material(self, key_id): ... + def get_key_id_set(self): ... + @classmethod + def for_file(cls, parquet_file_path, filesystem: FileSystem | None=None): ... + +def create_decryption_properties(footer_key, *, aad_prefix=None, check_footer_integrity: bool=True, allow_plaintext_files: bool=False): ... +def create_encryption_properties(footer_key, *, aad_prefix=None, store_aad_prefix: bool=True, encryption_algorithm='AES_GCM_V1', plaintext_footer: bool=False): ... diff --git a/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi b/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi new file mode 100644 index 000000000000..2baa854b53fe --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi @@ -0,0 +1,20 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_pyarrow_cpp_tests.pyx + +from typing import Callable + +from _typeshed import Incomplete +from pyarrow.includes.common import * +from pyarrow.includes.libarrow import * +from pyarrow.includes.libarrow import CStatus +from typing_extensions import TypeAlias + +cb_test_func: TypeAlias = Callable[[], CStatus] +CTestCase: TypeAlias = Incomplete + +class CppTestCase: + @property + def name(self): ... + def __repr__(self): ... + def __call__(self): ... + +def get_cpp_tests(): ... diff --git a/python/pyarrow-stubs/pyarrow/_s3fs.pyi b/python/pyarrow-stubs/pyarrow/_s3fs.pyi new file mode 100644 index 000000000000..f0d4c150fd68 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_s3fs.pyi @@ -0,0 +1,39 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_s3fs.pyx + +from enum import IntEnum + +from pyarrow._fs import FileSystem +from pyarrow.includes.common import * +from pyarrow.includes.libarrow import * +from pyarrow.includes.libarrow_fs import * + + +class S3LogLevel(IntEnum): + Off = ... + Fatal = ... + Error = ... + Warn = ... + Info = ... + Debug = ... + Trace = ... + +class S3RetryStrategy: + def __init__(self, max_attempts=3): ... + +class AwsStandardS3RetryStrategy(S3RetryStrategy): ... + +class AwsDefaultS3RetryStrategy(S3RetryStrategy): ... + +class S3FileSystem(FileSystem): + def __init__(self, *, access_key=None, secret_key=None, session_token=None, anonymous: bool=False, region=None, request_timeout=None, connect_timeout=None, scheme=None, endpoint_override=None, background_writes: bool=True, default_metadata=None, role_arn=None, session_name=None, external_id=None, load_frequency=900, proxy_options=None, allow_delayed_open=False, allow_bucket_creation=False, allow_bucket_deletion=False, check_directory_existence_before_creation=False, retry_strategy: S3RetryStrategy=AwsStandardS3RetryStrategy(max_attempts=3), force_virtual_addressing=False, tls_ca_file_path=None): ... + @staticmethod + def _reconstruct(kwargs): ... + def __reduce__(self): ... + @property + def region(self): ... + +def initialize_s3(log_level: S3LogLevel=S3LogLevel.Fatal, num_event_loop_threads: int=1): ... +def ensure_s3_initialized(): ... +def finalize_s3(): ... +def ensure_s3_finalized(): ... +def resolve_s3_region(bucket): ... diff --git a/python/pyarrow-stubs/pyarrow/_substrait.pyi b/python/pyarrow-stubs/pyarrow/_substrait.pyi new file mode 100644 index 000000000000..3b099457ff18 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_substrait.pyi @@ -0,0 +1,28 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_substrait.pyx + +from pyarrow.includes.libarrow import * +from pyarrow.includes.libarrow_substrait import * +from pyarrow.lib import * + + +class SubstraitSchema: + def __init__(self, schema, expression): ... + def to_pysubstrait(self): ... + +class BoundExpressions(_Weakrefable): + def __init__(self): ... + @property + def schema(self): ... + @property + def expressions(self): ... + @classmethod + def from_substrait(cls, message): ... + +def run_query(plan, *, table_provider=None, use_threads=True): ... +def _parse_json_plan(plan): ... +def serialize_schema(schema): ... +def _serialize_namedstruct_schema(schema): ... +def deserialize_schema(buf): ... +def serialize_expressions(exprs, names, schema, *, allow_arrow_extensions=False): ... +def deserialize_expressions(buf): ... +def get_supported_functions(): ... diff --git a/python/pyarrow-stubs/pyarrow/gandiva.pyi b/python/pyarrow-stubs/pyarrow/gandiva.pyi new file mode 100644 index 000000000000..e4cb7360332b --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/gandiva.pyi @@ -0,0 +1,75 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/gandiva.pyx + +from pyarrow.includes.libarrow import * +from pyarrow.lib import (DataType, Field, MemoryPool, RecordBatch, Schema, + _Weakrefable) + + +class Node(_Weakrefable): + def __init__(self): ... + def __str__(self): ... + def __repr__(self): ... + def return_type(self): ... + +class Expression(_Weakrefable): + def __str__(self): ... + def __repr__(self): ... + def root(self): ... + def result(self): ... + +class Condition(_Weakrefable): + def __init__(self): ... + def __str__(self): ... + def __repr__(self): ... + def root(self): ... + def result(self): ... + +class SelectionVector(_Weakrefable): + def __init__(self): ... + def to_array(self): ... + +class Projector(_Weakrefable): + def __init__(self): ... + @property + def llvm_ir(self): ... + def evaluate(self, batch: RecordBatch, selection: SelectionVector | None=None): ... + +class Filter(_Weakrefable): + def __init__(self): ... + @property + def llvm_ir(self): ... + def evaluate(self, batch: RecordBatch, pool: MemoryPool, dtype='int32'): ... + +class TreeExprBuilder(_Weakrefable): + def make_literal(self, value, dtype): ... + def make_expression(self, root_node: Node, return_field: Field): ... + def make_function(self, name, children, return_type: DataType): ... + def make_field(self, field: Field): ... + def make_if(self, condition: Node, this_node: Node, else_node: Node, return_type: DataType): ... + def make_and(self, children): ... + def make_or(self, children): ... + def _make_in_expression_int32(self, node: Node, values): ... + def _make_in_expression_int64(self, node: Node, values): ... + def _make_in_expression_time32(self, node: Node, values): ... + def _make_in_expression_time64(self, node: Node, values): ... + def _make_in_expression_date32(self, node: Node, values): ... + def _make_in_expression_date64(self, node: Node, values): ... + def _make_in_expression_timestamp(self, node: Node, values): ... + def _make_in_expression_binary(self, node: Node, values): ... + def _make_in_expression_string(self, node: Node, values): ... + def make_in_expression(self, node: Node, values, dtype): ... + def make_condition(self, condition: Node): ... + +class Configuration(_Weakrefable): + def __init__(self, optimize: bool=True, dump_ir: bool=False): ... + +class FunctionSignature(_Weakrefable): + def __init__(self): ... + def return_type(self): ... + def param_types(self): ... + def name(self): ... + def __repr__(self): ... + +def make_projector(schema: Schema, children, pool: MemoryPool, selection_mode: str='NONE', configuration: Configuration | None=None): ... +def make_filter(schema: Schema, condition: Condition, configuration: Configuration | None=None): ... +def get_registered_function_signatures(): ... diff --git a/python/pyarrow-stubs/pyarrow/lib.pyi b/python/pyarrow-stubs/pyarrow/lib.pyi new file mode 100644 index 000000000000..21ea20375f14 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/lib.pyi @@ -0,0 +1,2020 @@ +# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/lib.pyx + +import sys +from collections import namedtuple +from collections.abc import Mapping, Sequence +from enum import IntEnum +from typing import Any + +from pyarrow.includes.common import * +from pyarrow.includes.libarrow import * +from pyarrow.includes.libarrow_python import * +from typing_extensions import TypeAlias + + +class DeviceAllocationType(IntEnum): + CPU = ... + CUDA = ... + CUDA_HOST = ... + OPENCL = ... + VULKAN = ... + METAL = ... + VPI = ... + ROCM = ... + ROCM_HOST = ... + EXT_DEV = ... + CUDA_MANAGED = ... + ONEAPI = ... + WEBGPU = ... + HEXAGON = ... + +class MetadataVersion(IntEnum): + V1 = ... + V2 = ... + V3 = ... + V4 = ... + V5 = ... + +class Alignment(IntEnum): + Any = ... + DataTypeSpecific = ... + At64Byte = ... + +MonthDayNano = ... +Type_NA = ... +Type_BOOL = ... +Type_UINT8 = ... +Type_INT8 = ... +Type_UINT16 = ... +Type_INT16 = ... +Type_UINT32 = ... +Type_INT32 = ... +Type_UINT64 = ... +Type_INT64 = ... +Type_HALF_FLOAT = ... +Type_FLOAT = ... +Type_DOUBLE = ... +Type_DECIMAL32 = ... +Type_DECIMAL64 = ... +Type_DECIMAL128 = ... +Type_DECIMAL256 = ... +Type_DATE32 = ... +Type_DATE64 = ... +Type_TIMESTAMP = ... +Type_TIME32 = ... +Type_TIME64 = ... +Type_DURATION = ... +Type_INTERVAL_MONTH_DAY_NANO = ... +Type_BINARY = ... +Type_STRING = ... +Type_LARGE_BINARY = ... +Type_LARGE_STRING = ... +Type_FIXED_SIZE_BINARY = ... +Type_BINARY_VIEW = ... +Type_STRING_VIEW = ... +Type_LIST = ... +Type_LARGE_LIST = ... +Type_LIST_VIEW = ... +Type_LARGE_LIST_VIEW = ... +Type_MAP = ... +Type_FIXED_SIZE_LIST = ... +Type_STRUCT = ... +Type_SPARSE_UNION = ... +Type_DENSE_UNION = ... +Type_DICTIONARY = ... +Type_RUN_END_ENCODED = ... +Type_INTERVAL_MONTHS = ... +Type_INTERVAL_DAY_TIME = ... +UnionMode_SPARSE = ... +UnionMode_DENSE = ... +__pc = None +__pac = None +__cuda_loaded = None +ordered_dict = dict +ArrowIOError = IOError +have_signal_refcycle = sys.version_info < (3, 8, 10) or (3, 9) <= sys.version_info or sys.version_info[:2] == (3, 10) +VersionInfo = namedtuple('VersionInfo', ('major', 'minor', 'patch')) +RuntimeInfo = namedtuple('RuntimeInfo', ('simd_level', 'detected_simd_level')) +BuildInfo = namedtuple('BuildInfo', ('build_type', 'cpp_build_info')) +CppBuildInfo = namedtuple('CppBuildInfo', ('version', 'version_info', 'so_version', 'full_so_version', 'compiler_id', 'compiler_version', 'compiler_flags', 'git_id', 'git_description', 'package_kind', 'build_type')) +build_info = _build_info() +cpp_build_info = build_info.cpp_build_info +cpp_version = build_info.cpp_build_info.version +cpp_version_info = build_info.cpp_build_info.version_info +_pandas_api = ... +_CFixedWidthTypePtr: TypeAlias = CFixedWidthType +_python_extension_types_registry = [] +_registry_nanny = _ExtensionRegistryNanny() +_CSparseCOOIndexPtr: TypeAlias = CSparseCOOIndex +DEFAULT_BUFFER_SIZE = 2 ** 16 +_CBufferedInputStreamPtr: TypeAlias = CBufferedInputStream +_CBufferedOutputStreamPtr: TypeAlias = CBufferedOutputStream +_RandomAccessFilePtr: TypeAlias = CRandomAccessFile +_WriteStats = namedtuple('WriteStats', ('num_messages', 'num_record_batches', 'num_dictionary_batches', 'num_dictionary_deltas', 'num_replaced_dictionaries')) +_ReadStats = namedtuple('ReadStats', ('num_messages', 'num_record_batches', 'num_dictionary_batches', 'num_dictionary_deltas', 'num_replaced_dictionaries')) +_RecordBatchWithMetadata = namedtuple('RecordBatchWithMetadata', ('batch', 'custom_metadata')) +_CPyExtensionTypePtr: TypeAlias = CPyExtensionType + +class ArrowException(Exception): ... + +class ArrowInvalid(ValueError, ArrowException): ... + +class ArrowMemoryError(MemoryError, ArrowException): ... + +class ArrowKeyError(KeyError, ArrowException): + def __str__(self): ... + +class ArrowTypeError(TypeError, ArrowException): ... + +class ArrowNotImplementedError(NotImplementedError, ArrowException): ... + +class ArrowCapacityError(ArrowException): ... + +class ArrowIndexError(IndexError, ArrowException): ... + +class ArrowSerializationError(ArrowException): ... + +class ArrowCancelled(ArrowException): + def __init__(self, message, signum=None): ... + +class StopToken: ... + +class SignalStopHandler: + def __init__(self): ... + def _init_signals(self): ... + def __enter__(self): ... + def __exit__(self, exc_type, exc_value, exc_tb): ... + def __dealloc__(self): ... + @property + def stop_token(self): ... + +class _PandasAPIShim(object): + _loose_version: object + _version: object + _pd: object + _types_api: object + _compat_module: object + _data_frame: object + _index: object + _series: object + _categorical_type: object + _datetimetz_type: object + _extension_array: object + _extension_dtype: object + _array_like_types: object + _lock: object + _is_ge_v23: bool + _is_ge_v3: bool + _is_ge_v3_strict: bool + + def __init__(self): ... + def series(self, *args, **kwargs): ... + def data_frame(self, *args, **kwargs): ... + @property + def have_pandas(self): ... + @property + def compat(self): ... + @property + def pd(self): ... + def infer_dtype(self, obj): ... + def pandas_dtype(self, dtype): ... + @property + def loose_version(self): ... + @property + def version(self): ... + def is_ge_v23(self): ... + def is_ge_v3(self): ... + def is_ge_v3_strict(self): ... + def uses_string_dtype(self): ... + @property + def categorical_type(self): ... + @property + def datetimetz_type(self): ... + @property + def extension_dtype(self): ... + def is_array_like(self, obj): ... + def is_categorical(self, obj): ... + def is_datetimetz(self, obj): ... + def is_extension_array_dtype(self, obj): ... + def is_sparse(self, obj): ... + def is_data_frame(self, obj): ... + def is_series(self, obj): ... + def is_index(self, obj): ... + def get_values(self, obj): ... + +class MemoryPool(_Weakrefable): + def __init__(self): ... + def release_unused(self): ... + def bytes_allocated(self): ... + def total_bytes_allocated(self): ... + def max_memory(self): ... + def num_allocations(self): ... + def print_stats(self): ... + @property + def backend_name(self): ... + def __repr__(self): ... + +class LoggingMemoryPool(MemoryPool): + def __init__(self): ... + +class ProxyMemoryPool(MemoryPool): + def __init__(self): ... + +class Device(_Weakrefable): + def __init__(self): ... + def __eq__(self, other): ... + def __repr__(self): ... + @property + def type_name(self): ... + @property + def device_id(self): ... + @property + def is_cpu(self): ... + @property + def device_type(self): ... + +class MemoryManager(_Weakrefable): + def __init__(self): ... + def __repr__(self): ... + @property + def device(self): ... + @property + def is_cpu(self): ... + +class DataType(_Weakrefable): + def __init__(self): ... + def field(self, i) -> Field: ... + @property + def id(self): ... + @property + def bit_width(self): ... + @property + def byte_width(self): ... + @property + def num_fields(self): ... + @property + def num_buffers(self): ... + @property + def has_variadic_buffers(self): ... + def __str__(self): ... + def __hash__(self): ... + def __reduce__(self): ... + def __repr__(self): ... + def __eq__(self, other): ... + def equals(self, other, *, check_metadata=False): ... + def to_pandas_dtype(self): ... + def _export_to_c(self, out_ptr): ... + @staticmethod + def _import_from_c(in_ptr): ... + def __arrow_c_schema__(self): ... + @staticmethod + def _import_from_c_capsule(schema): ... + +class DictionaryMemo(_Weakrefable): + def __init__(self): ... + +class DictionaryType(DataType): + def __reduce__(self): ... + @property + def ordered(self): ... + @property + def index_type(self): ... + @property + def value_type(self): ... + +class ListType(DataType): + def __reduce__(self): ... + @property + def value_field(self): ... + @property + def value_type(self): ... + +class LargeListType(DataType): + def __reduce__(self): ... + @property + def value_field(self): ... + @property + def value_type(self): ... + +class ListViewType(DataType): + def __reduce__(self): ... + @property + def value_field(self): ... + @property + def value_type(self): ... + +class LargeListViewType(DataType): + def __reduce__(self): ... + @property + def value_field(self): ... + @property + def value_type(self): ... + +class MapType(DataType): + def __reduce__(self): ... + @property + def key_field(self): ... + @property + def key_type(self): ... + @property + def item_field(self): ... + @property + def item_type(self): ... + @property + def keys_sorted(self): ... + +class FixedSizeListType(DataType): + def __reduce__(self): ... + @property + def value_field(self): ... + @property + def value_type(self): ... + @property + def list_size(self): ... + +class StructType(DataType): + def get_field_index(self, name): ... + def field(self, i) -> Field: ... + def get_all_field_indices(self, name): ... + def __len__(self): ... + def __iter__(self): ... + def __getitem__(self, i): ... + def __reduce__(self): ... + @property + def names(self): ... + @property + def fields(self): ... + +class UnionType(DataType): + @property + def mode(self): ... + @property + def type_codes(self): ... + def __len__(self): ... + def __iter__(self): ... + def field(self, i) -> Field: ... + def __getitem__(self, i): ... + def __reduce__(self): ... + +class SparseUnionType(UnionType): ... + +class DenseUnionType(UnionType): ... + +class TimestampType(DataType): + @property + def unit(self): ... + @property + def tz(self): ... + def __reduce__(self): ... + +class Time32Type(DataType): + @property + def unit(self): ... + +class Time64Type(DataType): + @property + def unit(self): ... + +class DurationType(DataType): + @property + def unit(self): ... + +class FixedSizeBinaryType(DataType): + def __reduce__(self): ... + +class Decimal32Type(FixedSizeBinaryType): + def __reduce__(self): ... + @property + def precision(self): ... + @property + def scale(self): ... + +class Decimal64Type(FixedSizeBinaryType): + def __reduce__(self): ... + @property + def precision(self): ... + @property + def scale(self): ... + +class Decimal128Type(FixedSizeBinaryType): + def __reduce__(self): ... + @property + def precision(self): ... + @property + def scale(self): ... + +class Decimal256Type(FixedSizeBinaryType): + def __reduce__(self): ... + @property + def precision(self): ... + @property + def scale(self): ... + +class RunEndEncodedType(DataType): + def __reduce__(self): ... + @property + def run_end_type(self): ... + @property + def value_type(self): ... + +class BaseExtensionType(DataType): + def __arrow_ext_class__(self): ... + def __arrow_ext_scalar_class__(self): ... + @property + def extension_name(self): ... + @property + def storage_type(self): ... + @property + def byte_width(self): ... + @property + def bit_width(self): ... + def wrap_array(self, storage): ... + +class ExtensionType(BaseExtensionType): + def __init__(self, storage_type: DataType, extension_name): ... + def __eq__(self, other): ... + def __repr__(self): ... + def __arrow_ext_serialize__(self): ... + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): ... + def __reduce__(self): ... + def __arrow_ext_class__(self): ... + def __arrow_ext_scalar_class__(self): ... + +class JsonType(BaseExtensionType): + def __arrow_ext_class__(self): ... + def __reduce__(self): ... + def __arrow_ext_scalar_class__(self): ... + +class UuidType(BaseExtensionType): + def __arrow_ext_class__(self): ... + def __reduce__(self): ... + def __arrow_ext_scalar_class__(self): ... + +class FixedShapeTensorType(BaseExtensionType): + @property + def value_type(self): ... + @property + def shape(self): ... + @property + def dim_names(self): ... + @property + def permutation(self): ... + def __arrow_ext_class__(self): ... + def __reduce__(self): ... + def __arrow_ext_scalar_class__(self): ... + +class Bool8Type(BaseExtensionType): + def __arrow_ext_class__(self): ... + def __reduce__(self): ... + def __arrow_ext_scalar_class__(self): ... + +class OpaqueType(BaseExtensionType): + @property + def type_name(self): ... + @property + def vendor_name(self): ... + def __arrow_ext_class__(self): ... + def __reduce__(self): ... + def __arrow_ext_scalar_class__(self): ... + +class UnknownExtensionType(ExtensionType): + def __init__(self, storage_type: DataType, serialized): ... + def __arrow_ext_serialize__(self): ... + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): ... + +class KeyValueMetadata(_Metadata, Mapping): + def __init__(self, __arg0__=None, **kwargs): ... + def equals(self, other: KeyValueMetadata): ... + def __repr__(self): ... + def __str__(self): ... + def __eq__(self, other): ... + def __len__(self): ... + def __contains__(self, key): ... + def __getitem__(self, key): ... + def __iter__(self): ... + def __reduce__(self): ... + def key(self, i): ... + def value(self, i): ... + def keys(self): ... + def values(self): ... + def items(self): ... + def get_all(self, key): ... + def to_dict(self): ... + +class Field(_Weakrefable): + type: DataType + + def __init__(self): ... + def equals(self, other: Field, check_metadata: bool=False): ... + def __eq__(self, other): ... + def __reduce__(self): ... + def __str__(self): ... + def __repr__(self): ... + def __hash__(self): ... + @property + def nullable(self): ... + @property + def name(self): ... + @property + def metadata(self): ... + def with_metadata(self, metadata): ... + def remove_metadata(self): ... + def with_type(self, new_type: DataType): ... + def with_name(self, name): ... + def with_nullable(self, nullable): ... + def flatten(self): ... + def _export_to_c(self, out_ptr): ... + @staticmethod + def _import_from_c(in_ptr): ... + def __arrow_c_schema__(self): ... + @staticmethod + def _import_from_c_capsule(schema): ... + +class Schema(_Weakrefable): + def __init__(self): ... + def __len__(self): ... + def __getitem__(self, key): ... + def __iter__(self): ... + def __reduce__(self): ... + def __hash__(self): ... + def __sizeof__(self): ... + @property + def pandas_metadata(self): ... + @property + def names(self): ... + @property + def types(self): ... + @property + def metadata(self): ... + def __eq__(self, other): ... + def empty_table(self): ... + def equals(self, other: Schema, check_metadata: bool=False): ... + @classmethod + def from_pandas(cls, df, preserve_index=None): ... + def field(self, i): ... + def _field(self, i: int): ... + def field_by_name(self, name): ... + def get_field_index(self, name): ... + def get_all_field_indices(self, name): ... + def append(self, field: Field): ... + def insert(self, i: int, field: Field): ... + def remove(self, i: int): ... + def set(self, i: int, field: Field): ... + def add_metadata(self, metadata): ... + def with_metadata(self, metadata): ... + def serialize(self, memory_pool=None): ... + def remove_metadata(self): ... + def to_string(self, truncate_metadata=True, show_field_metadata=True, show_schema_metadata=True, element_size_limit=100): ... + def _export_to_c(self, out_ptr): ... + @staticmethod + def _import_from_c(in_ptr): ... + def __str__(self): ... + def __repr__(self): ... + def __arrow_c_schema__(self): ... + @staticmethod + def _import_from_c_capsule(schema): ... + +class _ExtensionRegistryNanny(_Weakrefable): + def __init__(self): ... + def release_registry(self): ... + +class Scalar(_Weakrefable): + def __init__(self): ... + @property + def type(self): ... + @property + def is_valid(self): ... + def cast(self, target_type: object | None=None, safe=None, options=None, memory_pool=None): ... + def validate(self, *, full=False): ... + def __repr__(self): ... + def __str__(self): ... + def equals(self, other: Scalar): ... + def __eq__(self, other): ... + def __hash__(self): ... + def __reduce__(self): ... + def as_py(self, *, maps_as_pydicts=None): ... + def __abs__(self): ... + def __add__(self, other: object): ... + def __truediv__(self, other: object): ... + def __mul__(self, other: object): ... + def __neg__(self): ... + def __pow__(self, other: object): ... + def __sub__(self, other: object): ... + def __and__(self, other: object): ... + def __or__(self, other: object): ... + def __xor__(self, other: object): ... + def __lshift__(self, other: object): ... + def __rshift__(self, other: object): ... + +class NullScalar(Scalar): + def __init__(self): ... + def as_py(self, *, maps_as_pydicts=None): ... + +class BooleanScalar(Scalar): + def as_py(self, *, maps_as_pydicts=None): ... + def __bool__(self): ... + +class UInt8Scalar(Scalar): + def as_py(self, *, maps_as_pydicts=None): ... + def __index__(self): ... + +class Int8Scalar(Scalar): + def as_py(self, *, maps_as_pydicts=None): ... + def __index__(self): ... + +class UInt16Scalar(Scalar): + def as_py(self, *, maps_as_pydicts=None): ... + def __index__(self): ... + +class Int16Scalar(Scalar): + def as_py(self, *, maps_as_pydicts=None): ... + def __index__(self): ... + +class UInt32Scalar(Scalar): + def as_py(self, *, maps_as_pydicts=None): ... + def __index__(self): ... + +class Int32Scalar(Scalar): + def as_py(self, *, maps_as_pydicts=None): ... + def __index__(self): ... + +class UInt64Scalar(Scalar): + def as_py(self, *, maps_as_pydicts=None): ... + def __index__(self): ... + +class Int64Scalar(Scalar): + def as_py(self, *, maps_as_pydicts=None): ... + def __index__(self): ... + +class HalfFloatScalar(Scalar): + def as_py(self, *, maps_as_pydicts=None): ... + def __float__(self): ... + def __int__(self): ... + +class FloatScalar(Scalar): + def as_py(self, *, maps_as_pydicts=None): ... + def __float__(self): ... + def __int__(self): ... + +class DoubleScalar(Scalar): + def as_py(self, *, maps_as_pydicts=None): ... + def __float__(self): ... + def __int__(self): ... + +class Decimal32Scalar(Scalar): + def as_py(self, *, maps_as_pydicts=None): ... + +class Decimal64Scalar(Scalar): + def as_py(self, *, maps_as_pydicts=None): ... + +class Decimal128Scalar(Scalar): + def as_py(self, *, maps_as_pydicts=None): ... + +class Decimal256Scalar(Scalar): + def as_py(self, *, maps_as_pydicts=None): ... + +class Date32Scalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts=None): ... + +class Date64Scalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts=None): ... + +class Time32Scalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts=None): ... + +class Time64Scalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts=None): ... + +class TimestampScalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts=None): ... + def __repr__(self): ... + +class DurationScalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts=None): ... + +class MonthDayNanoIntervalScalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts=None): ... + +class BinaryScalar(Scalar): + def as_buffer(self): ... + def as_py(self, *, maps_as_pydicts=None): ... + def __bytes__(self): ... + def __getbuffer__(self, buffer: cp.Py_buffer, flags: int): ... + +class LargeBinaryScalar(BinaryScalar): ... + +class FixedSizeBinaryScalar(BinaryScalar): ... + +class StringScalar(BinaryScalar): + def as_py(self, *, maps_as_pydicts=None): ... + +class LargeStringScalar(StringScalar): ... + +class BinaryViewScalar(BinaryScalar): ... + +class StringViewScalar(StringScalar): ... + +class ListScalar(Scalar, Sequence): + @property + def values(self): ... + def __len__(self): ... + def __getitem__(self, i): ... + def __iter__(self): ... + def as_py(self, *, maps_as_pydicts=None): ... + +class FixedSizeListScalar(ListScalar): ... + +class LargeListScalar(ListScalar): ... + +class ListViewScalar(ListScalar): ... + +class LargeListViewScalar(ListScalar): ... + +class StructScalar(Scalar, Mapping): + def __len__(self): ... + def __iter__(self): ... + def items(self): ... + def __contains__(self, key): ... + def __getitem__(self, key): ... + def as_py(self, *, maps_as_pydicts=None): ... + def _as_py_tuple(self): ... + def __repr__(self): ... + def __str__(self): ... + +class MapScalar(ListScalar, Mapping): + def __getitem__(self, i): ... + def __iter__(self): ... + def as_py(self, *, maps_as_pydicts=None): ... + def keys(self): ... + +class DictionaryScalar(Scalar): + @staticmethod + def _reconstruct(type, is_valid, index, dictionary): ... + def __reduce__(self): ... + @property + def index(self): ... + @property + def value(self): ... + @property + def dictionary(self): ... + def as_py(self, *, maps_as_pydicts=None): ... + +class RunEndEncodedScalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts=None): ... + +class UnionScalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts=None): ... + @property + def type_code(self): ... + +class ExtensionScalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts=None): ... + @staticmethod + def from_storage(typ: BaseExtensionType, value): ... + +class JsonScalar(ExtensionScalar): ... + +class UuidScalar(ExtensionScalar): + def as_py(self, *, maps_as_pydicts=None): ... + +class FixedShapeTensorScalar(ExtensionScalar): + def to_numpy(self): ... + def to_tensor(self): ... + def __dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None): ... + def __dlpack_device__(self): ... + +class OpaqueScalar(ExtensionScalar): ... + +class Bool8Scalar(ExtensionScalar): + def as_py(self, *, maps_as_pydicts=None): ... + +class ArrayStatistics(_Weakrefable): + def __init__(self): ... + def __repr__(self): ... + @property + def null_count(self): ... + @property + def is_null_count_exact(self): ... + @property + def distinct_count(self): ... + @property + def is_distinct_count_exact(self): ... + @property + def min(self): ... + @property + def is_min_exact(self): ... + @property + def max(self): ... + @property + def is_max_exact(self): ... + +class _PandasConvertible(_Weakrefable): + def to_pandas(self, memory_pool=None, categories=None, strings_to_categorical: bool=False, zero_copy_only: bool=False, integer_object_nulls: bool=False, date_as_object: bool=True, timestamp_as_object: bool=False, use_threads: bool=True, deduplicate_objects: bool=True, ignore_metadata: bool=False, safe: bool=True, split_blocks: bool=False, self_destruct: bool=False, maps_as_pydicts: str | None=None, types_mapper=None, coerce_temporal_nanoseconds: bool=False): ... + +class Array(_PandasConvertible): + type: DataType + _name: object + + def __init__(self): ... + def _debug_print(self): ... + def diff(self, other: Array): ... + def cast(self, target_type: object | None=None, safe=None, options=None, memory_pool=None): ... + def view(self, target_type: object): ... + def sum(self, **kwargs): ... + def unique(self): ... + def dictionary_encode(self, null_encoding='mask'): ... + def value_counts(self): ... + @staticmethod + def from_pandas(obj, mask=None, type=None, safe: bool=True, memory_pool: MemoryPool | None=None): ... + def __reduce__(self): ... + @staticmethod + def from_buffers(type: DataType, length, buffers, null_count=-1, offset=0, children=None): ... + @property + def null_count(self): ... + @property + def nbytes(self): ... + def get_total_buffer_size(self): ... + def __sizeof__(self): ... + def __iter__(self): ... + def __repr__(self): ... + def to_string(self, *, indent: int=2, top_level_indent: int=0, window: int=10, container_window: int=2, skip_new_lines: c_bool=False, element_size_limit: int=100): ... + def __str__(self): ... + def __eq__(self, other): ... + def equals(self: Array, other: Array): ... + def __len__(self): ... + def is_null(self, *, nan_is_null=False): ... + def is_nan(self): ... + def is_valid(self): ... + def fill_null(self, fill_value): ... + def __getitem__(self, key): ... + def slice(self, offset=0, length=None): ... + def take(self, indices: object): ... + def drop_null(self): ... + def filter(self, mask: object, *, null_selection_behavior='drop'): ... + def index(self, value, start=None, end=None, *, memory_pool=None): ... + def sort(self, order='ascending', null_placement='at_end', **kwargs): ... + def _to_pandas(self, options, types_mapper=None, **kwargs): ... + def __array__(self, dtype=None, copy=None): ... + def to_numpy(self, zero_copy_only=True, writable=False): ... + def to_tensor(self, *, allow_nulls=False): ... + def to_pylist(self, *, maps_as_pydicts=None): ... + def tolist(self): ... + def validate(self, *, full=False): ... + @property + def offset(self): ... + def buffers(self): ... + def copy_to(self, destination): ... + def _export_to_c(self, out_ptr, out_schema_ptr=0): ... + @staticmethod + def _import_from_c(in_ptr, type): ... + def __arrow_c_array__(self, requested_schema=None): ... + @staticmethod + def _import_from_c_capsule(schema_capsule, array_capsule): ... + def _export_to_c_device(self, out_ptr, out_schema_ptr=0): ... + @staticmethod + def _import_from_c_device(in_ptr, type): ... + def __arrow_c_device_array__(self, requested_schema=None, **kwargs): ... + @staticmethod + def _import_from_c_device_capsule(schema_capsule, array_capsule): ... + @staticmethod + def from_dlpack(x, /, *, device=None, copy=None): ... + def __dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None): ... + def __dlpack_device__(self): ... + @property + def device_type(self): ... + @property + def is_cpu(self): ... + @property + def statistics(self): ... + def __abs__(self): ... + def __add__(self, other: object): ... + def __truediv__(self, other: object): ... + def __mul__(self, other: object): ... + def __neg__(self): ... + def __pow__(self, other: object): ... + def __sub__(self, other: object): ... + def __and__(self, other: object): ... + def __or__(self, other: object): ... + def __xor__(self, other: object): ... + def __lshift__(self, other: object): ... + def __rshift__(self, other: object): ... + +class NullArray(Array): ... + +class BooleanArray(Array): + @property + def false_count(self): ... + @property + def true_count(self): ... + +class NumericArray(Array): ... + +class IntegerArray(NumericArray): ... + +class FloatingPointArray(NumericArray): ... + +class Int8Array(IntegerArray): ... + +class UInt8Array(IntegerArray): ... + +class Int16Array(IntegerArray): ... + +class UInt16Array(IntegerArray): ... + +class Int32Array(IntegerArray): ... + +class UInt32Array(IntegerArray): ... + +class Int64Array(IntegerArray): ... + +class UInt64Array(IntegerArray): ... + +class Date32Array(NumericArray): ... + +class Date64Array(NumericArray): ... + +class TimestampArray(NumericArray): ... + +class Time32Array(NumericArray): ... + +class Time64Array(NumericArray): ... + +class DurationArray(NumericArray): ... + +class MonthDayNanoIntervalArray(Array): + def to_pylist(self, *, maps_as_pydicts=None): ... + +class HalfFloatArray(FloatingPointArray): ... + +class FloatArray(FloatingPointArray): ... + +class DoubleArray(FloatingPointArray): ... + +class FixedSizeBinaryArray(Array): ... + +class Decima32Array(FixedSizeBinaryArray): ... + +class Decimal64Array(FixedSizeBinaryArray): ... + +class Decimal128Array(FixedSizeBinaryArray): ... + +class Decimal256Array(FixedSizeBinaryArray): ... + +class BaseListArray(Array): + def flatten(self, recursive=False): ... + def value_parent_indices(self): ... + def value_lengths(self): ... + +class ListArray(BaseListArray): + @staticmethod + def from_arrays(offsets, values, type: DataType | None=None, pool: MemoryPool | None=None, mask=None): ... + @property + def values(self): ... + @property + def offsets(self): ... + +class LargeListArray(BaseListArray): + @staticmethod + def from_arrays(offsets, values, type: DataType | None=None, pool: MemoryPool | None=None, mask=None): ... + @property + def values(self): ... + @property + def offsets(self): ... + +class ListViewArray(BaseListArray): + @staticmethod + def from_arrays(offsets, sizes, values, type: DataType | None=None, pool: MemoryPool | None=None, mask=None): ... + @property + def values(self): ... + @property + def offsets(self): ... + @property + def sizes(self): ... + +class LargeListViewArray(BaseListArray): + @staticmethod + def from_arrays(offsets, sizes, values, type: DataType | None=None, pool: MemoryPool | None=None, mask=None): ... + @property + def values(self): ... + @property + def offsets(self): ... + @property + def sizes(self): ... + +class MapArray(ListArray): + @staticmethod + def from_arrays(offsets, keys, items, type: DataType | None=None, pool: MemoryPool | None=None, mask=None): ... + @property + def keys(self): ... + @property + def items(self): ... + +class FixedSizeListArray(BaseListArray): + @staticmethod + def from_arrays(values, list_size=None, type: DataType | None=None, mask=None): ... + @property + def values(self): ... + +class UnionArray(Array): + def child(self, pos: int): ... + def field(self, pos: int): ... + @property + def type_codes(self): ... + @property + def offsets(self): ... + @staticmethod + def from_dense(types: Array, value_offsets: Array, children: list[Any], field_names: list[Any] | None=None, type_codes: list[Any] | None=None): ... + @staticmethod + def from_sparse(types: Array, children: list[Any], field_names: list[Any] | None=None, type_codes: list[Any] | None=None): ... + +class StringArray(Array): + @staticmethod + def from_buffers(length: int, value_offsets: Buffer, data: Buffer, null_bitmap: Buffer | None=None, null_count: int=-1, offset: int=0): ... + +class LargeStringArray(Array): + @staticmethod + def from_buffers(length: int, value_offsets: Buffer, data: Buffer, null_bitmap: Buffer | None=None, null_count: int=-1, offset: int=0): ... + +class StringViewArray(Array): ... + +class BinaryArray(Array): + @property + def total_values_length(self): ... + +class LargeBinaryArray(Array): + @property + def total_values_length(self): ... + +class BinaryViewArray(Array): ... + +class DictionaryArray(Array): + def dictionary_encode(self): ... + def dictionary_decode(self): ... + @property + def dictionary(self): ... + @property + def indices(self): ... + @staticmethod + def from_buffers(type: DataType, length: int, buffers, dictionary: Array, null_count: int=-1, offset: int=0): ... + @staticmethod + def from_arrays(indices, dictionary, mask=None, ordered: bool=False, from_pandas: bool=False, safe: bool=True, memory_pool: MemoryPool | None=None): ... + +class StructArray(Array): + def field(self, index): ... + def _flattened_field(self, index, memory_pool: MemoryPool | None=None): ... + def flatten(self, memory_pool: MemoryPool | None=None): ... + @staticmethod + def from_arrays(arrays, names=None, fields=None, mask=None, memory_pool=None, type=None): ... + def sort(self, order='ascending', null_placement='at_end', by=None, **kwargs): ... + +class RunEndEncodedArray(Array): + @staticmethod + def _from_arrays(type, allow_none_for_type, logical_length, run_ends, values, logical_offset): ... + @staticmethod + def from_arrays(run_ends, values, type=None): ... + @staticmethod + def from_buffers(type: DataType, length, buffers, null_count=-1, offset=0, children=None): ... + @property + def run_ends(self): ... + @property + def values(self): ... + def find_physical_offset(self): ... + def find_physical_length(self): ... + +class ExtensionArray(Array): + @property + def storage(self): ... + @staticmethod + def from_storage(typ: BaseExtensionType, storage: Array): ... + +class JsonArray(ExtensionArray): ... + +class UuidArray(ExtensionArray): ... + +class FixedShapeTensorArray(ExtensionArray): + def to_numpy_ndarray(self): ... + @staticmethod + def from_tensor(tensor: Tensor): ... + @staticmethod + def from_numpy_ndarray(obj, dim_names=None): ... + @staticmethod + def from_dlpack(x, /, *, device=None, copy=None): ... + def __dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None): ... + +class OpaqueArray(ExtensionArray): ... + +class Bool8Array(ExtensionArray): + def to_numpy(self, zero_copy_only=True, writable=False): ... + @staticmethod + def from_storage(storage: Int8Array): ... + @staticmethod + def from_numpy(obj): ... + +class StringBuilder(_Weakrefable): + def __init__(self, memory_pool: MemoryPool | None=None): ... + def append(self, value): ... + def append_values(self, values): ... + def finish(self): ... + @property + def null_count(self): ... + def __len__(self): ... + +class StringViewBuilder(_Weakrefable): + def __init__(self, memory_pool: MemoryPool | None=None): ... + def append(self, value): ... + def append_values(self, values): ... + def finish(self): ... + @property + def null_count(self): ... + def __len__(self): ... + +class ChunkedArray(_PandasConvertible): + _name: object + + def __init__(self): ... + def __reduce__(self): ... + @property + def data(self): ... + @property + def type(self): ... + def length(self): ... + def __len__(self): ... + def __repr__(self): ... + def to_string(self, *, indent: int=0, window: int=5, container_window: int=2, skip_new_lines: c_bool=False, element_size_limit: int=100): ... + def format(self, **kwargs): ... + def __str__(self): ... + def validate(self, *, full=False): ... + @property + def null_count(self): ... + @property + def nbytes(self): ... + def get_total_buffer_size(self): ... + def __sizeof__(self): ... + def __iter__(self): ... + def __getitem__(self, key): ... + def is_null(self, *, nan_is_null=False): ... + def is_nan(self): ... + def is_valid(self): ... + def __eq__(self, other): ... + def fill_null(self, fill_value): ... + def equals(self, other: ChunkedArray): ... + def _to_pandas(self, options, types_mapper=None, **kwargs): ... + def to_numpy(self, zero_copy_only=False): ... + def __array__(self, dtype=None, copy=None): ... + def cast(self, target_type: object | None=None, safe=None, options=None): ... + def dictionary_encode(self, null_encoding='mask'): ... + def flatten(self, memory_pool: MemoryPool | None=None): ... + def combine_chunks(self, memory_pool: MemoryPool | None=None): ... + def unique(self): ... + def value_counts(self): ... + def slice(self, offset=0, length=None): ... + def filter(self, mask, null_selection_behavior: object='drop'): ... + def index(self, value, start=None, end=None, *, memory_pool=None): ... + def take(self, indices: object): ... + def drop_null(self): ... + def sort(self, order='ascending', null_placement='at_end', **kwargs): ... + def unify_dictionaries(self, memory_pool: MemoryPool | None=None): ... + @property + def num_chunks(self): ... + def chunk(self, i): ... + @property + def chunks(self): ... + def iterchunks(self): ... + def to_pylist(self, *, maps_as_pydicts=None): ... + def __arrow_c_stream__(self, requested_schema=None): ... + @staticmethod + def _import_from_c_capsule(stream): ... + @property + def is_cpu(self): ... + def _assert_cpu(self): ... + +class _Tabular(_PandasConvertible): + def __init__(self): ... + def __array__(self, dtype=None, copy=None): ... + def __dataframe__(self, nan_as_null: bool=False, allow_copy: bool=True): ... + def __eq__(self, other): ... + def __getitem__(self, key): ... + def __len__(self): ... + def __repr__(self): ... + def _column(self, i: int): ... + def _ensure_integer_index(self, i): ... + def _is_initialized(self): ... + def column(self, i): ... + @property + def column_names(self): ... + @property + def columns(self): ... + def drop_null(self): ... + def field(self, i): ... + @classmethod + def from_pydict(cls, mapping, schema=None, metadata=None): ... + @classmethod + def from_pylist(cls, mapping, schema=None, metadata=None): ... + def itercolumns(self): ... + @property + def num_columns(self): ... + @property + def num_rows(self): ... + @property + def shape(self): ... + @property + def schema(self): ... + def sort_by(self, sorting, **kwargs): ... + def take(self, indices: object): ... + def filter(self, mask, null_selection_behavior: object='drop'): ... + def to_pydict(self, *, maps_as_pydicts=None): ... + def to_pylist(self, *, maps_as_pydicts=None): ... + def to_string(self, *, show_metadata=False, preview_cols=0): ... + def remove_column(self, i: int): ... + def drop_columns(self, columns): ... + def add_column(self, i: int, field_, column): ... + def append_column(self, field_, column): ... + +class RecordBatch(_Tabular): + def __init__(self): ... + def _is_initialized(self): ... + def __reduce__(self): ... + def validate(self, *, full=False): ... + def replace_schema_metadata(self, metadata=None): ... + @property + def num_columns(self): ... + @property + def num_rows(self): ... + @property + def schema(self): ... + def _column(self, i: int): ... + @property + def nbytes(self): ... + def get_total_buffer_size(self): ... + def __sizeof__(self): ... + def add_column(self, i: int, field_, column): ... + def remove_column(self, i: int): ... + def set_column(self, i: int, field_, column): ... + def rename_columns(self, names): ... + def serialize(self, memory_pool=None): ... + def slice(self, offset=0, length=None): ... + def equals(self, other: object, check_metadata: bool=False): ... + def select(self, columns: object): ... + def cast(self, target_schema: Schema, safe=None, options=None): ... + def _to_pandas(self, options, **kwargs): ... + @classmethod + def from_pandas(cls, df, schema: Schema | None=None, preserve_index=None, nthreads=None, columns=None): ... + @staticmethod + def from_arrays(arrays: list[Any], names=None, schema=None, metadata=None): ... + @staticmethod + def from_struct_array(struct_array: StructArray): ... + def to_struct_array(self): ... + def to_tensor(self, null_to_nan: c_bool=False, row_major: c_bool=True, memory_pool: MemoryPool | None=None): ... + def copy_to(self, destination): ... + def _export_to_c(self, out_ptr, out_schema_ptr=0): ... + @staticmethod + def _import_from_c(in_ptr, schema): ... + def __arrow_c_array__(self, requested_schema=None): ... + def __arrow_c_stream__(self, requested_schema=None): ... + @staticmethod + def _import_from_c_capsule(schema_capsule, array_capsule): ... + def _export_to_c_device(self, out_ptr, out_schema_ptr=0): ... + @staticmethod + def _import_from_c_device(in_ptr, schema): ... + def __arrow_c_device_array__(self, requested_schema=None, **kwargs): ... + @staticmethod + def _import_from_c_device_capsule(schema_capsule, array_capsule): ... + @property + def device_type(self): ... + @property + def is_cpu(self): ... + +class Table(_Tabular): + def __init__(self): ... + def _is_initialized(self): ... + def validate(self, *, full=False): ... + def __reduce__(self): ... + def slice(self, offset=0, length=None): ... + def select(self, columns: object): ... + def replace_schema_metadata(self, metadata=None): ... + def flatten(self, memory_pool: MemoryPool | None=None): ... + def combine_chunks(self, memory_pool: MemoryPool | None=None): ... + def unify_dictionaries(self, memory_pool: MemoryPool | None=None): ... + def equals(self, other: Table, check_metadata: bool=False): ... + def cast(self, target_schema: Schema, safe=None, options=None): ... + @classmethod + def from_pandas(cls, df, schema: Schema | None=None, preserve_index=None, nthreads=None, columns=None, safe: bool=True): ... + @staticmethod + def from_arrays(arrays, names=None, schema=None, metadata=None): ... + @staticmethod + def from_struct_array(struct_array): ... + def to_struct_array(self, max_chunksize=None): ... + @staticmethod + def from_batches(batches, schema: Schema | None=None): ... + def to_batches(self, max_chunksize=None): ... + def to_tensor(self, null_to_nan: c_bool=False, row_major: c_bool=True, memory_pool: MemoryPool | None=None): ... + def to_reader(self, max_chunksize=None): ... + def _to_pandas(self, options, categories=None, ignore_metadata=False, types_mapper=None): ... + @property + def schema(self): ... + def _column(self, i: int): ... + @property + def num_columns(self): ... + @property + def num_rows(self): ... + @property + def nbytes(self): ... + def get_total_buffer_size(self): ... + def __sizeof__(self): ... + def add_column(self, i: int, field_, column): ... + def remove_column(self, i: int): ... + def set_column(self, i: int, field_, column): ... + def rename_columns(self, names): ... + def drop(self, columns): ... + def group_by(self, keys, use_threads=True): ... + def join(self, right_table, keys, right_keys=None, join_type='left outer', left_suffix=None, right_suffix=None, coalesce_keys=True, use_threads=True, filter_expression=None): ... + def join_asof(self, right_table, on, by, tolerance, right_on=None, right_by=None): ... + def __arrow_c_stream__(self, requested_schema=None): ... + @property + def is_cpu(self): ... + +class TableGroupBy: + def __init__(self, table, keys, use_threads=True): ... + def aggregate(self, aggregations): ... + +class Tensor(_Weakrefable): + type: DataType + _ssize_t_shape: bytes + _ssize_t_strides: bytes + + def __init__(self): ... + def _make_shape_or_strides_buffer(self, values): ... + def __repr__(self): ... + @staticmethod + def from_numpy(obj, dim_names=None): ... + def to_numpy(self): ... + def equals(self, other: Tensor): ... + def __eq__(self, other): ... + def dim_name(self, i): ... + @property + def dim_names(self): ... + @property + def is_mutable(self): ... + @property + def is_contiguous(self): ... + @property + def ndim(self): ... + @property + def size(self): ... + @property + def shape(self): ... + @property + def strides(self): ... + def __getbuffer__(self, buffer: cp.Py_buffer, flags: int): ... + @staticmethod + def from_dlpack(x, /, *, device=None, copy=None): ... + def __dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None): ... + def __dlpack_device__(self): ... + +class SparseCOOTensor(_Weakrefable): + type: DataType + + def __init__(self): ... + def __repr__(self): ... + @classmethod + def from_dense_numpy(cls, obj, dim_names=None): ... + @staticmethod + def from_numpy(data, coords, shape, dim_names=None): ... + @staticmethod + def from_scipy(obj, dim_names=None): ... + @staticmethod + def from_pydata_sparse(obj, dim_names=None): ... + @staticmethod + def from_tensor(obj): ... + def to_numpy(self): ... + def to_scipy(self): ... + def to_pydata_sparse(self): ... + def to_tensor(self): ... + def equals(self, other: SparseCOOTensor): ... + def __eq__(self, other): ... + @property + def is_mutable(self): ... + @property + def ndim(self): ... + @property + def shape(self): ... + @property + def size(self): ... + def dim_name(self, i): ... + @property + def dim_names(self): ... + @property + def non_zero_length(self): ... + @property + def has_canonical_format(self): ... + +class SparseCSRMatrix(_Weakrefable): + type: DataType + + def __init__(self): ... + def __repr__(self): ... + @classmethod + def from_dense_numpy(cls, obj, dim_names=None): ... + @staticmethod + def from_numpy(data, indptr, indices, shape, dim_names=None): ... + @staticmethod + def from_scipy(obj, dim_names=None): ... + @staticmethod + def from_tensor(obj): ... + def to_numpy(self): ... + def to_scipy(self): ... + def to_tensor(self): ... + def equals(self, other: SparseCSRMatrix): ... + def __eq__(self, other): ... + @property + def is_mutable(self): ... + @property + def ndim(self): ... + @property + def shape(self): ... + @property + def size(self): ... + def dim_name(self, i): ... + @property + def dim_names(self): ... + @property + def non_zero_length(self): ... + +class SparseCSCMatrix(_Weakrefable): + type: DataType + + def __init__(self): ... + def __repr__(self): ... + @classmethod + def from_dense_numpy(cls, obj, dim_names=None): ... + @staticmethod + def from_numpy(data, indptr, indices, shape, dim_names=None): ... + @staticmethod + def from_scipy(obj, dim_names=None): ... + @staticmethod + def from_tensor(obj): ... + def to_numpy(self): ... + def to_scipy(self): ... + def to_tensor(self): ... + def equals(self, other: SparseCSCMatrix): ... + def __eq__(self, other): ... + @property + def is_mutable(self): ... + @property + def ndim(self): ... + @property + def shape(self): ... + @property + def size(self): ... + def dim_name(self, i): ... + @property + def dim_names(self): ... + @property + def non_zero_length(self): ... + +class SparseCSFTensor(_Weakrefable): + type: DataType + + def __init__(self): ... + def __repr__(self): ... + @classmethod + def from_dense_numpy(cls, obj, dim_names=None): ... + @staticmethod + def from_numpy(data, indptr, indices, shape, axis_order=None, dim_names=None): ... + @staticmethod + def from_tensor(obj): ... + def to_numpy(self): ... + def to_tensor(self): ... + def equals(self, other: SparseCSFTensor): ... + def __eq__(self, other): ... + @property + def is_mutable(self): ... + @property + def ndim(self): ... + @property + def shape(self): ... + @property + def size(self): ... + def dim_name(self, i): ... + @property + def dim_names(self): ... + @property + def non_zero_length(self): ... + +class NativeFile(_Weakrefable): + _default_chunk_size = 256 * 1024 + + def __init__(self): ... + def __dealloc__(self): ... + def __enter__(self): ... + def __exit__(self, exc_type, exc_value, tb): ... + def __repr__(self): ... + @property + def mode(self): ... + def readable(self): ... + def writable(self): ... + def seekable(self): ... + def isatty(self): ... + def fileno(self): ... + @property + def closed(self): ... + def close(self): ... + def _assert_open(self): ... + def _assert_readable(self): ... + def _assert_writable(self): ... + def _assert_seekable(self): ... + def size(self): ... + def metadata(self): ... + def tell(self): ... + def seek(self, position: int, whence: int=0): ... + def flush(self): ... + def write(self, data): ... + def read(self, nbytes=None): ... + def get_stream(self, file_offset, nbytes): ... + def read_at(self, nbytes, offset): ... + def read1(self, nbytes=None): ... + def readall(self): ... + def readinto(self, b): ... + def readline(self, size=None): ... + def readlines(self, hint=None): ... + def __iter__(self): ... + def __next__(self): ... + def read_buffer(self, nbytes=None): ... + def truncate(self): ... + def writelines(self, lines): ... + def download(self, stream_or_path, buffer_size=None): ... + def _download_nothreads(self, stream_or_path, buffer_size=None): ... + def upload(self, stream, buffer_size=None): ... + def _upload_nothreads(self, stream, buffer_size=None): ... + +class PythonFile(NativeFile): + def __init__(self, handle, mode=None): ... + def truncate(self, pos=None): ... + def readline(self, size=None): ... + def readlines(self, hint=None): ... + +class MemoryMappedFile(NativeFile): + @staticmethod + def create(path, size): ... + def _open(self, path, mode='r'): ... + def resize(self, new_size): ... + def fileno(self): ... + +class OSFile(NativeFile): + def __init__(self, path, mode='r', memory_pool: MemoryPool | None=None): ... + def fileno(self): ... + +class FixedSizeBufferWriter(NativeFile): + def __init__(self, buffer: Buffer): ... + def set_memcopy_threads(self, num_threads: int): ... + def set_memcopy_blocksize(self, blocksize: int): ... + def set_memcopy_threshold(self, threshold: int): ... + +class Buffer(_Weakrefable): + def __init__(self): ... + def __len__(self): ... + def __repr__(self): ... + def _assert_cpu(self): ... + @property + def size(self): ... + @property + def address(self): ... + def hex(self): ... + @property + def is_mutable(self): ... + @property + def is_cpu(self): ... + @property + def device(self): ... + @property + def memory_manager(self): ... + @property + def device_type(self): ... + @property + def parent(self): ... + def __getitem__(self, key): ... + def slice(self, offset=0, length=None): ... + def equals(self, other: Buffer): ... + def __eq__(self, other): ... + def __reduce_ex__(self, protocol): ... + def to_pybytes(self): ... + def __getbuffer__(self, buffer: cp.Py_buffer, flags: int): ... + +class ResizableBuffer(Buffer): + def resize(self, new_size: int, shrink_to_fit=False): ... + +class BufferOutputStream(NativeFile): + def __init__(self, memory_pool: MemoryPool | None=None): ... + def getvalue(self): ... + +class MockOutputStream(NativeFile): + def __init__(self): ... + def size(self): ... + +class BufferReader(NativeFile): + def __init__(self, obj): ... + +class CompressedInputStream(NativeFile): + def __init__(self, stream: object, compression: str): ... + +class CompressedOutputStream(NativeFile): + def __init__(self, stream: object, compression: str): ... + +class BufferedInputStream(NativeFile): + def __init__(self, stream: NativeFile, buffer_size: int, memory_pool: MemoryPool | None=None): ... + def detach(self): ... + +class BufferedOutputStream(NativeFile): + def __init__(self, stream: NativeFile, buffer_size: int, memory_pool: MemoryPool | None=None): ... + def detach(self): ... + +class TransformInputStream(NativeFile): + def __init__(self, stream: NativeFile, transform_func): ... + +class Transcoder: + def __init__(self, decoder, encoder): ... + def __call__(self, buf): ... + +class CacheOptions(_Weakrefable): + def __init__(self, *, hole_size_limit=None, range_size_limit=None, lazy=None, prefetch_limit=None): ... + @property + def hole_size_limit(self): ... + @hole_size_limit.setter + def hole_size_limit(self, hole_size_limit): ... + @property + def range_size_limit(self): ... + @range_size_limit.setter + def range_size_limit(self, range_size_limit): ... + @property + def lazy(self): ... + @lazy.setter + def lazy(self, lazy): ... + @property + def prefetch_limit(self): ... + @prefetch_limit.setter + def prefetch_limit(self, prefetch_limit): ... + def __eq__(self, other: CacheOptions): ... + @staticmethod + def from_network_metrics(time_to_first_byte_millis, transfer_bandwidth_mib_per_sec, ideal_bandwidth_utilization_frac=0.9, max_ideal_request_size_mib=64): ... + @staticmethod + def _reconstruct(kwargs): ... + def __reduce__(self): ... + +class Codec(_Weakrefable): + def __init__(self, compression: str, compression_level=None): ... + @staticmethod + def detect(path): ... + @staticmethod + def is_available(compression: str): ... + @staticmethod + def supports_compression_level(compression: str): ... + @staticmethod + def default_compression_level(compression: str): ... + @staticmethod + def minimum_compression_level(compression: str): ... + @staticmethod + def maximum_compression_level(compression: str): ... + @property + def name(self): ... + @property + def compression_level(self): ... + def compress(self, buf: object, asbytes=False, memory_pool=None): ... + def decompress(self, buf: object, decompressed_size=None, asbytes=False, memory_pool=None): ... + def __repr__(self): ... + +class WriteStats(_WriteStats): + __slots__ = () + +class ReadStats(_ReadStats): + __slots__ = () + +class IpcReadOptions(_Weakrefable): + __slots__ = () + + def __init__(self, *, ensure_native_endian: bool=True, ensure_alignment: Alignment=Alignment.Any, use_threads: bool=True, included_fields: list[Any] | None=None): ... + @property + def ensure_native_endian(self): ... + @ensure_native_endian.setter + def ensure_native_endian(self, value: bool): ... + @property + def ensure_alignment(self): ... + @ensure_alignment.setter + def ensure_alignment(self, value: Alignment): ... + @property + def use_threads(self): ... + @use_threads.setter + def use_threads(self, value: bool): ... + @property + def included_fields(self): ... + @included_fields.setter + def included_fields(self, value: list[Any]): ... + def __repr__(self): ... + +class IpcWriteOptions(_Weakrefable): + __slots__ = () + + def __init__(self, *, metadata_version=MetadataVersion.V5, allow_64bit: bool=False, use_legacy_format=False, compression=None, use_threads: bool=True, emit_dictionary_deltas: bool=False, unify_dictionaries: bool=False): ... + @property + def allow_64bit(self): ... + @allow_64bit.setter + def allow_64bit(self, value: bool): ... + @property + def use_legacy_format(self): ... + @use_legacy_format.setter + def use_legacy_format(self, value: bool): ... + @property + def metadata_version(self): ... + @metadata_version.setter + def metadata_version(self, value): ... + @property + def compression(self): ... + @compression.setter + def compression(self, value): ... + @property + def use_threads(self): ... + @use_threads.setter + def use_threads(self, value: bool): ... + @property + def emit_dictionary_deltas(self): ... + @emit_dictionary_deltas.setter + def emit_dictionary_deltas(self, value: bool): ... + @property + def unify_dictionaries(self): ... + @unify_dictionaries.setter + def unify_dictionaries(self, value: bool): ... + def __repr__(self): ... + +class Message(_Weakrefable): + def __init__(self): ... + @property + def type(self): ... + @property + def metadata(self): ... + @property + def metadata_version(self): ... + @property + def body(self): ... + def equals(self, other: Message): ... + def serialize_to(self, sink: NativeFile, alignment=8, memory_pool=None): ... + def serialize(self, alignment=8, memory_pool=None): ... + def __repr__(self): ... + +class MessageReader(_Weakrefable): + def __init__(self): ... + @staticmethod + def open_stream(source): ... + def __iter__(self): ... + def __next__(self): ... + def read_next_message(self): ... + +class _CRecordBatchWriter(_Weakrefable): + def write(self, table_or_batch): ... + def write_batch(self, batch: RecordBatch, custom_metadata=None): ... + def write_table(self, table: Table, max_chunksize=None): ... + def close(self): ... + def __enter__(self): ... + def __exit__(self, exc_type, exc_val, exc_tb): ... + @property + def stats(self): ... + +class _RecordBatchStreamWriter(_CRecordBatchWriter): + def __init__(self): ... + def __dealloc__(self): ... + @property + def _use_legacy_format(self): ... + @property + def _metadata_version(self): ... + def _open(self, sink, schema: Schema, options: IpcWriteOptions=IpcWriteOptions()): ... + +class _ReadPandasMixin: + def read_pandas(self, **options): ... + +class RecordBatchReader(_Weakrefable): + read_pandas = _ReadPandasMixin.read_pandas + + def __init__(self): ... + def __iter__(self): ... + def __next__(self): ... + @property + def schema(self): ... + def read_next_batch(self): ... + def read_next_batch_with_custom_metadata(self): ... + def iter_batches_with_custom_metadata(self): ... + def read_all(self): ... + def close(self): ... + def __enter__(self): ... + def __exit__(self, exc_type, exc_val, exc_tb): ... + def cast(self, target_schema): ... + def _export_to_c(self, out_ptr): ... + @staticmethod + def _import_from_c(in_ptr): ... + def __arrow_c_stream__(self, requested_schema=None): ... + @staticmethod + def _import_from_c_capsule(stream): ... + @staticmethod + def from_stream(data, schema=None): ... + @staticmethod + def from_batches(schema: Schema, batches): ... + +class _RecordBatchStreamReader(RecordBatchReader): + def __init__(self): ... + def _open(self, source, options: IpcReadOptions=IpcReadOptions(), memory_pool: MemoryPool | None=None): ... + @property + def stats(self): ... + +class _RecordBatchFileWriter(_RecordBatchStreamWriter): + def _open(self, sink, schema: Schema, options: IpcWriteOptions=IpcWriteOptions(), metadata=None): ... + +class RecordBatchWithMetadata(_RecordBatchWithMetadata): + __slots__ = () + +class _RecordBatchFileReader(_Weakrefable): + get_record_batch = get_batch + read_pandas = _ReadPandasMixin.read_pandas + schema: Schema + + def __init__(self): ... + def _open(self, source, footer_offset=None, options: IpcReadOptions=IpcReadOptions(), memory_pool: MemoryPool | None=None): ... + @property + def num_record_batches(self): ... + def count_rows(self): ... + def get_batch(self, i: int): ... + def get_batch_with_custom_metadata(self, i: int): ... + def read_all(self): ... + def __enter__(self): ... + def __exit__(self, exc_type, exc_value, traceback): ... + @property + def stats(self): ... + @property + def metadata(self): ... + +class _Weakrefable: ... + +class _Metadata(_Weakrefable): ... + +class Decimal32Array(FixedSizeBinaryArray): ... + +def cpu_count(): ... +def set_cpu_count(count: int): ... +def is_opentelemetry_enabled() -> bool: ... +def is_threading_enabled() -> bool: ... +def _pc(): ... +def _pac(): ... +def _ensure_cuda_loaded(): ... +def _gdb_test_session(): ... +def encode_file_path(path): ... +def tobytes(o): ... +def frombytes(o, *, safe=False): ... +def enable_signal_handlers(enable: c_bool): ... +def runtime_info(): ... +def _build_info(): ... +def set_timezone_db_path(path): ... +def default_memory_pool(): ... +def proxy_memory_pool(parent: MemoryPool): ... +def logging_memory_pool(parent: MemoryPool): ... +def system_memory_pool(): ... +def jemalloc_memory_pool(): ... +def mimalloc_memory_pool(): ... +def set_memory_pool(pool: MemoryPool): ... +def log_memory_allocations(enable=True): ... +def total_allocated_bytes(): ... +def jemalloc_set_decay_ms(decay_ms): ... +def supported_memory_backends(): ... +def default_cpu_memory_manager(): ... +def _get_pandas_type_map(): ... +def _is_primitive(type: Type): ... +def _is_nested(type: Type): ... +def _get_pandas_type(arrow_type, coerce_to_ns=False): ... +def _get_pandas_tz_type(arrow_type, coerce_to_ns=False): ... +def _to_pandas_dtype(arrow_type, options=None): ... +def register_extension_type(ext_type): ... +def unregister_extension_type(type_name): ... +def ensure_metadata(meta: object, allow_none: c_bool=False) -> KeyValueMetadata: ... +def unify_schemas(schemas, *, promote_options='default'): ... +def field(name, type=None, nullable=None, metadata=None): ... +def null(): ... +def bool_(): ... +def uint8(): ... +def int8(): ... +def uint16(): ... +def int16(): ... +def uint32(): ... +def int32(): ... +def uint64(): ... +def int64(): ... +def tzinfo_to_string(tz): ... +def string_to_tzinfo(name, *, prefer_zoneinfo=True): ... +def timestamp(unit, tz=None): ... +def time32(unit): ... +def time64(unit): ... +def duration(unit): ... +def month_day_nano_interval(): ... +def date32(): ... +def date64(): ... +def float16(): ... +def float32(): ... +def float64(): ... +def decimal32(precision: int, scale: int=0) -> DataType: ... +def decimal64(precision: int, scale: int=0) -> DataType: ... +def decimal128(precision: int, scale: int=0) -> DataType: ... +def decimal256(precision: int, scale: int=0) -> DataType: ... +def string(): ... +def utf8(): ... +def binary(length: int=-1): ... +def large_binary(): ... +def large_string(): ... +def large_utf8(): ... +def binary_view(): ... +def string_view(): ... +def list_(value_type, list_size: int=-1): ... +def large_list(value_type) -> LargeListType: ... +def list_view(value_type) -> ListViewType: ... +def large_list_view(value_type) -> LargeListViewType: ... +def map_(key_type, item_type, keys_sorted=False) -> MapType: ... +def dictionary(index_type, value_type, ordered: bool=False) -> DictionaryType: ... +def struct(fields): ... +def sparse_union(child_fields, type_codes=None): ... +def dense_union(child_fields, type_codes=None): ... +def union(child_fields, mode, type_codes=None): ... +def run_end_encoded(run_end_type, value_type): ... +def json_(storage_type: DataType=utf8()): ... +def uuid(): ... +def fixed_shape_tensor(value_type: DataType, shape, dim_names=None, permutation=None): ... +def bool8(): ... +def opaque(storage_type: DataType, type_name: str, vendor_name: str): ... +def type_for_alias(name): ... +def ensure_type(ty: object, allow_none: bool=False) -> DataType: ... +def schema(fields, metadata=None): ... +def from_numpy_dtype(dtype: object): ... +def is_boolean_value(obj: object): ... +def is_integer_value(obj: object): ... +def is_float_value(obj: object): ... +def _datetime_from_int(value: int, unit: TimeUnit, tzinfo=None): ... +def scalar(value, type=None, *, from_pandas=None, memory_pool: MemoryPool | None=None): ... +def _ndarray_to_arrow_type(values: object, type: DataType): ... +def _handle_arrow_array_protocol(obj, type, mask, size): ... +def array(obj: object, type=None, mask=None, size=None, from_pandas=None, safe: bool=True, memory_pool: MemoryPool | None=None): ... +def asarray(values, type=None): ... +def nulls(size, type=None, memory_pool: MemoryPool | None=None): ... +def repeat(value, size, memory_pool: MemoryPool | None=None): ... +def infer_type(values, mask=None, from_pandas=False): ... +def arange(start: int, stop: int, step: int=1, *, memory_pool=None): ... +def _normalize_slice(arrow_obj: object, key: slice): ... +def _restore_array(data): ... +def _compute_binary_op(func_name, left, right): ... +def concat_arrays(arrays, memory_pool: MemoryPool | None=None): ... +def _empty_array(type: DataType): ... +def chunked_array(arrays, type=None): ... +def _reconstruct_record_batch(columns, schema): ... +def table_to_blocks(options, table: Table, categories, extension_columns): ... +def _reconstruct_table(arrays, schema): ... +def record_batch(data, names=None, schema=None, metadata=None): ... +def table(data, names=None, schema=None, metadata=None, nthreads=None): ... +def concat_tables(tables, memory_pool: MemoryPool | None=None, promote_options: str='none', **kwargs): ... +def concat_batches(recordbatches, memory_pool: MemoryPool | None=None): ... +def _from_pydict(cls, mapping, schema, metadata): ... +def _from_pylist(cls, mapping, schema, metadata): ... +def io_thread_count(): ... +def set_io_thread_count(count: int): ... +def memory_map(path, mode='r'): ... +def create_memory_map(path, size): ... +def allocate_buffer(size: int, memory_pool: MemoryPool | None=None, resizable=False): ... +def transcoding_input_stream(stream, src_encoding, dest_encoding): ... +def py_buffer(obj: object): ... +def foreign_buffer(address, size, base=None): ... +def as_buffer(o: object): ... +def _detect_compression(path): ... +def compress(buf: object, codec='lz4', asbytes=False, memory_pool=None): ... +def decompress(buf: object, decompressed_size=None, codec='lz4', asbytes=False, memory_pool=None): ... +def input_stream(source, compression='detect', buffer_size=None): ... +def output_stream(source, compression='detect', buffer_size=None): ... +def get_tensor_size(tensor: Tensor): ... +def get_record_batch_size(batch: RecordBatch): ... +def write_tensor(tensor: Tensor, dest: NativeFile): ... +def read_tensor(source): ... +def read_message(source): ... +def read_schema(obj, dictionary_memo: DictionaryMemo | None=None): ... +def read_record_batch(obj, schema: Schema, dictionary_memo: DictionaryMemo | None=None): ... +def benchmark_PandasObjectIsNull(obj: list[Any]): ... From b6b73f5cfbe08320b5c899e0d037709460f7af41 Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Fri, 9 Oct 2026 18:13:47 +0100 Subject: [PATCH 3/8] [Python] Make the Cython stubs match the runtime The sed rules of scripts/generate_stubs.sh: drop special methods and C typedefs without a Python counterpart, add the mod argument and the reflected methods of operator slots, declare unresolved names Incomplete, declare __slots__ and TracingServerMiddleware.trace_context, and shorten the provenance comment. Append the symbols stubgen-pyx cannot derive from the sources: NA and _NULL, parquet_encryption_enabled and py_substrait. --- python/pyarrow-stubs/pyarrow/_acero.pyi | 3 +- python/pyarrow-stubs/pyarrow/_azurefs.pyi | 3 +- python/pyarrow-stubs/pyarrow/_compute.pyi | 35 +++-- python/pyarrow-stubs/pyarrow/_csv.pyi | 3 +- python/pyarrow-stubs/pyarrow/_cuda.pyi | 4 +- python/pyarrow-stubs/pyarrow/_dataset.pyi | 3 +- python/pyarrow-stubs/pyarrow/_dataset_orc.pyi | 3 +- .../pyarrow/_dataset_parquet.pyi | 6 +- .../pyarrow/_dataset_parquet_encryption.pyi | 3 +- python/pyarrow-stubs/pyarrow/_feather.pyi | 3 +- python/pyarrow-stubs/pyarrow/_flight.pyi | 4 +- python/pyarrow-stubs/pyarrow/_fs.pyi | 3 +- python/pyarrow-stubs/pyarrow/_gcsfs.pyi | 3 +- python/pyarrow-stubs/pyarrow/_hdfs.pyi | 3 +- python/pyarrow-stubs/pyarrow/_json.pyi | 3 +- python/pyarrow-stubs/pyarrow/_orc.pyi | 9 +- python/pyarrow-stubs/pyarrow/_parquet.pyi | 3 +- .../pyarrow/_parquet_encryption.pyi | 3 +- .../pyarrow/_pyarrow_cpp_tests.pyi | 5 +- python/pyarrow-stubs/pyarrow/_s3fs.pyi | 3 +- python/pyarrow-stubs/pyarrow/_substrait.pyi | 6 +- python/pyarrow-stubs/pyarrow/gandiva.pyi | 3 +- python/pyarrow-stubs/pyarrow/lib.pyi | 138 ++++++++++-------- 23 files changed, 143 insertions(+), 109 deletions(-) diff --git a/python/pyarrow-stubs/pyarrow/_acero.pyi b/python/pyarrow-stubs/pyarrow/_acero.pyi index 9a45895ff7be..ba6b37dbc95e 100644 --- a/python/pyarrow-stubs/pyarrow/_acero.pyi +++ b/python/pyarrow-stubs/pyarrow/_acero.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_acero.pyx +# Generated by stubgen-pyx from python/pyarrow/_acero.pyx +from _typeshed import Incomplete from pyarrow._compute import Expression from pyarrow.includes.common import * diff --git a/python/pyarrow-stubs/pyarrow/_azurefs.pyi b/python/pyarrow-stubs/pyarrow/_azurefs.pyi index 8c5529dfa843..367472e95887 100644 --- a/python/pyarrow-stubs/pyarrow/_azurefs.pyi +++ b/python/pyarrow-stubs/pyarrow/_azurefs.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_azurefs.pyx +# Generated by stubgen-pyx from python/pyarrow/_azurefs.pyx +from _typeshed import Incomplete from pyarrow._fs import FileSystem from pyarrow.includes.libarrow_fs import * diff --git a/python/pyarrow-stubs/pyarrow/_compute.pyi b/python/pyarrow-stubs/pyarrow/_compute.pyi index 7ce2cbe9460a..0a6a506da3fd 100644 --- a/python/pyarrow-stubs/pyarrow/_compute.pyi +++ b/python/pyarrow-stubs/pyarrow/_compute.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_compute.pyx +# Generated by stubgen-pyx from python/pyarrow/_compute.pyx +from _typeshed import Incomplete from collections import namedtuple from typing import Any, Callable @@ -14,7 +15,6 @@ _substrait_msg = 'The pyarrow installation is not built with support for Substra SUPPORTED_INPUT_ARR_TYPES = (list, tuple) FunctionDoc = namedtuple('FunctionDoc', ('summary', 'description', 'arg_names', 'options_class', 'options_required')) _deserialize = Expression._deserialize -CRegisterUdf: TypeAlias = Callable[[PyObject, function[CallbackUdf], CUdfOptions, CFunctionRegistry], CStatus] class Kernel(_Weakrefable): def __init__(self): ... @@ -32,7 +32,7 @@ class HashAggregateKernel(Kernel): def __repr__(self): ... class Function(_Weakrefable): - _kind_map = ... + _kind_map: Incomplete def __init__(self): ... def __repr__(self): ... @@ -152,7 +152,7 @@ class RoundToMultipleOptions(_RoundToMultipleOptions): def __init__(self, multiple=1.0, round_mode='half_to_even'): ... class _JoinOptions(FunctionOptions): - _null_handling_map = ... + _null_handling_map: Incomplete def _set_options(self, null_handling, null_replacement): ... @@ -220,7 +220,7 @@ class ReplaceSliceOptions(_ReplaceSliceOptions): def __init__(self, start, stop, replacement): ... class _FilterOptions(FunctionOptions): - _null_selection_map = ... + _null_selection_map: Incomplete def _set_options(self, null_selection_behavior): ... @@ -228,7 +228,7 @@ class FilterOptions(_FilterOptions): def __init__(self, null_selection_behavior='drop'): ... class _DictionaryEncodeOptions(FunctionOptions): - _null_encoding_map = ... + _null_encoding_map: Incomplete def _set_options(self, null_encoding): ... @@ -278,7 +278,7 @@ class ScalarAggregateOptions(_ScalarAggregateOptions): def __init__(self, *, skip_nulls=True, min_count=1): ... class _CountOptions(FunctionOptions): - _mode_map = ... + _mode_map: Incomplete def _set_options(self, mode): ... @@ -292,7 +292,7 @@ class IndexOptions(_IndexOptions): def __init__(self, value): ... class _MapLookupOptions(FunctionOptions): - _occurrence_map = ... + _occurrence_map: Incomplete def _set_options(self, query_key, occurrence): ... @@ -312,7 +312,7 @@ class SetLookupOptions(_SetLookupOptions): def __init__(self, value_set, *, skip_nulls=False): ... class _StrptimeOptions(FunctionOptions): - _unit_map = ... + _unit_map: Incomplete def _set_options(self, format, unit, error_is_null): ... @@ -338,8 +338,8 @@ class WeekOptions(_WeekOptions): def __init__(self, *, week_starts_monday=True, count_from_zero=False, first_week_is_fully_in_year=False): ... class _AssumeTimezoneOptions(FunctionOptions): - _ambiguous_map = ... - _nonexistent_map = ... + _ambiguous_map: Incomplete + _nonexistent_map: Incomplete def _set_options(self, timezone, ambiguous, nonexistent): ... @@ -434,7 +434,7 @@ class SelectKOptions(_SelectKOptions): def __init__(self, k, sort_keys): ... class _QuantileOptions(FunctionOptions): - _interp_map = ... + _interp_map: Incomplete def _set_options(self, quantiles, interp, skip_nulls, min_count): ... @@ -448,7 +448,7 @@ class TDigestOptions(_TDigestOptions): def __init__(self, q=0.5, *, delta=100, buffer_size=500, skip_nulls=True, min_count=0): ... class _Utf8NormalizeOptions(FunctionOptions): - _form_map = ... + _form_map: Incomplete def _set_options(self, form): ... @@ -462,7 +462,7 @@ class RandomOptions(_RandomOptions): def __init__(self, *, initializer='system'): ... class _RankOptions(FunctionOptions): - _tiebreaker_map = ... + _tiebreaker_map: Incomplete def _set_options(self, sort_keys, null_placement, tiebreaker): ... @@ -494,15 +494,20 @@ class Expression(_Weakrefable): def __reduce__(self): ... @staticmethod def _call(function_name: str, arguments: list[Any], options: FunctionOptions | None=None): ... - def __richcmp__(self, other, op: int): ... def __bool__(self): ... def __invert__(self): ... def __and__(self: Expression, other): ... + def __rand__(self: Expression, other): ... def __or__(self: Expression, other): ... + def __ror__(self: Expression, other): ... def __add__(self: Expression, other): ... + def __radd__(self: Expression, other): ... def __mul__(self: Expression, other): ... + def __rmul__(self: Expression, other): ... def __sub__(self: Expression, other): ... + def __rsub__(self: Expression, other): ... def __truediv__(self: Expression, other): ... + def __rtruediv__(self: Expression, other): ... def is_valid(self): ... def is_null(self, nan_is_null: bool=False): ... def is_nan(self): ... diff --git a/python/pyarrow-stubs/pyarrow/_csv.pyi b/python/pyarrow-stubs/pyarrow/_csv.pyi index 80d81b7ff330..acacdc4922c7 100644 --- a/python/pyarrow-stubs/pyarrow/_csv.pyi +++ b/python/pyarrow-stubs/pyarrow/_csv.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_csv.pyx +# Generated by stubgen-pyx from python/pyarrow/_csv.pyx +from _typeshed import Incomplete from collections import namedtuple diff --git a/python/pyarrow-stubs/pyarrow/_cuda.pyi b/python/pyarrow-stubs/pyarrow/_cuda.pyi index ee268f28786a..0bba62778e79 100644 --- a/python/pyarrow-stubs/pyarrow/_cuda.pyi +++ b/python/pyarrow-stubs/pyarrow/_cuda.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_cuda.pyx +# Generated by stubgen-pyx from python/pyarrow/_cuda.pyx +from _typeshed import Incomplete from pyarrow.includes.common import * from pyarrow.includes.libarrow import * @@ -51,7 +52,6 @@ class CudaBuffer(Buffer): def context(self): ... def slice(self, offset=0, length=None): ... def to_pybytes(self): ... - def __getbuffer__(self, buffer: cp.Py_buffer, flags: int): ... class HostBuffer(Buffer): def __init__(self): ... diff --git a/python/pyarrow-stubs/pyarrow/_dataset.pyi b/python/pyarrow-stubs/pyarrow/_dataset.pyi index 7b1e38cd8dd4..d020e2a8ed98 100644 --- a/python/pyarrow-stubs/pyarrow/_dataset.pyi +++ b/python/pyarrow-stubs/pyarrow/_dataset.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_dataset.pyx +# Generated by stubgen-pyx from python/pyarrow/_dataset.pyx +from _typeshed import Incomplete import collections from typing import Any diff --git a/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi b/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi index e64547c8a965..199e17e188bc 100644 --- a/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi +++ b/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_dataset_orc.pyx +# Generated by stubgen-pyx from python/pyarrow/_dataset_orc.pyx +from _typeshed import Incomplete from pyarrow._dataset import FileFormat from pyarrow.includes.libarrow import * diff --git a/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi b/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi index cb84f890be0f..d0102528dfe4 100644 --- a/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi +++ b/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_dataset_parquet.pyx +# Generated by stubgen-pyx from python/pyarrow/_dataset_parquet.pyx +from _typeshed import Incomplete from typing import Any @@ -14,7 +15,6 @@ from pyarrow.includes.libarrow_dataset_parquet import * from pyarrow.lib import * from typing_extensions import TypeAlias -_CParquetFileWriterPtr: TypeAlias = CParquetFileWriter class ParquetFileFormat(FileFormat): def __init__(self, read_options=None, default_fragment_scan_options=None, **kwargs): ... @@ -157,3 +157,5 @@ class ParquetFactoryOptions(_Weakrefable): class ParquetDatasetFactory(DatasetFactory): def __init__(self, metadata_path, filesystem: FileSystem, format: FileFormat, options: ParquetFactoryOptions | None=None): ... + +parquet_encryption_enabled: bool diff --git a/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi b/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi index 5f863e9deb3c..e4c8500826d6 100644 --- a/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi +++ b/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_dataset_parquet_encryption.pyx +# Generated by stubgen-pyx from python/pyarrow/_dataset_parquet_encryption.pyx +from _typeshed import Incomplete from pyarrow._dataset_parquet import (ParquetFileWriteOptions, ParquetFragmentScanOptions) diff --git a/python/pyarrow-stubs/pyarrow/_feather.pyi b/python/pyarrow-stubs/pyarrow/_feather.pyi index 24cbe0124462..699764c9d026 100644 --- a/python/pyarrow-stubs/pyarrow/_feather.pyi +++ b/python/pyarrow-stubs/pyarrow/_feather.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_feather.pyx +# Generated by stubgen-pyx from python/pyarrow/_feather.pyx +from _typeshed import Incomplete from pyarrow.includes.common import * from pyarrow.includes.libarrow import * diff --git a/python/pyarrow-stubs/pyarrow/_flight.pyi b/python/pyarrow-stubs/pyarrow/_flight.pyi index 6415d2305353..612030dc5892 100644 --- a/python/pyarrow-stubs/pyarrow/_flight.pyi +++ b/python/pyarrow-stubs/pyarrow/_flight.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_flight.pyx +# Generated by stubgen-pyx from python/pyarrow/_flight.pyx +from _typeshed import Incomplete import collections import enum @@ -323,6 +324,7 @@ class ServerMiddleware(_Weakrefable): def call_completed(self, exception): ... class TracingServerMiddleware(ServerMiddleware): + trace_context: Incomplete __slots__ = ['trace_context'] def __init__(self, trace_context): ... diff --git a/python/pyarrow-stubs/pyarrow/_fs.pyi b/python/pyarrow-stubs/pyarrow/_fs.pyi index c5e539a5c3f4..1609c31f6b37 100644 --- a/python/pyarrow-stubs/pyarrow/_fs.pyi +++ b/python/pyarrow-stubs/pyarrow/_fs.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_fs.pyx +# Generated by stubgen-pyx from python/pyarrow/_fs.pyx +from _typeshed import Incomplete from abc import ABC, abstractmethod from datetime import datetime diff --git a/python/pyarrow-stubs/pyarrow/_gcsfs.pyi b/python/pyarrow-stubs/pyarrow/_gcsfs.pyi index 53423286ed97..99067b959485 100644 --- a/python/pyarrow-stubs/pyarrow/_gcsfs.pyi +++ b/python/pyarrow-stubs/pyarrow/_gcsfs.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_gcsfs.pyx +# Generated by stubgen-pyx from python/pyarrow/_gcsfs.pyx +from _typeshed import Incomplete from pyarrow._fs import FileSystem from pyarrow.includes.common import * diff --git a/python/pyarrow-stubs/pyarrow/_hdfs.pyi b/python/pyarrow-stubs/pyarrow/_hdfs.pyi index 0aba678d6025..9d526298fa5e 100644 --- a/python/pyarrow-stubs/pyarrow/_hdfs.pyi +++ b/python/pyarrow-stubs/pyarrow/_hdfs.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_hdfs.pyx +# Generated by stubgen-pyx from python/pyarrow/_hdfs.pyx +from _typeshed import Incomplete from pyarrow._fs import FileSystem from pyarrow.includes.common import * diff --git a/python/pyarrow-stubs/pyarrow/_json.pyi b/python/pyarrow-stubs/pyarrow/_json.pyi index 4a00403ed74c..64b0c2de30ad 100644 --- a/python/pyarrow-stubs/pyarrow/_json.pyi +++ b/python/pyarrow-stubs/pyarrow/_json.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_json.pyx +# Generated by stubgen-pyx from python/pyarrow/_json.pyx +from _typeshed import Incomplete from pyarrow.includes.common import * from pyarrow.includes.libarrow import * diff --git a/python/pyarrow-stubs/pyarrow/_orc.pyi b/python/pyarrow-stubs/pyarrow/_orc.pyi index 91fba12ca14d..0880ad15ffed 100644 --- a/python/pyarrow-stubs/pyarrow/_orc.pyi +++ b/python/pyarrow-stubs/pyarrow/_orc.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_orc.pyx +# Generated by stubgen-pyx from python/pyarrow/_orc.pyx +from _typeshed import Incomplete from typing import TypedDict @@ -9,12 +10,6 @@ from pyarrow.includes.libarrow import CCompressionType from pyarrow.lib import MemoryPool, Table, _Weakrefable from typing_extensions import TypeAlias -CompressionStrategy: TypeAlias = int -WriterId: TypeAlias = int -WriterVersion: TypeAlias = int -FileVersion: TypeAlias = Incomplete -ORCFileReader: TypeAlias = Incomplete -ORCFileWriter: TypeAlias = Incomplete class ORCReader(_Weakrefable): def __init__(self, memory_pool: MemoryPool | None=None): ... diff --git a/python/pyarrow-stubs/pyarrow/_parquet.pyi b/python/pyarrow-stubs/pyarrow/_parquet.pyi index b34fe8bd6df4..494f6ea9a9d2 100644 --- a/python/pyarrow-stubs/pyarrow/_parquet.pyi +++ b/python/pyarrow-stubs/pyarrow/_parquet.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_parquet.pyx +# Generated by stubgen-pyx from python/pyarrow/_parquet.pyx +from _typeshed import Incomplete from typing import Any diff --git a/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi b/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi index 0eabec387f63..c14ce445549c 100644 --- a/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi +++ b/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_parquet_encryption.pyx +# Generated by stubgen-pyx from python/pyarrow/_parquet_encryption.pyx +from _typeshed import Incomplete from typing import Any diff --git a/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi b/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi index 2baa854b53fe..e5f8211a4959 100644 --- a/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi +++ b/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_pyarrow_cpp_tests.pyx +# Generated by stubgen-pyx from python/pyarrow/_pyarrow_cpp_tests.pyx +from _typeshed import Incomplete from typing import Callable @@ -8,8 +9,6 @@ from pyarrow.includes.libarrow import * from pyarrow.includes.libarrow import CStatus from typing_extensions import TypeAlias -cb_test_func: TypeAlias = Callable[[], CStatus] -CTestCase: TypeAlias = Incomplete class CppTestCase: @property diff --git a/python/pyarrow-stubs/pyarrow/_s3fs.pyi b/python/pyarrow-stubs/pyarrow/_s3fs.pyi index f0d4c150fd68..c40968b1a95e 100644 --- a/python/pyarrow-stubs/pyarrow/_s3fs.pyi +++ b/python/pyarrow-stubs/pyarrow/_s3fs.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_s3fs.pyx +# Generated by stubgen-pyx from python/pyarrow/_s3fs.pyx +from _typeshed import Incomplete from enum import IntEnum diff --git a/python/pyarrow-stubs/pyarrow/_substrait.pyi b/python/pyarrow-stubs/pyarrow/_substrait.pyi index 3b099457ff18..f68dae186cbc 100644 --- a/python/pyarrow-stubs/pyarrow/_substrait.pyi +++ b/python/pyarrow-stubs/pyarrow/_substrait.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/_substrait.pyx +# Generated by stubgen-pyx from python/pyarrow/_substrait.pyx +from _typeshed import Incomplete from pyarrow.includes.libarrow import * from pyarrow.includes.libarrow_substrait import * @@ -26,3 +27,6 @@ def deserialize_schema(buf): ... def serialize_expressions(exprs, names, schema, *, allow_arrow_extensions=False): ... def deserialize_expressions(buf): ... def get_supported_functions(): ... + +from types import ModuleType +py_substrait: ModuleType | None diff --git a/python/pyarrow-stubs/pyarrow/gandiva.pyi b/python/pyarrow-stubs/pyarrow/gandiva.pyi index e4cb7360332b..3b7ca3be7e1d 100644 --- a/python/pyarrow-stubs/pyarrow/gandiva.pyi +++ b/python/pyarrow-stubs/pyarrow/gandiva.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/gandiva.pyx +# Generated by stubgen-pyx from python/pyarrow/gandiva.pyx +from _typeshed import Incomplete from pyarrow.includes.libarrow import * from pyarrow.lib import (DataType, Field, MemoryPool, RecordBatch, Schema, diff --git a/python/pyarrow-stubs/pyarrow/lib.pyi b/python/pyarrow-stubs/pyarrow/lib.pyi index 21ea20375f14..357830b43bba 100644 --- a/python/pyarrow-stubs/pyarrow/lib.pyi +++ b/python/pyarrow-stubs/pyarrow/lib.pyi @@ -1,4 +1,5 @@ -# This file was generated by stubgen-pyx v0.2.23 from python/pyarrow/lib.pyx +# Generated by stubgen-pyx from python/pyarrow/lib.pyx +from _typeshed import Incomplete import sys from collections import namedtuple @@ -40,53 +41,53 @@ class Alignment(IntEnum): DataTypeSpecific = ... At64Byte = ... -MonthDayNano = ... -Type_NA = ... -Type_BOOL = ... -Type_UINT8 = ... -Type_INT8 = ... -Type_UINT16 = ... -Type_INT16 = ... -Type_UINT32 = ... -Type_INT32 = ... -Type_UINT64 = ... -Type_INT64 = ... -Type_HALF_FLOAT = ... -Type_FLOAT = ... -Type_DOUBLE = ... -Type_DECIMAL32 = ... -Type_DECIMAL64 = ... -Type_DECIMAL128 = ... -Type_DECIMAL256 = ... -Type_DATE32 = ... -Type_DATE64 = ... -Type_TIMESTAMP = ... -Type_TIME32 = ... -Type_TIME64 = ... -Type_DURATION = ... -Type_INTERVAL_MONTH_DAY_NANO = ... -Type_BINARY = ... -Type_STRING = ... -Type_LARGE_BINARY = ... -Type_LARGE_STRING = ... -Type_FIXED_SIZE_BINARY = ... -Type_BINARY_VIEW = ... -Type_STRING_VIEW = ... -Type_LIST = ... -Type_LARGE_LIST = ... -Type_LIST_VIEW = ... -Type_LARGE_LIST_VIEW = ... -Type_MAP = ... -Type_FIXED_SIZE_LIST = ... -Type_STRUCT = ... -Type_SPARSE_UNION = ... -Type_DENSE_UNION = ... -Type_DICTIONARY = ... -Type_RUN_END_ENCODED = ... -Type_INTERVAL_MONTHS = ... -Type_INTERVAL_DAY_TIME = ... -UnionMode_SPARSE = ... -UnionMode_DENSE = ... +MonthDayNano: Incomplete +Type_NA: Incomplete +Type_BOOL: Incomplete +Type_UINT8: Incomplete +Type_INT8: Incomplete +Type_UINT16: Incomplete +Type_INT16: Incomplete +Type_UINT32: Incomplete +Type_INT32: Incomplete +Type_UINT64: Incomplete +Type_INT64: Incomplete +Type_HALF_FLOAT: Incomplete +Type_FLOAT: Incomplete +Type_DOUBLE: Incomplete +Type_DECIMAL32: Incomplete +Type_DECIMAL64: Incomplete +Type_DECIMAL128: Incomplete +Type_DECIMAL256: Incomplete +Type_DATE32: Incomplete +Type_DATE64: Incomplete +Type_TIMESTAMP: Incomplete +Type_TIME32: Incomplete +Type_TIME64: Incomplete +Type_DURATION: Incomplete +Type_INTERVAL_MONTH_DAY_NANO: Incomplete +Type_BINARY: Incomplete +Type_STRING: Incomplete +Type_LARGE_BINARY: Incomplete +Type_LARGE_STRING: Incomplete +Type_FIXED_SIZE_BINARY: Incomplete +Type_BINARY_VIEW: Incomplete +Type_STRING_VIEW: Incomplete +Type_LIST: Incomplete +Type_LARGE_LIST: Incomplete +Type_LIST_VIEW: Incomplete +Type_LARGE_LIST_VIEW: Incomplete +Type_MAP: Incomplete +Type_FIXED_SIZE_LIST: Incomplete +Type_STRUCT: Incomplete +Type_SPARSE_UNION: Incomplete +Type_DENSE_UNION: Incomplete +Type_DICTIONARY: Incomplete +Type_RUN_END_ENCODED: Incomplete +Type_INTERVAL_MONTHS: Incomplete +Type_INTERVAL_DAY_TIME: Incomplete +UnionMode_SPARSE: Incomplete +UnionMode_DENSE: Incomplete __pc = None __pac = None __cuda_loaded = None @@ -101,19 +102,13 @@ build_info = _build_info() cpp_build_info = build_info.cpp_build_info cpp_version = build_info.cpp_build_info.version cpp_version_info = build_info.cpp_build_info.version_info -_pandas_api = ... -_CFixedWidthTypePtr: TypeAlias = CFixedWidthType +_pandas_api: Incomplete _python_extension_types_registry = [] _registry_nanny = _ExtensionRegistryNanny() -_CSparseCOOIndexPtr: TypeAlias = CSparseCOOIndex DEFAULT_BUFFER_SIZE = 2 ** 16 -_CBufferedInputStreamPtr: TypeAlias = CBufferedInputStream -_CBufferedOutputStreamPtr: TypeAlias = CBufferedOutputStream -_RandomAccessFilePtr: TypeAlias = CRandomAccessFile _WriteStats = namedtuple('WriteStats', ('num_messages', 'num_record_batches', 'num_dictionary_batches', 'num_dictionary_deltas', 'num_replaced_dictionaries')) _ReadStats = namedtuple('ReadStats', ('num_messages', 'num_record_batches', 'num_dictionary_batches', 'num_dictionary_deltas', 'num_replaced_dictionaries')) _RecordBatchWithMetadata = namedtuple('RecordBatchWithMetadata', ('batch', 'custom_metadata')) -_CPyExtensionTypePtr: TypeAlias = CPyExtensionType class ArrowException(Exception): ... @@ -144,7 +139,6 @@ class SignalStopHandler: def _init_signals(self): ... def __enter__(self): ... def __exit__(self, exc_type, exc_value, exc_tb): ... - def __dealloc__(self): ... @property def stop_token(self): ... @@ -597,16 +591,26 @@ class Scalar(_Weakrefable): def as_py(self, *, maps_as_pydicts=None): ... def __abs__(self): ... def __add__(self, other: object): ... + def __radd__(self, other: object): ... def __truediv__(self, other: object): ... + def __rtruediv__(self, other: object): ... def __mul__(self, other: object): ... + def __rmul__(self, other: object): ... def __neg__(self): ... - def __pow__(self, other: object): ... + def __pow__(self, other: object, mod: None = None): ... + def __rpow__(self, other: object, mod: None = None): ... def __sub__(self, other: object): ... + def __rsub__(self, other: object): ... def __and__(self, other: object): ... + def __rand__(self, other: object): ... def __or__(self, other: object): ... + def __ror__(self, other: object): ... def __xor__(self, other: object): ... + def __rxor__(self, other: object): ... def __lshift__(self, other: object): ... + def __rlshift__(self, other: object): ... def __rshift__(self, other: object): ... + def __rrshift__(self, other: object): ... class NullScalar(Scalar): def __init__(self): ... @@ -715,7 +719,6 @@ class BinaryScalar(Scalar): def as_buffer(self): ... def as_py(self, *, maps_as_pydicts=None): ... def __bytes__(self): ... - def __getbuffer__(self, buffer: cp.Py_buffer, flags: int): ... class LargeBinaryScalar(BinaryScalar): ... @@ -910,16 +913,26 @@ class Array(_PandasConvertible): def statistics(self): ... def __abs__(self): ... def __add__(self, other: object): ... + def __radd__(self, other: object): ... def __truediv__(self, other: object): ... + def __rtruediv__(self, other: object): ... def __mul__(self, other: object): ... + def __rmul__(self, other: object): ... def __neg__(self): ... - def __pow__(self, other: object): ... + def __pow__(self, other: object, mod: None = None): ... + def __rpow__(self, other: object, mod: None = None): ... def __sub__(self, other: object): ... + def __rsub__(self, other: object): ... def __and__(self, other: object): ... + def __rand__(self, other: object): ... def __or__(self, other: object): ... + def __ror__(self, other: object): ... def __xor__(self, other: object): ... + def __rxor__(self, other: object): ... def __lshift__(self, other: object): ... + def __rlshift__(self, other: object): ... def __rshift__(self, other: object): ... + def __rrshift__(self, other: object): ... class NullArray(Array): ... @@ -1387,7 +1400,6 @@ class Tensor(_Weakrefable): def shape(self): ... @property def strides(self): ... - def __getbuffer__(self, buffer: cp.Py_buffer, flags: int): ... @staticmethod def from_dlpack(x, /, *, device=None, copy=None): ... def __dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None): ... @@ -1527,7 +1539,6 @@ class NativeFile(_Weakrefable): _default_chunk_size = 256 * 1024 def __init__(self): ... - def __dealloc__(self): ... def __enter__(self): ... def __exit__(self, exc_type, exc_value, tb): ... def __repr__(self): ... @@ -1620,7 +1631,6 @@ class Buffer(_Weakrefable): def __eq__(self, other): ... def __reduce_ex__(self, protocol): ... def to_pybytes(self): ... - def __getbuffer__(self, buffer: cp.Py_buffer, flags: int): ... class ResizableBuffer(Buffer): def resize(self, new_size: int, shrink_to_fit=False): ... @@ -1801,7 +1811,6 @@ class _CRecordBatchWriter(_Weakrefable): class _RecordBatchStreamWriter(_CRecordBatchWriter): def __init__(self): ... - def __dealloc__(self): ... @property def _use_legacy_format(self): ... @property @@ -2018,3 +2027,6 @@ def read_message(source): ... def read_schema(obj, dictionary_memo: DictionaryMemo | None=None): ... def read_record_batch(obj, schema: Schema, dictionary_memo: DictionaryMemo | None=None): ... def benchmark_PandasObjectIsNull(obj: list[Any]): ... + +_NULL: NullScalar +NA: NullScalar From a1b3bcf3bc57b30f0a6a638f94e6e13c82591f5b Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Fri, 9 Oct 2026 18:13:53 +0100 Subject: [PATCH 4/8] [Python] Generate stubs for the Python modules with mypy stubgen The pure Python modules are parsed without importing them, so optional dependencies are not needed; pyarrow.compute builds its functions at import time and is inspected. NULL and EXCLUDED_PARQUET_PATHS are appended, as stubgen drops imports of private names and variables outside __all__. --- python/pyarrow-stubs/pyarrow/__init__.pyi | 18 ++ .../pyarrow/_compute_docstrings.pyi | 3 + python/pyarrow-stubs/pyarrow/acero.pyi | 16 + python/pyarrow-stubs/pyarrow/benchmark.pyi | 1 + python/pyarrow-stubs/pyarrow/cffi.pyi | 4 + python/pyarrow-stubs/pyarrow/compute.pyi | 305 ++++++++++++++++++ python/pyarrow-stubs/pyarrow/csv.pyi | 1 + python/pyarrow-stubs/pyarrow/cuda.pyi | 1 + python/pyarrow-stubs/pyarrow/dataset.pyi | 25 ++ python/pyarrow-stubs/pyarrow/feather.pyi | 24 ++ python/pyarrow-stubs/pyarrow/flight.pyi | 1 + python/pyarrow-stubs/pyarrow/fs.pyi | 39 +++ .../pyarrow/interchange/__init__.pyi | 1 + .../pyarrow/interchange/buffer.pyi | 24 ++ .../pyarrow/interchange/column.pyi | 67 ++++ .../pyarrow/interchange/dataframe.pyi | 23 ++ .../pyarrow/interchange/from_dataframe.pyi | 20 ++ python/pyarrow-stubs/pyarrow/ipc.pyi | 29 ++ python/pyarrow-stubs/pyarrow/json.pyi | 1 + python/pyarrow-stubs/pyarrow/jvm.pyi | 17 + python/pyarrow-stubs/pyarrow/orc.pyi | 61 ++++ .../pyarrow-stubs/pyarrow/pandas_compat.pyi | 45 +++ .../pyarrow/parquet/__init__.pyi | 1 + python/pyarrow-stubs/pyarrow/parquet/core.pyi | 87 +++++ .../pyarrow/parquet/encryption.pyi | 1 + python/pyarrow-stubs/pyarrow/substrait.pyi | 1 + python/pyarrow-stubs/pyarrow/types.pyi | 115 +++++++ python/pyarrow-stubs/pyarrow/util.pyi | 16 + 28 files changed, 947 insertions(+) create mode 100644 python/pyarrow-stubs/pyarrow/__init__.pyi create mode 100644 python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi create mode 100644 python/pyarrow-stubs/pyarrow/acero.pyi create mode 100644 python/pyarrow-stubs/pyarrow/benchmark.pyi create mode 100644 python/pyarrow-stubs/pyarrow/cffi.pyi create mode 100644 python/pyarrow-stubs/pyarrow/compute.pyi create mode 100644 python/pyarrow-stubs/pyarrow/csv.pyi create mode 100644 python/pyarrow-stubs/pyarrow/cuda.pyi create mode 100644 python/pyarrow-stubs/pyarrow/dataset.pyi create mode 100644 python/pyarrow-stubs/pyarrow/feather.pyi create mode 100644 python/pyarrow-stubs/pyarrow/flight.pyi create mode 100644 python/pyarrow-stubs/pyarrow/fs.pyi create mode 100644 python/pyarrow-stubs/pyarrow/interchange/__init__.pyi create mode 100644 python/pyarrow-stubs/pyarrow/interchange/buffer.pyi create mode 100644 python/pyarrow-stubs/pyarrow/interchange/column.pyi create mode 100644 python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi create mode 100644 python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi create mode 100644 python/pyarrow-stubs/pyarrow/ipc.pyi create mode 100644 python/pyarrow-stubs/pyarrow/json.pyi create mode 100644 python/pyarrow-stubs/pyarrow/jvm.pyi create mode 100644 python/pyarrow-stubs/pyarrow/orc.pyi create mode 100644 python/pyarrow-stubs/pyarrow/pandas_compat.pyi create mode 100644 python/pyarrow-stubs/pyarrow/parquet/__init__.pyi create mode 100644 python/pyarrow-stubs/pyarrow/parquet/core.pyi create mode 100644 python/pyarrow-stubs/pyarrow/parquet/encryption.pyi create mode 100644 python/pyarrow-stubs/pyarrow/substrait.pyi create mode 100644 python/pyarrow-stubs/pyarrow/types.pyi create mode 100644 python/pyarrow-stubs/pyarrow/util.pyi diff --git a/python/pyarrow-stubs/pyarrow/__init__.pyi b/python/pyarrow-stubs/pyarrow/__init__.pyi new file mode 100644 index 000000000000..25da79a19730 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/__init__.pyi @@ -0,0 +1,18 @@ +from pyarrow.ipc import Message as Message, MessageReader as MessageReader, MetadataVersion as MetadataVersion, RecordBatchFileReader as RecordBatchFileReader, RecordBatchFileWriter as RecordBatchFileWriter, RecordBatchStreamReader as RecordBatchStreamReader, RecordBatchStreamWriter as RecordBatchStreamWriter, deserialize_pandas as deserialize_pandas, serialize_pandas as serialize_pandas +from pyarrow.lib import Array as Array, ArrowCancelled as ArrowCancelled, ArrowCapacityError as ArrowCapacityError, ArrowException as ArrowException, ArrowIOError as ArrowIOError, ArrowIndexError as ArrowIndexError, ArrowInvalid as ArrowInvalid, ArrowKeyError as ArrowKeyError, ArrowMemoryError as ArrowMemoryError, ArrowNotImplementedError as ArrowNotImplementedError, ArrowSerializationError as ArrowSerializationError, ArrowTypeError as ArrowTypeError, BaseExtensionType as BaseExtensionType, BinaryArray as BinaryArray, BinaryScalar as BinaryScalar, BinaryViewArray as BinaryViewArray, BinaryViewScalar as BinaryViewScalar, Bool8Array as Bool8Array, Bool8Scalar as Bool8Scalar, Bool8Type as Bool8Type, BooleanArray as BooleanArray, BooleanScalar as BooleanScalar, Buffer as Buffer, BufferOutputStream as BufferOutputStream, BufferReader as BufferReader, BufferedInputStream as BufferedInputStream, BufferedOutputStream as BufferedOutputStream, BuildInfo as BuildInfo, CacheOptions as CacheOptions, ChunkedArray as ChunkedArray, Codec as Codec, CompressedInputStream as CompressedInputStream, CompressedOutputStream as CompressedOutputStream, CppBuildInfo as CppBuildInfo, DataType as DataType, Date32Array as Date32Array, Date32Scalar as Date32Scalar, Date64Array as Date64Array, Date64Scalar as Date64Scalar, Decimal128Array as Decimal128Array, Decimal128Scalar as Decimal128Scalar, Decimal128Type as Decimal128Type, Decimal256Array as Decimal256Array, Decimal256Scalar as Decimal256Scalar, Decimal256Type as Decimal256Type, Decimal32Array as Decimal32Array, Decimal32Scalar as Decimal32Scalar, Decimal32Type as Decimal32Type, Decimal64Array as Decimal64Array, Decimal64Scalar as Decimal64Scalar, Decimal64Type as Decimal64Type, DenseUnionType as DenseUnionType, Device as Device, DeviceAllocationType as DeviceAllocationType, DictionaryArray as DictionaryArray, DictionaryMemo as DictionaryMemo, DictionaryScalar as DictionaryScalar, DictionaryType as DictionaryType, DoubleArray as DoubleArray, DoubleScalar as DoubleScalar, DurationArray as DurationArray, DurationScalar as DurationScalar, DurationType as DurationType, ExtensionArray as ExtensionArray, ExtensionScalar as ExtensionScalar, ExtensionType as ExtensionType, Field as Field, FixedShapeTensorArray as FixedShapeTensorArray, FixedShapeTensorScalar as FixedShapeTensorScalar, FixedShapeTensorType as FixedShapeTensorType, FixedSizeBinaryArray as FixedSizeBinaryArray, FixedSizeBinaryScalar as FixedSizeBinaryScalar, FixedSizeBinaryType as FixedSizeBinaryType, FixedSizeBufferWriter as FixedSizeBufferWriter, FixedSizeListArray as FixedSizeListArray, FixedSizeListScalar as FixedSizeListScalar, FixedSizeListType as FixedSizeListType, FloatArray as FloatArray, FloatScalar as FloatScalar, FloatingPointArray as FloatingPointArray, HalfFloatArray as HalfFloatArray, HalfFloatScalar as HalfFloatScalar, Int16Array as Int16Array, Int16Scalar as Int16Scalar, Int32Array as Int32Array, Int32Scalar as Int32Scalar, Int64Array as Int64Array, Int64Scalar as Int64Scalar, Int8Array as Int8Array, Int8Scalar as Int8Scalar, IntegerArray as IntegerArray, JsonArray as JsonArray, JsonScalar as JsonScalar, JsonType as JsonType, KeyValueMetadata as KeyValueMetadata, LargeBinaryArray as LargeBinaryArray, LargeBinaryScalar as LargeBinaryScalar, LargeListArray as LargeListArray, LargeListScalar as LargeListScalar, LargeListType as LargeListType, LargeListViewArray as LargeListViewArray, LargeListViewScalar as LargeListViewScalar, LargeListViewType as LargeListViewType, LargeStringArray as LargeStringArray, LargeStringScalar as LargeStringScalar, ListArray as ListArray, ListScalar as ListScalar, ListType as ListType, ListViewArray as ListViewArray, ListViewScalar as ListViewScalar, ListViewType as ListViewType, LoggingMemoryPool as LoggingMemoryPool, MapArray as MapArray, MapScalar as MapScalar, MapType as MapType, MemoryManager as MemoryManager, MemoryMappedFile as MemoryMappedFile, MemoryPool as MemoryPool, MockOutputStream as MockOutputStream, MonthDayNano as MonthDayNano, MonthDayNanoIntervalArray as MonthDayNanoIntervalArray, MonthDayNanoIntervalScalar as MonthDayNanoIntervalScalar, NA as NA, NativeFile as NativeFile, NullArray as NullArray, NullScalar as NullScalar, NumericArray as NumericArray, OSFile as OSFile, OpaqueArray as OpaqueArray, OpaqueScalar as OpaqueScalar, OpaqueType as OpaqueType, ProxyMemoryPool as ProxyMemoryPool, PythonFile as PythonFile, RecordBatch as RecordBatch, RecordBatchReader as RecordBatchReader, ResizableBuffer as ResizableBuffer, RunEndEncodedArray as RunEndEncodedArray, RunEndEncodedScalar as RunEndEncodedScalar, RunEndEncodedType as RunEndEncodedType, RuntimeInfo as RuntimeInfo, Scalar as Scalar, Schema as Schema, SparseCOOTensor as SparseCOOTensor, SparseCSCMatrix as SparseCSCMatrix, SparseCSFTensor as SparseCSFTensor, SparseCSRMatrix as SparseCSRMatrix, SparseUnionType as SparseUnionType, StringArray as StringArray, StringScalar as StringScalar, StringViewArray as StringViewArray, StringViewScalar as StringViewScalar, StructArray as StructArray, StructScalar as StructScalar, StructType as StructType, Table as Table, TableGroupBy as TableGroupBy, Tensor as Tensor, Time32Array as Time32Array, Time32Scalar as Time32Scalar, Time32Type as Time32Type, Time64Array as Time64Array, Time64Scalar as Time64Scalar, Time64Type as Time64Type, TimestampArray as TimestampArray, TimestampScalar as TimestampScalar, TimestampType as TimestampType, TransformInputStream as TransformInputStream, UInt16Array as UInt16Array, UInt16Scalar as UInt16Scalar, UInt32Array as UInt32Array, UInt32Scalar as UInt32Scalar, UInt64Array as UInt64Array, UInt64Scalar as UInt64Scalar, UInt8Array as UInt8Array, UInt8Scalar as UInt8Scalar, UnionArray as UnionArray, UnionScalar as UnionScalar, UnionType as UnionType, UnknownExtensionType as UnknownExtensionType, UuidArray as UuidArray, UuidScalar as UuidScalar, UuidType as UuidType, VersionInfo as VersionInfo, allocate_buffer as allocate_buffer, arange as arange, array as array, binary as binary, binary_view as binary_view, bool8 as bool8, bool_ as bool_, build_info as build_info, chunked_array as chunked_array, compress as compress, concat_arrays as concat_arrays, concat_batches as concat_batches, concat_tables as concat_tables, cpp_build_info as cpp_build_info, cpp_version as cpp_version, cpp_version_info as cpp_version_info, cpu_count as cpu_count, create_memory_map as create_memory_map, date32 as date32, date64 as date64, decimal128 as decimal128, decimal256 as decimal256, decimal32 as decimal32, decimal64 as decimal64, decompress as decompress, default_cpu_memory_manager as default_cpu_memory_manager, default_memory_pool as default_memory_pool, dense_union as dense_union, dictionary as dictionary, duration as duration, enable_signal_handlers as enable_signal_handlers, field as field, fixed_shape_tensor as fixed_shape_tensor, float16 as float16, float32 as float32, float64 as float64, foreign_buffer as foreign_buffer, from_numpy_dtype as from_numpy_dtype, infer_type as infer_type, input_stream as input_stream, int16 as int16, int32 as int32, int64 as int64, int8 as int8, io_thread_count as io_thread_count, is_opentelemetry_enabled as is_opentelemetry_enabled, jemalloc_memory_pool as jemalloc_memory_pool, jemalloc_set_decay_ms as jemalloc_set_decay_ms, json_ as json_, large_binary as large_binary, large_list as large_list, large_list_view as large_list_view, large_string as large_string, large_utf8 as large_utf8, list_ as list_, list_view as list_view, log_memory_allocations as log_memory_allocations, logging_memory_pool as logging_memory_pool, map_ as map_, memory_map as memory_map, mimalloc_memory_pool as mimalloc_memory_pool, month_day_nano_interval as month_day_nano_interval, null as null, nulls as nulls, opaque as opaque, output_stream as output_stream, proxy_memory_pool as proxy_memory_pool, py_buffer as py_buffer, record_batch as record_batch, register_extension_type as register_extension_type, repeat as repeat, run_end_encoded as run_end_encoded, runtime_info as runtime_info, scalar as scalar, schema as schema, set_cpu_count as set_cpu_count, set_io_thread_count as set_io_thread_count, set_memory_pool as set_memory_pool, set_timezone_db_path as set_timezone_db_path, sparse_union as sparse_union, string as string, string_view as string_view, struct as struct, supported_memory_backends as supported_memory_backends, system_memory_pool as system_memory_pool, table as table, time32 as time32, time64 as time64, timestamp as timestamp, total_allocated_bytes as total_allocated_bytes, transcoding_input_stream as transcoding_input_stream, type_for_alias as type_for_alias, uint16 as uint16, uint32 as uint32, uint64 as uint64, uint8 as uint8, unify_schemas as unify_schemas, union as union, unregister_extension_type as unregister_extension_type, utf8 as utf8, uuid as uuid +from pyarrow.util import _deprecate_api as _deprecate_api, _deprecate_class as _deprecate_class + +def parse_git(root, **kwargs): ... +def show_versions() -> None: ... +def _module_is_available(module): ... +def _filesystem_is_available(fs): ... +def have_libhdfs(): ... +def show_info() -> None: ... +def get_include(): ... +def _get_pkg_config_executable(): ... +def _has_pkg_config(pkgname): ... +def _read_pkg_config_variable(pkgname, cli_args): ... +def get_libraries(): ... +def create_library_symlinks(): ... +def get_library_dirs(): ... +from pyarrow.lib import _NULL as NULL diff --git a/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi b/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi new file mode 100644 index 000000000000..322bd8a07773 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi @@ -0,0 +1,3 @@ +from _typeshed import Incomplete + +function_doc_additions: Incomplete diff --git a/python/pyarrow-stubs/pyarrow/acero.pyi b/python/pyarrow-stubs/pyarrow/acero.pyi new file mode 100644 index 000000000000..81f825799fc7 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/acero.pyi @@ -0,0 +1,16 @@ +from pyarrow._acero import AggregateNodeOptions as AggregateNodeOptions, AsofJoinNodeOptions as AsofJoinNodeOptions, Declaration as Declaration, ExecNodeOptions as ExecNodeOptions, FilterNodeOptions as FilterNodeOptions, HashJoinNodeOptions as HashJoinNodeOptions, OrderByNodeOptions as OrderByNodeOptions, ProjectNodeOptions as ProjectNodeOptions, RecordBatchReaderSourceNodeOptions as RecordBatchReaderSourceNodeOptions, TableSourceNodeOptions as TableSourceNodeOptions +from pyarrow._dataset import ScanNodeOptions as ScanNodeOptions +from pyarrow.compute import Expression as Expression, field as field +from pyarrow.lib import RecordBatch as RecordBatch, Table as Table, array as array + +class DatasetModuleStub: + class Dataset: ... + class InMemoryDataset: ... +ds = DatasetModuleStub + +def _dataset_to_decl(dataset, use_threads: bool = True, implicit_ordering: bool = False): ... +def _perform_join(join_type, left_operand, left_keys, right_operand, right_keys, left_suffix=None, right_suffix=None, use_threads: bool = True, coalesce_keys: bool = False, output_type=..., filter_expression=None): ... +def _perform_join_asof(left_operand, left_on, left_by, right_operand, right_on, right_by, tolerance, use_threads: bool = True, output_type=...): ... +def _filter_table(table, expression): ... +def _sort_source(table_or_dataset, sort_keys, output_type=..., **kwargs): ... +def _group_by(table, aggregates, keys, use_threads: bool = True): ... diff --git a/python/pyarrow-stubs/pyarrow/benchmark.pyi b/python/pyarrow-stubs/pyarrow/benchmark.pyi new file mode 100644 index 000000000000..0d2a20d9ae7c --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/benchmark.pyi @@ -0,0 +1 @@ +from pyarrow.lib import benchmark_PandasObjectIsNull as benchmark_PandasObjectIsNull diff --git a/python/pyarrow-stubs/pyarrow/cffi.pyi b/python/pyarrow-stubs/pyarrow/cffi.pyi new file mode 100644 index 000000000000..3bf775cb5cd1 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/cffi.pyi @@ -0,0 +1,4 @@ +from _typeshed import Incomplete + +c_source: str +ffi: Incomplete diff --git a/python/pyarrow-stubs/pyarrow/compute.pyi b/python/pyarrow-stubs/pyarrow/compute.pyi new file mode 100644 index 000000000000..dad6b78caaad --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/compute.pyi @@ -0,0 +1,305 @@ +import pyarrow.lib +from _typeshed import Incomplete +from pyarrow._compute import ArraySortOptions as ArraySortOptions, AssumeTimezoneOptions as AssumeTimezoneOptions, CastOptions as CastOptions, CountOptions as CountOptions, CumulativeOptions as CumulativeOptions, CumulativeSumOptions as CumulativeSumOptions, DayOfWeekOptions as DayOfWeekOptions, DictionaryEncodeOptions as DictionaryEncodeOptions, ElementWiseAggregateOptions as ElementWiseAggregateOptions, Expression as Expression, ExtractRegexOptions as ExtractRegexOptions, ExtractRegexSpanOptions as ExtractRegexSpanOptions, FilterOptions as FilterOptions, Function as Function, FunctionOptions as FunctionOptions, FunctionRegistry as FunctionRegistry, HashAggregateFunction as HashAggregateFunction, HashAggregateKernel as HashAggregateKernel, IndexOptions as IndexOptions, InversePermutationOptions as InversePermutationOptions, JoinOptions as JoinOptions, Kernel as Kernel, ListFlattenOptions as ListFlattenOptions, ListSliceOptions as ListSliceOptions, MakeStructOptions as MakeStructOptions, MapLookupOptions as MapLookupOptions, MatchSubstringOptions as MatchSubstringOptions, ModeOptions as ModeOptions, NullOptions as NullOptions, PadOptions as PadOptions, PairwiseOptions as PairwiseOptions, PartitionNthOptions as PartitionNthOptions, PivotWiderOptions as PivotWiderOptions, QuantileOptions as QuantileOptions, RandomOptions as RandomOptions, RankOptions as RankOptions, RankQuantileOptions as RankQuantileOptions, ReplaceSliceOptions as ReplaceSliceOptions, ReplaceSubstringOptions as ReplaceSubstringOptions, RoundBinaryOptions as RoundBinaryOptions, RoundOptions as RoundOptions, RoundTemporalOptions as RoundTemporalOptions, RoundToMultipleOptions as RoundToMultipleOptions, RunEndEncodeOptions as RunEndEncodeOptions, ScalarAggregateFunction as ScalarAggregateFunction, ScalarAggregateKernel as ScalarAggregateKernel, ScalarAggregateOptions as ScalarAggregateOptions, ScalarFunction as ScalarFunction, ScalarKernel as ScalarKernel, ScatterOptions as ScatterOptions, SearchSortedOptions as SearchSortedOptions, SelectKOptions as SelectKOptions, SetLookupOptions as SetLookupOptions, SkewOptions as SkewOptions, SliceOptions as SliceOptions, SortOptions as SortOptions, SplitOptions as SplitOptions, SplitPatternOptions as SplitPatternOptions, StrftimeOptions as StrftimeOptions, StrptimeOptions as StrptimeOptions, StructFieldOptions as StructFieldOptions, TDigestOptions as TDigestOptions, TakeOptions as TakeOptions, TrimOptions as TrimOptions, UdfContext as UdfContext, Utf8NormalizeOptions as Utf8NormalizeOptions, VarianceOptions as VarianceOptions, VectorFunction as VectorFunction, VectorKernel as VectorKernel, WeekOptions as WeekOptions, WinsorizeOptions as WinsorizeOptions, ZeroFillOptions as ZeroFillOptions, call_function as call_function, call_tabular_function as call_tabular_function, function_registry as function_registry, get_function as get_function, list_functions as list_functions, register_aggregate_function as register_aggregate_function, register_scalar_function as register_scalar_function, register_tabular_function as register_tabular_function, register_vector_function as register_vector_function +from typing import ClassVar + +class _OptionsClassDoc(tuple): + _fields: ClassVar[tuple] = ... + _field_defaults: ClassVar[dict] = ... + __match_args__: ClassVar[tuple] = ... + params: Incomplete + def __init__(self, _cls, params) -> None: ... + def __replace__(self, **kwds): ... + def __getnewargs__(self): ... +def abs(x, *, memory_pool: Incomplete | None = ...): ... +def abs_checked(x, *, memory_pool: Incomplete | None = ...): ... +def acos(x, *, memory_pool: Incomplete | None = ...): ... +def acos_checked(x, *, memory_pool: Incomplete | None = ...): ... +def acosh(x, *, memory_pool: Incomplete | None = ...): ... +def acosh_checked(x, *, memory_pool: Incomplete | None = ...): ... +def add(x, y, *, memory_pool: Incomplete | None = ...): ... +def add_checked(x, y, *, memory_pool: Incomplete | None = ...): ... +def all(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def and_(x, y, *, memory_pool: Incomplete | None = ...): ... +def and_kleene(x, y, *, memory_pool: Incomplete | None = ...): ... +def and_not(x, y, *, memory_pool: Incomplete | None = ...): ... +def and_not_kleene(x, y, *, memory_pool: Incomplete | None = ...): ... +def any(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def approximate_median(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def array_filter(array, selection_filter, null_selection_behavior: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def array_sort_indices(array, order: str = ..., *, null_placement: str = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def array_take(array, indices, *, boundscheck: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def ascii_capitalize(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_center(strings, width, padding: str = ..., lean_left_on_odd_padding: bool = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def ascii_is_alnum(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_alpha(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_decimal(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_lower(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_printable(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_space(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_title(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_upper(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_lower(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_lpad(strings, width, padding: str = ..., lean_left_on_odd_padding: bool = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def ascii_ltrim(strings, characters, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def ascii_ltrim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_reverse(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_rpad(strings, width, padding: str = ..., lean_left_on_odd_padding: bool = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def ascii_rtrim(strings, characters, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def ascii_rtrim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_split_whitespace(strings, *, max_splits: Incomplete | None = ..., reverse: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def ascii_swapcase(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_title(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_trim(strings, characters, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def ascii_trim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_upper(strings, *, memory_pool: Incomplete | None = ...): ... +def asin(x, *, memory_pool: Incomplete | None = ...): ... +def asin_checked(x, *, memory_pool: Incomplete | None = ...): ... +def asinh(x, *, memory_pool: Incomplete | None = ...): ... +def assume_timezone(timestamps, timezone, *, ambiguous: str = ..., nonexistent: str = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def atan(x, *, memory_pool: Incomplete | None = ...): ... +def atan2(y, x, *, memory_pool: Incomplete | None = ...): ... +def atanh(x, *, memory_pool: Incomplete | None = ...): ... +def atanh_checked(x, *, memory_pool: Incomplete | None = ...): ... +def binary_join(strings, separator, *, memory_pool: Incomplete | None = ...): ... +def binary_join_element_wise(*strings, null_handling: str = ..., null_replacement: str = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def binary_length(strings, *, memory_pool: Incomplete | None = ...): ... +def binary_repeat(strings, num_repeats, *, memory_pool: Incomplete | None = ...): ... +def binary_replace_slice(strings, start, stop, replacement, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def binary_reverse(strings, *, memory_pool: Incomplete | None = ...): ... +def binary_slice(strings, start, stop: Incomplete | None = ..., step: int = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def bit_wise_and(x, y, *, memory_pool: Incomplete | None = ...): ... +def bit_wise_not(x, *, memory_pool: Incomplete | None = ...): ... +def bit_wise_or(x, y, *, memory_pool: Incomplete | None = ...): ... +def bit_wise_xor(x, y, *, memory_pool: Incomplete | None = ...): ... +def case_when(cond, *cases, memory_pool: Incomplete | None = ...): ... +def cast(arr, target_type: Incomplete | None = ..., safe: Incomplete | None = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def ceil(x, *, memory_pool: Incomplete | None = ...): ... +def ceil_temporal(timestamps, multiple: int = ..., unit: str = ..., *, week_starts_monday: bool = ..., ceil_is_strictly_greater: bool = ..., calendar_based_origin: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def choose(indices, *values, memory_pool: Incomplete | None = ...): ... +def coalesce(*values, memory_pool: Incomplete | None = ...): ... +def cos(x, *, memory_pool: Incomplete | None = ...): ... +def cos_checked(x, *, memory_pool: Incomplete | None = ...): ... +def cosh(x, *, memory_pool: Incomplete | None = ...): ... +def count(array, mode: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def count_distinct(array, mode: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def count_substring(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def count_substring_regex(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def cumulative_max(values, start: Incomplete | None = ..., *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def cumulative_mean(values, start: Incomplete | None = ..., *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def cumulative_min(values, start: Incomplete | None = ..., *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def cumulative_prod(values, start: Incomplete | None = ..., *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def cumulative_prod_checked(values, start: Incomplete | None = ..., *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def cumulative_sum(values, start: Incomplete | None = ..., *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def cumulative_sum_checked(values, start: Incomplete | None = ..., *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def day(values, *, memory_pool: Incomplete | None = ...): ... +def day_of_week(values, *, count_from_zero: bool = ..., week_start: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def day_of_year(values, *, memory_pool: Incomplete | None = ...): ... +def day_time_interval_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def days_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def dictionary_decode(dictionary_array, *, memory_pool: Incomplete | None = ...): ... +def dictionary_encode(array, null_encoding: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def divide(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def divide_checked(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def drop_null(input, *, memory_pool: Incomplete | None = ...): ... +def ends_with(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def equal(x, y, *, memory_pool: Incomplete | None = ...): ... +def exp(exponent, *, memory_pool: Incomplete | None = ...): ... +def expm1(exponent, *, memory_pool: Incomplete | None = ...): ... +def extract_regex(strings, pattern, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def extract_regex_span(strings, pattern, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def fill_null_backward(values, *, memory_pool: Incomplete | None = ...): ... +def fill_null_forward(values, *, memory_pool: Incomplete | None = ...): ... +def filter(input, selection_filter, null_selection_behavior: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def find_substring(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def find_substring_regex(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def first(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def first_last(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def floor(x, *, memory_pool: Incomplete | None = ...): ... +def floor_temporal(timestamps, multiple: int = ..., unit: str = ..., *, week_starts_monday: bool = ..., ceil_is_strictly_greater: bool = ..., calendar_based_origin: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def greater(x, y, *, memory_pool: Incomplete | None = ...): ... +def greater_equal(x, y, *, memory_pool: Incomplete | None = ...): ... +def hour(values, *, memory_pool: Incomplete | None = ...): ... +def hours_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def hypot(x, y, *, memory_pool: Incomplete | None = ...): ... +def if_else(cond, left, right, *, memory_pool: Incomplete | None = ...): ... +def index(data, value, start: Incomplete | None = ..., end: Incomplete | None = ..., *, memory_pool: Incomplete | None = ...): ... +def index_in(values, value_set, *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def index_in_meta_binary(values, value_set, *, memory_pool: Incomplete | None = ...): ... +def indices_nonzero(values, *, memory_pool: Incomplete | None = ...): ... +def inverse_permutation(indices, max_index: int = ..., output_type: Incomplete | None = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def invert(values, *, memory_pool: Incomplete | None = ...): ... +def is_dst(values, *, memory_pool: Incomplete | None = ...): ... +def is_finite(values, *, memory_pool: Incomplete | None = ...): ... +def is_in(values, value_set, *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def is_in_meta_binary(values, value_set, *, memory_pool: Incomplete | None = ...): ... +def is_inf(values, *, memory_pool: Incomplete | None = ...): ... +def is_leap_year(values, *, memory_pool: Incomplete | None = ...): ... +def is_nan(values, *, memory_pool: Incomplete | None = ...): ... +def is_null(values, *, nan_is_null: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def is_valid(values, *, memory_pool: Incomplete | None = ...): ... +def iso_calendar(values, *, memory_pool: Incomplete | None = ...): ... +def iso_week(values, *, memory_pool: Incomplete | None = ...): ... +def iso_year(values, *, memory_pool: Incomplete | None = ...): ... +def kurtosis(array, *, skip_nulls: bool = ..., biased: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def last(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def less(x, y, *, memory_pool: Incomplete | None = ...): ... +def less_equal(x, y, *, memory_pool: Incomplete | None = ...): ... +def list_element(lists, index, *, memory_pool: Incomplete | None = ...): ... +def list_flatten(lists, recursive: bool = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def list_parent_indices(lists, *, memory_pool: Incomplete | None = ...): ... +def list_slice(lists, start, stop: Incomplete | None = ..., step: int = ..., return_fixed_size_list: Incomplete | None = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def list_value_length(lists, *, memory_pool: Incomplete | None = ...): ... +def ln(x, *, memory_pool: Incomplete | None = ...): ... +def ln_checked(x, *, memory_pool: Incomplete | None = ...): ... +def local_timestamp(values, *, memory_pool: Incomplete | None = ...): ... +def log10(x, *, memory_pool: Incomplete | None = ...): ... +def log10_checked(x, *, memory_pool: Incomplete | None = ...): ... +def log1p(x, *, memory_pool: Incomplete | None = ...): ... +def log1p_checked(x, *, memory_pool: Incomplete | None = ...): ... +def log2(x, *, memory_pool: Incomplete | None = ...): ... +def log2_checked(x, *, memory_pool: Incomplete | None = ...): ... +def logb(x, b, *, memory_pool: Incomplete | None = ...): ... +def logb_checked(x, b, *, memory_pool: Incomplete | None = ...): ... +def make_struct(*args, field_names: tuple = ..., field_nullability: Incomplete | None = ..., field_metadata: Incomplete | None = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def map_lookup(container, query_key, occurrence, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def match_like(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def match_substring(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def match_substring_regex(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def max(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def max_element_wise(*args, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def mean(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def microsecond(values, *, memory_pool: Incomplete | None = ...): ... +def microseconds_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def millisecond(values, *, memory_pool: Incomplete | None = ...): ... +def milliseconds_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def min(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def min_element_wise(*args, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def min_max(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def minute(values, *, memory_pool: Incomplete | None = ...): ... +def minutes_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def mode(array, n: int = ..., *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def modulo(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def modulo_checked(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def month(values, *, memory_pool: Incomplete | None = ...): ... +def month_day_nano_interval_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def month_interval_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def multiply(x, y, *, memory_pool: Incomplete | None = ...): ... +def multiply_checked(x, y, *, memory_pool: Incomplete | None = ...): ... +def nanosecond(values, *, memory_pool: Incomplete | None = ...): ... +def nanoseconds_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def negate(x, *, memory_pool: Incomplete | None = ...): ... +def negate_checked(x, *, memory_pool: Incomplete | None = ...): ... +def not_equal(x, y, *, memory_pool: Incomplete | None = ...): ... +def or_(x, y, *, memory_pool: Incomplete | None = ...): ... +def or_kleene(x, y, *, memory_pool: Incomplete | None = ...): ... +def pairwise_diff(input, period: int = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def pairwise_diff_checked(input, period: int = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def partition_nth_indices(array, pivot, *, null_placement: str = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def pivot_wider(pivot_keys, pivot_values, key_names, *, unexpected_key_behavior: str = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def power(base, exponent, *, memory_pool: Incomplete | None = ...): ... +def power_checked(base, exponent, *, memory_pool: Incomplete | None = ...): ... +def product(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def quantile(array, q: float = ..., *, interpolation: str = ..., skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def quarter(values, *, memory_pool: Incomplete | None = ...): ... +def quarters_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def random(n, *, initializer: str = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def rank(input, sort_keys: str = ..., *, null_placement: Incomplete | None = ..., tiebreaker: str = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def rank_normal(input, sort_keys: str = ..., *, null_placement: Incomplete | None = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def rank_quantile(input, sort_keys: str = ..., *, null_placement: Incomplete | None = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def remainder(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def remainder_checked(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def replace_substring(strings, pattern, replacement, *, max_replacements: Incomplete | None = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def replace_substring_regex(strings, pattern, replacement, *, max_replacements: Incomplete | None = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def replace_with_mask(values, mask, replacements, *, memory_pool: Incomplete | None = ...): ... +def round(x, ndigits: int = ..., round_mode: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def round_binary(x, s, round_mode: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def round_temporal(timestamps, multiple: int = ..., unit: str = ..., *, week_starts_monday: bool = ..., ceil_is_strictly_greater: bool = ..., calendar_based_origin: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def round_to_multiple(x, multiple: float = ..., round_mode: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def run_end_decode(array, *, memory_pool: Incomplete | None = ...): ... +def run_end_encode(array, run_end_type: pyarrow.lib.DataType = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def scatter(values, indices, max_index: int = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def search_sorted(values, needles, side: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def second(values, *, memory_pool: Incomplete | None = ...): ... +def seconds_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def select_k_unstable(input, k, sort_keys, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def shift_left(x, y, *, memory_pool: Incomplete | None = ...): ... +def shift_left_checked(x, y, *, memory_pool: Incomplete | None = ...): ... +def shift_right(x, y, *, memory_pool: Incomplete | None = ...): ... +def shift_right_checked(x, y, *, memory_pool: Incomplete | None = ...): ... +def sign(x, *, memory_pool: Incomplete | None = ...): ... +def sin(x, *, memory_pool: Incomplete | None = ...): ... +def sin_checked(x, *, memory_pool: Incomplete | None = ...): ... +def sinh(x, *, memory_pool: Incomplete | None = ...): ... +def skew(array, *, skip_nulls: bool = ..., biased: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def sort_indices(input, sort_keys: tuple = ..., *, null_placement: Incomplete | None = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def split_pattern(strings, pattern, *, max_splits: Incomplete | None = ..., reverse: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def split_pattern_regex(strings, pattern, *, max_splits: Incomplete | None = ..., reverse: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def sqrt(x, *, memory_pool: Incomplete | None = ...): ... +def sqrt_checked(x, *, memory_pool: Incomplete | None = ...): ... +def starts_with(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def stddev(array, *, ddof: int = ..., skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def strftime(timestamps, format: str = ..., locale: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def string_is_ascii(strings, *, memory_pool: Incomplete | None = ...): ... +def strptime(strings, format, unit, error_is_null: bool = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def struct_field(values, indices, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def subsecond(values, *, memory_pool: Incomplete | None = ...): ... +def subtract(x, y, *, memory_pool: Incomplete | None = ...): ... +def subtract_checked(x, y, *, memory_pool: Incomplete | None = ...): ... +def sum(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def take(data, indices, *, boundscheck: bool = ..., memory_pool: Incomplete | None = ...): ... +def tan(x, *, memory_pool: Incomplete | None = ...): ... +def tan_checked(x, *, memory_pool: Incomplete | None = ...): ... +def tanh(x, *, memory_pool: Incomplete | None = ...): ... +def tdigest(array, q: float = ..., *, delta: int = ..., buffer_size: int = ..., skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def true_unless_null(values, *, memory_pool: Incomplete | None = ...): ... +def trunc(x, *, memory_pool: Incomplete | None = ...): ... +def unique(array, *, memory_pool: Incomplete | None = ...): ... +def us_week(values, *, memory_pool: Incomplete | None = ...): ... +def us_year(values, *, memory_pool: Incomplete | None = ...): ... +def utf8_capitalize(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_center(strings, width, padding: str = ..., lean_left_on_odd_padding: bool = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def utf8_is_alnum(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_alpha(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_decimal(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_digit(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_lower(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_numeric(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_printable(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_space(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_title(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_upper(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_length(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_lower(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_lpad(strings, width, padding: str = ..., lean_left_on_odd_padding: bool = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def utf8_ltrim(strings, characters, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def utf8_ltrim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_normalize(strings, form, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def utf8_replace_slice(strings, start, stop, replacement, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def utf8_reverse(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_rpad(strings, width, padding: str = ..., lean_left_on_odd_padding: bool = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def utf8_rtrim(strings, characters, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def utf8_rtrim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_slice_codeunits(strings, start, stop: Incomplete | None = ..., step: int = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def utf8_split_whitespace(strings, *, max_splits: Incomplete | None = ..., reverse: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def utf8_swapcase(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_title(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_trim(strings, characters, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def utf8_trim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_upper(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_zero_fill(strings, width, padding: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def value_counts(array, *, memory_pool: Incomplete | None = ...): ... +def variance(array, *, ddof: int = ..., skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def week(values, *, week_starts_monday: bool = ..., count_from_zero: bool = ..., first_week_is_fully_in_year: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def weeks_between(start, end, *, count_from_zero: bool = ..., week_start: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def winsorize(array, lower_limit, upper_limit, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def xor(x, y, *, memory_pool: Incomplete | None = ...): ... +def year(values, *, memory_pool: Incomplete | None = ...): ... +def year_month_day(values, *, memory_pool: Incomplete | None = ...): ... +def years_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def utf8_zfill(strings, width, padding: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... +def fill_null(values, fill_value): ... +def top_k_unstable(values, k, sort_keys: Incomplete | None = ..., null_placements: Incomplete | None = ..., *, memory_pool: Incomplete | None = ...): ... +def bottom_k_unstable(values, k, sort_keys: Incomplete | None = ..., null_placements: Incomplete | None = ..., *, memory_pool: Incomplete | None = ...): ... +def field(*name_or_index): ... +def scalar(value): ... diff --git a/python/pyarrow-stubs/pyarrow/csv.pyi b/python/pyarrow-stubs/pyarrow/csv.pyi new file mode 100644 index 000000000000..2c1757380f1b --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/csv.pyi @@ -0,0 +1 @@ +from pyarrow._csv import CSVStreamingReader as CSVStreamingReader, CSVWriter as CSVWriter, ConvertOptions as ConvertOptions, ISO8601 as ISO8601, InvalidRow as InvalidRow, ParseOptions as ParseOptions, ReadOptions as ReadOptions, WriteOptions as WriteOptions, open_csv as open_csv, read_csv as read_csv, write_csv as write_csv diff --git a/python/pyarrow-stubs/pyarrow/cuda.pyi b/python/pyarrow-stubs/pyarrow/cuda.pyi new file mode 100644 index 000000000000..fe0377327475 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/cuda.pyi @@ -0,0 +1 @@ +from pyarrow._cuda import BufferReader as BufferReader, BufferWriter as BufferWriter, Context as Context, CudaBuffer as CudaBuffer, HostBuffer as HostBuffer, IpcMemHandle as IpcMemHandle, new_host_buffer as new_host_buffer, read_message as read_message, read_record_batch as read_record_batch, serialize_record_batch as serialize_record_batch diff --git a/python/pyarrow-stubs/pyarrow/dataset.pyi b/python/pyarrow-stubs/pyarrow/dataset.pyi new file mode 100644 index 000000000000..4019efd3e088 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/dataset.pyi @@ -0,0 +1,25 @@ +from pyarrow._dataset import CsvFileFormat as CsvFileFormat, CsvFragmentScanOptions as CsvFragmentScanOptions, Dataset as Dataset, DatasetFactory as DatasetFactory, DirectoryPartitioning as DirectoryPartitioning, FeatherFileFormat as FeatherFileFormat, FileFormat as FileFormat, FileFragment as FileFragment, FileSystemDataset as FileSystemDataset, FileSystemDatasetFactory as FileSystemDatasetFactory, FileSystemFactoryOptions as FileSystemFactoryOptions, FileWriteOptions as FileWriteOptions, FilenamePartitioning as FilenamePartitioning, Fragment as Fragment, FragmentScanOptions as FragmentScanOptions, HivePartitioning as HivePartitioning, InMemoryDataset as InMemoryDataset, IpcFileFormat as IpcFileFormat, IpcFileWriteOptions as IpcFileWriteOptions, JsonFileFormat as JsonFileFormat, JsonFragmentScanOptions as JsonFragmentScanOptions, Partitioning as Partitioning, PartitioningFactory as PartitioningFactory, Scanner as Scanner, TaggedRecordBatch as TaggedRecordBatch, UnionDataset as UnionDataset, UnionDatasetFactory as UnionDatasetFactory, WrittenFile as WrittenFile, _filesystemdataset_write as _filesystemdataset_write, get_partition_keys as get_partition_keys +from pyarrow._dataset_orc import OrcFileFormat as OrcFileFormat +from pyarrow._dataset_parquet import ParquetDatasetFactory as ParquetDatasetFactory, ParquetFactoryOptions as ParquetFactoryOptions, ParquetFileFormat as ParquetFileFormat, ParquetFileFragment as ParquetFileFragment, ParquetFileWriteOptions as ParquetFileWriteOptions, ParquetFragmentScanOptions as ParquetFragmentScanOptions, ParquetReadOptions as ParquetReadOptions, RowGroupInfo as RowGroupInfo +from pyarrow._dataset_parquet_encryption import ParquetDecryptionConfig as ParquetDecryptionConfig, ParquetEncryptionConfig as ParquetEncryptionConfig +from pyarrow.compute import Expression as Expression, field as field, scalar as scalar +from pyarrow.util import _is_iterable as _is_iterable, _is_path_like as _is_path_like, _stringify_path as _stringify_path + +_orc_available: bool +_orc_msg: str +_parquet_available: bool +_parquet_msg: str + +def __getattr__(name) -> None: ... +def partitioning(schema=None, field_names=None, flavor=None, dictionaries=None): ... +def _ensure_partitioning(scheme): ... +def _ensure_format(obj): ... +def _ensure_multiple_sources(paths, filesystem=None): ... +def _ensure_single_source(path, filesystem=None): ... +def _filesystem_dataset(source, schema=None, filesystem=None, partitioning=None, format=None, partition_base_dir=None, exclude_invalid_files=None, selector_ignore_prefixes=None): ... +def _in_memory_dataset(source, schema=None, **kwargs): ... +def _union_dataset(children, schema=None, **kwargs): ... +def parquet_dataset(metadata_path, schema=None, filesystem=None, format=None, partitioning=None, partition_base_dir=None): ... +def dataset(source, schema=None, format=None, filesystem=None, partitioning=None, partition_base_dir=None, exclude_invalid_files=None, ignore_prefixes=None): ... +def _ensure_write_partitioning(part, schema, flavor): ... +def write_dataset(data, base_dir, *, basename_template=None, format=None, partitioning=None, partitioning_flavor=None, schema=None, filesystem=None, file_options=None, use_threads: bool = True, preserve_order: bool = False, max_partitions=None, max_open_files=None, max_rows_per_file=None, min_rows_per_group=None, max_rows_per_group=None, file_visitor=None, existing_data_behavior: str = 'error', create_dir: bool = True) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/feather.pyi b/python/pyarrow-stubs/pyarrow/feather.pyi new file mode 100644 index 000000000000..65ab238475a1 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/feather.pyi @@ -0,0 +1,24 @@ +from _typeshed import Incomplete +from pyarrow import _feather as _feather +from pyarrow._feather import FeatherError as FeatherError +from pyarrow.lib import Codec as Codec, Table as Table, concat_tables as concat_tables, schema as schema +from pyarrow.pandas_compat import _pandas_api as _pandas_api + +class FeatherDataset: + paths: Incomplete + validate_schema: Incomplete + def __init__(self, path_or_paths, validate_schema: bool = True) -> None: ... + _tables: Incomplete + schema: Incomplete + def read_table(self, columns=None): ... + def validate_schemas(self, piece, table) -> None: ... + def read_pandas(self, columns=None, use_threads: bool = True): ... + +def check_chunked_overflow(name, col) -> None: ... + +_FEATHER_SUPPORTED_CODECS: Incomplete + +def write_feather(df, dest, compression=None, compression_level=None, chunksize=None, version: int = 2) -> None: ... +def read_feather(source, columns=None, use_threads: bool = True, memory_map: bool = False, **kwargs): ... +def _read_table_internal(source, columns=None, memory_map: bool = False, use_threads: bool = True): ... +def read_table(source, columns=None, memory_map: bool = False, use_threads: bool = True): ... diff --git a/python/pyarrow-stubs/pyarrow/flight.pyi b/python/pyarrow-stubs/pyarrow/flight.pyi new file mode 100644 index 000000000000..7f3b30863f9f --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/flight.pyi @@ -0,0 +1 @@ +from pyarrow._flight import Action as Action, ActionType as ActionType, BasicAuth as BasicAuth, CallInfo as CallInfo, CertKeyPair as CertKeyPair, ClientAuthHandler as ClientAuthHandler, ClientMiddleware as ClientMiddleware, ClientMiddlewareFactory as ClientMiddlewareFactory, DescriptorType as DescriptorType, FlightCallOptions as FlightCallOptions, FlightCancelledError as FlightCancelledError, FlightClient as FlightClient, FlightDataStream as FlightDataStream, FlightDescriptor as FlightDescriptor, FlightEndpoint as FlightEndpoint, FlightError as FlightError, FlightInfo as FlightInfo, FlightInternalError as FlightInternalError, FlightMetadataReader as FlightMetadataReader, FlightMetadataWriter as FlightMetadataWriter, FlightMethod as FlightMethod, FlightServerBase as FlightServerBase, FlightServerError as FlightServerError, FlightStreamChunk as FlightStreamChunk, FlightStreamReader as FlightStreamReader, FlightStreamWriter as FlightStreamWriter, FlightTimedOutError as FlightTimedOutError, FlightUnauthenticatedError as FlightUnauthenticatedError, FlightUnauthorizedError as FlightUnauthorizedError, FlightUnavailableError as FlightUnavailableError, FlightWriteSizeExceededError as FlightWriteSizeExceededError, GeneratorStream as GeneratorStream, Location as Location, MetadataRecordBatchReader as MetadataRecordBatchReader, MetadataRecordBatchWriter as MetadataRecordBatchWriter, RecordBatchStream as RecordBatchStream, Result as Result, SchemaResult as SchemaResult, ServerAuthHandler as ServerAuthHandler, ServerCallContext as ServerCallContext, ServerMiddleware as ServerMiddleware, ServerMiddlewareFactory as ServerMiddlewareFactory, Ticket as Ticket, TracingServerMiddlewareFactory as TracingServerMiddlewareFactory, connect as connect diff --git a/python/pyarrow-stubs/pyarrow/fs.pyi b/python/pyarrow-stubs/pyarrow/fs.pyi new file mode 100644 index 000000000000..b498d5a167d1 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/fs.pyi @@ -0,0 +1,39 @@ +from _typeshed import Incomplete +from pyarrow._azurefs import AzureFileSystem as AzureFileSystem +from pyarrow._fs import FileInfo as FileInfo, FileSelector as FileSelector, FileSystem as FileSystem, FileSystemHandler as FileSystemHandler, FileType as FileType, LocalFileSystem as LocalFileSystem, PyFileSystem as PyFileSystem, SubTreeFileSystem as SubTreeFileSystem, _MockFileSystem as _MockFileSystem, _copy_files as _copy_files, _copy_files_selector as _copy_files_selector +from pyarrow._gcsfs import GcsFileSystem as GcsFileSystem +from pyarrow._hdfs import HadoopFileSystem as HadoopFileSystem +from pyarrow._s3fs import AwsDefaultS3RetryStrategy as AwsDefaultS3RetryStrategy, AwsStandardS3RetryStrategy as AwsStandardS3RetryStrategy, S3FileSystem as S3FileSystem, S3LogLevel as S3LogLevel, S3RetryStrategy as S3RetryStrategy, ensure_s3_finalized as ensure_s3_finalized, ensure_s3_initialized as ensure_s3_initialized, finalize_s3 as finalize_s3, initialize_s3 as initialize_s3, resolve_s3_region as resolve_s3_region +from pyarrow.util import _is_path_like as _is_path_like, _stringify_path as _stringify_path + +FileStats = FileInfo +_not_imported: Incomplete + +def __getattr__(name) -> None: ... +def _ensure_filesystem(filesystem, *, use_mmap: bool = False): ... +def _resolve_filesystem_and_path(path, filesystem=None, *, memory_map: bool = False): ... +def copy_files(source, destination, source_filesystem=None, destination_filesystem=None, *, chunk_size=..., use_threads: bool = True) -> None: ... + +class FSSpecHandler(FileSystemHandler): + fs: Incomplete + def __init__(self, fs) -> None: ... + def __eq__(self, other): ... + def __ne__(self, other): ... + def get_type_name(self): ... + def normalize_path(self, path): ... + @staticmethod + def _create_file_info(path, info): ... + def get_file_info(self, paths): ... + def get_file_info_selector(self, selector): ... + def create_dir(self, path, recursive) -> None: ... + def delete_dir(self, path) -> None: ... + def _delete_dir_contents(self, path, missing_dir_ok) -> None: ... + def delete_dir_contents(self, path, missing_dir_ok) -> None: ... + def delete_root_dir_contents(self) -> None: ... + def delete_file(self, path) -> None: ... + def move(self, src, dest) -> None: ... + def copy_file(self, src, dest) -> None: ... + def open_input_stream(self, path): ... + def open_input_file(self, path): ... + def open_output_stream(self, path, metadata): ... + def open_append_stream(self, path, metadata): ... diff --git a/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi b/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi new file mode 100644 index 000000000000..ffa012116d98 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi @@ -0,0 +1 @@ +from .from_dataframe import from_dataframe as from_dataframe diff --git a/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi b/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi new file mode 100644 index 000000000000..471da201a818 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi @@ -0,0 +1,24 @@ +import enum +import pyarrow as pa +from _typeshed import Incomplete + +class DlpackDeviceType(enum.IntEnum): + CPU = 1 + CUDA = 2 + CPU_PINNED = 3 + OPENCL = 4 + VULKAN = 7 + METAL = 8 + VPI = 9 + ROCM = 10 + +class _PyArrowBuffer: + _x: Incomplete + def __init__(self, x: pa.Buffer, allow_copy: bool = True) -> None: ... + @property + def bufsize(self) -> int: ... + @property + def ptr(self) -> int: ... + def __dlpack__(self) -> None: ... + def __dlpack_device__(self) -> tuple[DlpackDeviceType, int | None]: ... + def __repr__(self) -> str: ... diff --git a/python/pyarrow-stubs/pyarrow/interchange/column.pyi b/python/pyarrow-stubs/pyarrow/interchange/column.pyi new file mode 100644 index 000000000000..fbbce73a9536 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/interchange/column.pyi @@ -0,0 +1,67 @@ +import enum +import pyarrow as pa +from _typeshed import Incomplete +from pyarrow.interchange.buffer import _PyArrowBuffer as _PyArrowBuffer +from typing import Any, Iterable, TypedDict + +class DtypeKind(enum.IntEnum): + INT = 0 + UINT = 1 + FLOAT = 2 + BOOL = 20 + STRING = 21 + DATETIME = 22 + CATEGORICAL = 23 +Dtype = tuple[DtypeKind, int, str, str] +_PYARROW_KINDS: Incomplete + +class ColumnNullType(enum.IntEnum): + NON_NULLABLE = 0 + USE_NAN = 1 + USE_SENTINEL = 2 + USE_BITMASK = 3 + USE_BYTEMASK = 4 + +class ColumnBuffers(TypedDict): + data: tuple[_PyArrowBuffer, Dtype] + validity: tuple[_PyArrowBuffer, Dtype] | None + offsets: tuple[_PyArrowBuffer, Dtype] | None + +class CategoricalDescription(TypedDict): + is_ordered: bool + is_dictionary: bool + categories: _PyArrowColumn | None + +class Endianness: + LITTLE: str + BIG: str + NATIVE: str + NA: str + +class NoBufferPresent(Exception): ... + +class _PyArrowColumn: + _allow_copy: Incomplete + _dtype: Incomplete + _col: Incomplete + def __init__(self, column: pa.Array | pa.ChunkedArray, allow_copy: bool = True) -> None: ... + def size(self) -> int: ... + @property + def offset(self) -> int: ... + @property + def dtype(self) -> tuple[DtypeKind, int, str, str]: ... + def _dtype_from_arrowdtype(self, dtype: pa.DataType, bit_width: int) -> tuple[DtypeKind, int, str, str]: ... + @property + def describe_categorical(self) -> CategoricalDescription: ... + @property + def describe_null(self) -> tuple[ColumnNullType, Any]: ... + @property + def null_count(self) -> int: ... + @property + def metadata(self) -> dict[str, Any]: ... + def num_chunks(self) -> int: ... + def get_chunks(self, n_chunks: int | None = None) -> Iterable[_PyArrowColumn]: ... + def get_buffers(self) -> ColumnBuffers: ... + def _get_data_buffer(self) -> tuple[_PyArrowBuffer, Any]: ... + def _get_validity_buffer(self) -> tuple[_PyArrowBuffer, Any]: ... + def _get_offsets_buffer(self) -> tuple[_PyArrowBuffer, Any]: ... diff --git a/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi b/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi new file mode 100644 index 000000000000..5c46eb9e754c --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi @@ -0,0 +1,23 @@ +import pyarrow as pa +from _typeshed import Incomplete +from pyarrow.interchange.column import _PyArrowColumn as _PyArrowColumn +from typing import Any, Iterable, Sequence + +class _PyArrowDataFrame: + _df: Incomplete + _nan_as_null: Incomplete + _allow_copy: Incomplete + def __init__(self, df: pa.Table | pa.RecordBatch, nan_as_null: bool = False, allow_copy: bool = True) -> None: ... + def __dataframe__(self, nan_as_null: bool = False, allow_copy: bool = True) -> _PyArrowDataFrame: ... + @property + def metadata(self) -> dict[str, Any]: ... + def num_columns(self) -> int: ... + def num_rows(self) -> int: ... + def num_chunks(self) -> int: ... + def column_names(self) -> Iterable[str]: ... + def get_column(self, i: int) -> _PyArrowColumn: ... + def get_column_by_name(self, name: str) -> _PyArrowColumn: ... + def get_columns(self) -> Iterable[_PyArrowColumn]: ... + def select_columns(self, indices: Sequence[int]) -> _PyArrowDataFrame: ... + def select_columns_by_name(self, names: Sequence[str]) -> _PyArrowDataFrame: ... + def get_chunks(self, n_chunks: int | None = None) -> Iterable[_PyArrowDataFrame]: ... diff --git a/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi b/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi new file mode 100644 index 000000000000..8c3ab6757686 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi @@ -0,0 +1,20 @@ +import pyarrow as pa +from pyarrow.interchange.column import ColumnBuffers as ColumnBuffers, ColumnNullType as ColumnNullType, Dtype as Dtype, DtypeKind as DtypeKind +from typing import Any + +DataFrameObject = Any +ColumnObject = Any +BufferObject = Any +_PYARROW_DTYPES: dict[DtypeKind, dict[int, Any]] + +def from_dataframe(df: DataFrameObject, allow_copy: bool = True) -> pa.Table: ... +def _from_dataframe(df: DataFrameObject, allow_copy: bool = True): ... +def protocol_df_chunk_to_pyarrow(df: DataFrameObject, allow_copy: bool = True) -> pa.RecordBatch: ... +def column_to_array(col: ColumnObject, allow_copy: bool = True) -> pa.Array: ... +def bool_column_to_array(col: ColumnObject, allow_copy: bool = True) -> pa.Array: ... +def categorical_column_to_dictionary(col: ColumnObject, allow_copy: bool = True) -> pa.DictionaryArray: ... +def parse_datetime_format_str(format_str): ... +def map_date_type(data_type): ... +def buffers_to_array(buffers: ColumnBuffers, data_type: tuple[DtypeKind, int, str, str], length: int, describe_null: ColumnNullType, offset: int = 0, allow_copy: bool = True) -> pa.Array: ... +def validity_buffer_from_mask(validity_buff: BufferObject, validity_dtype: Dtype, describe_null: ColumnNullType, length: int, offset: int = 0, allow_copy: bool = True) -> pa.Buffer: ... +def validity_buffer_nan_sentinel(data_pa_buffer: BufferObject, data_type: Dtype, describe_null: ColumnNullType, length: int, offset: int = 0, allow_copy: bool = True) -> pa.Buffer: ... diff --git a/python/pyarrow-stubs/pyarrow/ipc.pyi b/python/pyarrow-stubs/pyarrow/ipc.pyi new file mode 100644 index 000000000000..ce305f4df1c2 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/ipc.pyi @@ -0,0 +1,29 @@ +import pyarrow.lib as lib +from _typeshed import Incomplete +from pyarrow.lib import Alignment as Alignment, IpcReadOptions as IpcReadOptions, IpcWriteOptions as IpcWriteOptions, Message as Message, MessageReader as MessageReader, MetadataVersion as MetadataVersion, ReadStats as ReadStats, RecordBatchReader as RecordBatchReader, WriteStats as WriteStats, _ReadPandasMixin as _ReadPandasMixin, get_record_batch_size as get_record_batch_size, get_tensor_size as get_tensor_size, read_message as read_message, read_record_batch as read_record_batch, read_schema as read_schema, read_tensor as read_tensor, write_tensor as write_tensor + +class RecordBatchStreamReader(lib._RecordBatchStreamReader): + def __init__(self, source, *, options=None, memory_pool=None) -> None: ... + +_ipc_writer_class_doc: str +_ipc_file_writer_class_doc: Incomplete + +class RecordBatchStreamWriter(lib._RecordBatchStreamWriter): + __doc__: Incomplete + def __init__(self, sink, schema, *, options=None) -> None: ... + +class RecordBatchFileReader(lib._RecordBatchFileReader): + def __init__(self, source, footer_offset=None, *, options=None, memory_pool=None) -> None: ... + +class RecordBatchFileWriter(lib._RecordBatchFileWriter): + __doc__: Incomplete + def __init__(self, sink, schema, *, options=None, metadata=None) -> None: ... + +def _get_legacy_format_default(options): ... +def _ensure_default_ipc_read_options(options): ... +def new_stream(sink, schema, *, options=None): ... +def open_stream(source, *, options=None, memory_pool=None): ... +def new_file(sink, schema, *, options=None, metadata=None): ... +def open_file(source, footer_offset=None, *, options=None, memory_pool=None): ... +def serialize_pandas(df, *, nthreads=None, preserve_index=None): ... +def deserialize_pandas(buf, *, use_threads: bool = True): ... diff --git a/python/pyarrow-stubs/pyarrow/json.pyi b/python/pyarrow-stubs/pyarrow/json.pyi new file mode 100644 index 000000000000..1f4769cff92e --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/json.pyi @@ -0,0 +1 @@ +from pyarrow._json import ParseOptions as ParseOptions, ReadOptions as ReadOptions, open_json as open_json, read_json as read_json diff --git a/python/pyarrow-stubs/pyarrow/jvm.pyi b/python/pyarrow-stubs/pyarrow/jvm.pyi new file mode 100644 index 000000000000..56e7e88f0ecc --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/jvm.pyi @@ -0,0 +1,17 @@ +from _typeshed import Incomplete + +class _JvmBufferNanny: + ref_manager: Incomplete + def __init__(self, jvm_buf) -> None: ... + def __del__(self) -> None: ... + +def jvm_buffer(jvm_buf): ... +def _from_jvm_int_type(jvm_type): ... +def _from_jvm_float_type(jvm_type): ... +def _from_jvm_time_type(jvm_type): ... +def _from_jvm_timestamp_type(jvm_type): ... +def _from_jvm_date_type(jvm_type): ... +def field(jvm_field): ... +def schema(jvm_schema): ... +def array(jvm_array): ... +def record_batch(jvm_vector_schema_root): ... diff --git a/python/pyarrow-stubs/pyarrow/orc.pyi b/python/pyarrow-stubs/pyarrow/orc.pyi new file mode 100644 index 000000000000..67d572bcfc53 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/orc.pyi @@ -0,0 +1,61 @@ +import types +from _typeshed import Incomplete +from pyarrow.fs import _resolve_filesystem_and_path as _resolve_filesystem_and_path +from pyarrow.lib import Table as Table + +class ORCFile: + reader: Incomplete + def __init__(self, source) -> None: ... + @property + def metadata(self): ... + @property + def schema(self): ... + @property + def nrows(self): ... + @property + def nstripes(self): ... + @property + def file_version(self): ... + @property + def software_version(self): ... + @property + def compression(self): ... + @property + def compression_size(self): ... + @property + def writer(self): ... + @property + def writer_version(self): ... + @property + def row_index_stride(self): ... + @property + def nstripe_statistics(self): ... + @property + def content_length(self): ... + @property + def stripe_statistics_length(self): ... + @property + def file_footer_length(self): ... + @property + def file_postscript_length(self): ... + @property + def file_length(self): ... + def _select_names(self, columns=None): ... + def read_stripe(self, n, columns=None): ... + def read(self, columns=None): ... + +_orc_writer_args_docs: str + +class ORCWriter: + __doc__: Incomplete + is_open: bool + writer: Incomplete + def __init__(self, where, *, file_version: str = '0.12', batch_size: int = 1024, stripe_size=..., compression: str = 'uncompressed', compression_block_size: int = 65536, compression_strategy: str = 'speed', row_index_stride: int = 10000, padding_tolerance: float = 0.0, dictionary_key_size_threshold: float = 0.0, bloom_filter_columns=None, bloom_filter_fpp: float = 0.05) -> None: ... + def __del__(self) -> None: ... + def __enter__(self): ... + def __exit__(self, *args, **kwargs) -> None: ... + def write(self, table) -> None: ... + def close(self) -> None: ... + +def read_table(source, columns=None, filesystem=None): ... +def write_table(table, where, *, file_version: str = '0.12', batch_size: int = 1024, stripe_size=..., compression: str = 'uncompressed', compression_block_size: int = 65536, compression_strategy: str = 'speed', row_index_stride: int = 10000, padding_tolerance: float = 0.0, dictionary_key_size_threshold: float = 0.0, bloom_filter_columns=None, bloom_filter_fpp: float = 0.05) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/pandas_compat.pyi b/python/pyarrow-stubs/pyarrow/pandas_compat.pyi new file mode 100644 index 000000000000..6b80ab3e0d94 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/pandas_compat.pyi @@ -0,0 +1,45 @@ +from _typeshed import Incomplete +from pyarrow.lib import _pandas_api as _pandas_api, frombytes as frombytes, is_threading_enabled as is_threading_enabled + +_logical_type_map: Incomplete +_numpy_logical_type_map: Incomplete +_pandas_logical_type_map: Incomplete + +def get_logical_type_map(): ... +def get_logical_type(arrow_type): ... +def get_numpy_logical_type_map(): ... +def get_logical_type_from_numpy(pandas_collection): ... +def get_extension_dtype_info(column): ... +def get_column_metadata(column, name, arrow_type, field_name): ... +def construct_metadata(columns_to_convert, df, column_names, index_levels, index_descriptors, preserve_index, types, column_field_names=None): ... +def _get_simple_index_descriptor(level, name): ... +def _column_name_to_strings(name): ... +def _index_level_name(index, i, column_names): ... +def _get_columns_to_convert(df, schema, preserve_index, columns): ... +def _get_columns_to_convert_given_schema(df, schema, preserve_index): ... +def _get_index_level(df, name): ... +def _level_name(name): ... +def _get_range_index_descriptor(level): ... +def _get_index_level_values(index): ... +def _resolve_columns_of_interest(df, schema, columns): ... +def dataframe_to_types(df, preserve_index, columns=None): ... +def dataframe_to_arrays(df, schema, preserve_index, nthreads: int = 1, columns=None, safe: bool = True): ... +def get_datetimetz_type(values, dtype, type_): ... +def _reconstruct_block(item, columns=None, extension_columns=None, return_block: bool = True): ... +def make_datetimetz(unit, tz): ... +def table_to_dataframe(options, table, categories=None, ignore_metadata: bool = False, types_mapper=None): ... + +_pandas_supported_numpy_types: Incomplete + +def _get_extension_dtypes(table, columns_metadata, types_mapper, options, categories): ... +def _check_data_column_metadata_consistency(all_columns) -> None: ... +def _deserialize_column_index(block_table, all_columns, column_indexes): ... +def _reconstruct_index(table, index_descriptors, all_columns, types_mapper=None): ... +def _extract_index_level(table, result_table, field_name, field_name_to_metadata, types_mapper=None): ... +def _backwards_compatible_index_name(raw_name, logical_name): ... +def _is_generated_index_name(name): ... +def get_pandas_logical_type_map(): ... +def _pandas_type_to_numpy_type(pandas_type): ... +def _reconstruct_columns_from_metadata(columns, column_indexes): ... +def _add_any_metadata(table, pandas_metadata): ... +def make_tz_aware(series, tz): ... diff --git a/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi b/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi new file mode 100644 index 000000000000..bb67a43fa4e5 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi @@ -0,0 +1 @@ +from .core import * diff --git a/python/pyarrow-stubs/pyarrow/parquet/core.pyi b/python/pyarrow-stubs/pyarrow/parquet/core.pyi new file mode 100644 index 000000000000..16b1265c5fd6 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/parquet/core.pyi @@ -0,0 +1,87 @@ +import types +from _typeshed import Incomplete +from pyarrow._parquet import ColumnChunkMetaData as ColumnChunkMetaData, ColumnSchema as ColumnSchema, FileDecryptionProperties as FileDecryptionProperties, FileEncryptionProperties as FileEncryptionProperties, FileMetaData as FileMetaData, ParquetLogicalType as ParquetLogicalType, ParquetReader as ParquetReader, ParquetSchema as ParquetSchema, RowGroupMetaData as RowGroupMetaData, SortingColumn as SortingColumn, Statistics as Statistics + +__all__ = ['ParquetReader', 'Statistics', 'FileMetaData', 'RowGroupMetaData', 'ColumnChunkMetaData', 'ParquetSchema', 'ColumnSchema', 'ParquetLogicalType', 'FileEncryptionProperties', 'FileDecryptionProperties', 'SortingColumn', 'filters_to_expression', '_filters_to_expression', 'ParquetFile', 'ParquetWriter', 'ParquetDataset', 'read_table', 'read_pandas', 'write_table', 'write_to_dataset', 'write_metadata', 'read_metadata', 'read_schema'] + +def filters_to_expression(filters): ... + +_filters_to_expression: Incomplete + +class ParquetFile: + _close_source: Incomplete + reader: Incomplete + common_metadata: Incomplete + _nested_paths_by_prefix: Incomplete + def __init__(self, source, *, metadata=None, common_metadata=None, read_dictionary=None, binary_type=None, list_type=None, memory_map: bool = False, buffer_size: int = 0, pre_buffer: bool = True, coerce_int96_timestamp_unit=None, decryption_properties=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, filesystem=None, page_checksum_verification: bool = False, arrow_extensions_enabled: bool = True) -> None: ... + def __enter__(self): ... + def __exit__(self, *args, **kwargs) -> None: ... + def _build_nested_paths(self): ... + @property + def metadata(self): ... + @property + def schema(self): ... + @property + def schema_arrow(self): ... + @property + def num_row_groups(self): ... + def close(self, force: bool = False): ... + @property + def closed(self) -> bool: ... + def read_row_group(self, i, columns=None, use_threads: bool = True, use_pandas_metadata: bool = False): ... + def read_row_groups(self, row_groups, columns=None, use_threads: bool = True, use_pandas_metadata: bool = False): ... + def iter_batches(self, batch_size: int = 65536, row_groups=None, columns=None, use_threads: bool = True, use_pandas_metadata: bool = False): ... + def read(self, columns=None, use_threads: bool = True, use_pandas_metadata: bool = False): ... + def scan_contents(self, columns=None, batch_size: int = 65536): ... + def _get_column_indices(self, column_names, use_pandas_metadata: bool = False): ... + +class ParquetWriter: + __doc__: Incomplete + flavor: Incomplete + schema_changed: bool + schema: Incomplete + where: Incomplete + file_handle: Incomplete + _metadata_collector: Incomplete + writer: Incomplete + is_open: bool + def __init__(self, where, schema, filesystem=None, flavor=None, version: str = '2.6', use_dictionary: bool = True, compression: str = 'snappy', write_statistics: bool = True, use_deprecated_int96_timestamps=None, compression_level=None, use_byte_stream_split: bool = False, column_encoding=None, writer_engine_version=None, data_page_version: str = '1.0', use_compliant_nested_type: bool = True, encryption_properties=None, write_batch_size=None, dictionary_pagesize_limit=None, store_schema: bool = True, write_page_index: bool = False, write_page_checksum: bool = False, sorting_columns=None, store_decimal_as_integer: bool = False, write_time_adjusted_to_utc: bool = False, max_rows_per_page=None, bloom_filter_options=None, use_content_defined_chunking: bool = False, **options) -> None: ... + def __del__(self) -> None: ... + def __enter__(self): ... + def __exit__(self, *args, **kwargs): ... + def write(self, table_or_batch, row_group_size=None) -> None: ... + def write_batch(self, batch, row_group_size=None) -> None: ... + def write_table(self, table, row_group_size=None) -> None: ... + def close(self) -> None: ... + def add_key_value_metadata(self, key_value_metadata) -> None: ... + +class ParquetDataset: + __doc__: Incomplete + _filter_expression: Incomplete + _base_dir: Incomplete + _dataset: Incomplete + def __init__(self, path_or_paths, filesystem=None, schema=None, *, filters=None, read_dictionary=None, binary_type=None, list_type=None, memory_map: bool = False, buffer_size=None, partitioning: str = 'hive', ignore_prefixes=None, pre_buffer: bool = True, coerce_int96_timestamp_unit=None, decryption_properties=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, page_checksum_verification: bool = False, arrow_extensions_enabled: bool = True) -> None: ... + def equals(self, other): ... + def __eq__(self, other): ... + @property + def schema(self): ... + def read(self, columns=None, use_threads: bool = True, use_pandas_metadata: bool = False): ... + def _get_common_pandas_metadata(self): ... + def read_pandas(self, **kwargs): ... + @property + def fragments(self): ... + @property + def files(self): ... + @property + def filesystem(self): ... + @property + def partitioning(self): ... + +def read_table(source, *, columns=None, use_threads: bool = True, schema=None, use_pandas_metadata: bool = False, read_dictionary=None, binary_type=None, list_type=None, memory_map: bool = False, buffer_size: int = 0, partitioning: str = 'hive', filesystem=None, filters=None, ignore_prefixes=None, pre_buffer: bool = True, coerce_int96_timestamp_unit=None, decryption_properties=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, page_checksum_verification: bool = False, arrow_extensions_enabled: bool = True): ... +def read_pandas(source, columns=None, **kwargs): ... +def write_table(table, where, row_group_size=None, version: str = '2.6', use_dictionary: bool = True, compression: str = 'snappy', write_statistics: bool = True, use_deprecated_int96_timestamps=None, coerce_timestamps=None, allow_truncated_timestamps: bool = False, data_page_size=None, flavor=None, filesystem=None, compression_level=None, use_byte_stream_split: bool = False, column_encoding=None, data_page_version: str = '1.0', use_compliant_nested_type: bool = True, encryption_properties=None, write_batch_size=None, dictionary_pagesize_limit=None, store_schema: bool = True, write_page_index: bool = False, write_page_checksum: bool = False, sorting_columns=None, store_decimal_as_integer: bool = False, write_time_adjusted_to_utc: bool = False, max_rows_per_page=None, bloom_filter_options=None, use_content_defined_chunking: bool = False, **kwargs) -> None: ... +def write_to_dataset(table, root_path, partition_cols=None, filesystem=None, schema=None, partitioning=None, basename_template=None, use_threads=None, file_visitor=None, existing_data_behavior=None, **kwargs) -> None: ... +def write_metadata(schema, where, metadata_collector=None, filesystem=None, **kwargs) -> None: ... +def read_metadata(where, memory_map: bool = False, decryption_properties=None, filesystem=None, arrow_extensions_enabled: bool = True): ... +def read_schema(where, memory_map: bool = False, decryption_properties=None, filesystem=None, arrow_extensions_enabled: bool = True): ... +EXCLUDED_PARQUET_PATHS: set[str] diff --git a/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi b/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi new file mode 100644 index 000000000000..4b18b68cd1de --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi @@ -0,0 +1 @@ +from pyarrow._parquet_encryption import CryptoFactory as CryptoFactory, DecryptionConfiguration as DecryptionConfiguration, EncryptionConfiguration as EncryptionConfiguration, KmsClient as KmsClient, KmsConnectionConfig as KmsConnectionConfig, create_decryption_properties as create_decryption_properties, create_encryption_properties as create_encryption_properties diff --git a/python/pyarrow-stubs/pyarrow/substrait.pyi b/python/pyarrow-stubs/pyarrow/substrait.pyi new file mode 100644 index 000000000000..ff74e7100fb3 --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/substrait.pyi @@ -0,0 +1 @@ +from pyarrow._substrait import BoundExpressions as BoundExpressions, SubstraitSchema as SubstraitSchema, deserialize_expressions as deserialize_expressions, deserialize_schema as deserialize_schema, get_supported_functions as get_supported_functions, run_query as run_query, serialize_expressions as serialize_expressions, serialize_schema as serialize_schema diff --git a/python/pyarrow-stubs/pyarrow/types.pyi b/python/pyarrow-stubs/pyarrow/types.pyi new file mode 100644 index 000000000000..9ea7daf6651e --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/types.pyi @@ -0,0 +1,115 @@ +from _typeshed import Incomplete +from enum import IntEnum +from pyarrow.lib import is_boolean_value as is_boolean_value, is_float_value as is_float_value, is_integer_value as is_integer_value +from pyarrow.util import doc as doc + +_SIGNED_INTEGER_TYPES: Incomplete +_UNSIGNED_INTEGER_TYPES: Incomplete +_INTEGER_TYPES = _SIGNED_INTEGER_TYPES | _UNSIGNED_INTEGER_TYPES +_FLOATING_TYPES: Incomplete +_DECIMAL_TYPES: Incomplete +_DATE_TYPES: Incomplete +_TIME_TYPES: Incomplete +_INTERVAL_TYPES: Incomplete +_TEMPORAL_TYPES: Incomplete +_UNION_TYPES: Incomplete + +class TypesEnum(IntEnum): + NA = ... + BOOL = ... + INT8 = ... + INT16 = ... + INT32 = ... + INT64 = ... + UINT8 = ... + UINT16 = ... + UINT32 = ... + UINT64 = ... + HALF_FLOAT = ... + FLOAT = ... + DOUBLE = ... + BINARY = ... + BINARY_VIEW = ... + LARGE_BINARY = ... + STRING = ... + STRING_VIEW = ... + LARGE_STRING = ... + FIXED_SIZE_BINARY = ... + DECIMAL32 = ... + DECIMAL64 = ... + DECIMAL128 = ... + DECIMAL256 = ... + LIST = ... + LARGE_LIST = ... + LIST_VIEW = ... + LARGE_LIST_VIEW = ... + MAP = ... + FIXED_SIZE_LIST = ... + STRUCT = ... + SPARSE_UNION = ... + DENSE_UNION = ... + RUN_END_ENCODED = ... + DATE32 = ... + DATE64 = ... + TIME32 = ... + TIME64 = ... + TIMESTAMP = ... + INTERVAL_MONTHS = ... + INTERVAL_DAY_TIME = ... + INTERVAL_MONTH_DAY_NANO = ... + DURATION = ... + DICTIONARY = ... + +def is_null(t): ... +def is_boolean(t): ... +def is_integer(t): ... +def is_signed_integer(t): ... +def is_unsigned_integer(t): ... +def is_int8(t): ... +def is_int16(t): ... +def is_int32(t): ... +def is_int64(t): ... +def is_uint8(t): ... +def is_uint16(t): ... +def is_uint32(t): ... +def is_uint64(t): ... +def is_floating(t): ... +def is_float16(t): ... +def is_float32(t): ... +def is_float64(t): ... +def is_list(t): ... +def is_large_list(t): ... +def is_fixed_size_list(t): ... +def is_list_view(t): ... +def is_large_list_view(t): ... +def is_struct(t): ... +def is_union(t): ... +def is_nested(t): ... +def is_run_end_encoded(t): ... +def is_temporal(t): ... +def is_timestamp(t): ... +def is_duration(t): ... +def is_time(t): ... +def is_time32(t): ... +def is_time64(t): ... +def is_binary(t): ... +def is_large_binary(t): ... +def is_unicode(t): ... +def is_string(t): ... +def is_large_unicode(t): ... +def is_large_string(t): ... +def is_fixed_size_binary(t): ... +def is_binary_view(t): ... +def is_string_view(t): ... +def is_date(t): ... +def is_date32(t): ... +def is_date64(t): ... +def is_map(t): ... +def is_decimal(t): ... +def is_decimal32(t): ... +def is_decimal64(t): ... +def is_decimal128(t): ... +def is_decimal256(t): ... +def is_dictionary(t): ... +def is_interval(t): ... +def is_primitive(t): ... diff --git a/python/pyarrow-stubs/pyarrow/util.pyi b/python/pyarrow-stubs/pyarrow/util.pyi new file mode 100644 index 000000000000..640bb7f7646a --- /dev/null +++ b/python/pyarrow-stubs/pyarrow/util.pyi @@ -0,0 +1,16 @@ +_DEPR_MSG: str + +def doc(*docstrings, **params): ... +def _deprecate_api(old_name, new_name, api, next_version, type=...): ... +def _deprecate_class(old_name, new_class, next_version, instancecheck: bool = True): ... +def _is_iterable(obj): ... +def _is_path_like(path): ... +def _stringify_path(path): ... +def product(seq): ... +def get_contiguous_span(shape, strides, itemsize): ... +def find_free_port(): ... +def guid(): ... +def _break_traceback_cycle_from_frame(frame) -> None: ... +def _download_urllib(url, out_path) -> None: ... +def _download_requests(url, out_path) -> None: ... +def download_tzdata_on_windows() -> None: ... From ade3d13a594e11f4147f81bc98b09b97cb9c0a9f Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Fri, 9 Oct 2026 18:13:53 +0100 Subject: [PATCH 5/8] [Python] Add the license header to the generated stubs --- python/pyarrow-stubs/pyarrow/__init__.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/_acero.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/_azurefs.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/_compute.pyi | 17 +++++++++++++++++ .../pyarrow/_compute_docstrings.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/_csv.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/_cuda.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/_dataset.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/_dataset_orc.pyi | 17 +++++++++++++++++ .../pyarrow-stubs/pyarrow/_dataset_parquet.pyi | 17 +++++++++++++++++ .../pyarrow/_dataset_parquet_encryption.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/_feather.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/_flight.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/_fs.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/_gcsfs.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/_hdfs.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/_json.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/_orc.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/_parquet.pyi | 17 +++++++++++++++++ .../pyarrow/_parquet_encryption.pyi | 17 +++++++++++++++++ .../pyarrow/_pyarrow_cpp_tests.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/_s3fs.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/_substrait.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/acero.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/benchmark.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/cffi.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/compute.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/csv.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/cuda.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/dataset.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/feather.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/flight.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/fs.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/gandiva.pyi | 17 +++++++++++++++++ .../pyarrow/interchange/__init__.pyi | 17 +++++++++++++++++ .../pyarrow/interchange/buffer.pyi | 17 +++++++++++++++++ .../pyarrow/interchange/column.pyi | 17 +++++++++++++++++ .../pyarrow/interchange/dataframe.pyi | 17 +++++++++++++++++ .../pyarrow/interchange/from_dataframe.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/ipc.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/json.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/jvm.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/lib.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/orc.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/pandas_compat.pyi | 17 +++++++++++++++++ .../pyarrow-stubs/pyarrow/parquet/__init__.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/parquet/core.pyi | 17 +++++++++++++++++ .../pyarrow/parquet/encryption.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/substrait.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/types.pyi | 17 +++++++++++++++++ python/pyarrow-stubs/pyarrow/util.pyi | 17 +++++++++++++++++ 51 files changed, 867 insertions(+) diff --git a/python/pyarrow-stubs/pyarrow/__init__.pyi b/python/pyarrow-stubs/pyarrow/__init__.pyi index 25da79a19730..00007a0e7567 100644 --- a/python/pyarrow-stubs/pyarrow/__init__.pyi +++ b/python/pyarrow-stubs/pyarrow/__init__.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from pyarrow.ipc import Message as Message, MessageReader as MessageReader, MetadataVersion as MetadataVersion, RecordBatchFileReader as RecordBatchFileReader, RecordBatchFileWriter as RecordBatchFileWriter, RecordBatchStreamReader as RecordBatchStreamReader, RecordBatchStreamWriter as RecordBatchStreamWriter, deserialize_pandas as deserialize_pandas, serialize_pandas as serialize_pandas from pyarrow.lib import Array as Array, ArrowCancelled as ArrowCancelled, ArrowCapacityError as ArrowCapacityError, ArrowException as ArrowException, ArrowIOError as ArrowIOError, ArrowIndexError as ArrowIndexError, ArrowInvalid as ArrowInvalid, ArrowKeyError as ArrowKeyError, ArrowMemoryError as ArrowMemoryError, ArrowNotImplementedError as ArrowNotImplementedError, ArrowSerializationError as ArrowSerializationError, ArrowTypeError as ArrowTypeError, BaseExtensionType as BaseExtensionType, BinaryArray as BinaryArray, BinaryScalar as BinaryScalar, BinaryViewArray as BinaryViewArray, BinaryViewScalar as BinaryViewScalar, Bool8Array as Bool8Array, Bool8Scalar as Bool8Scalar, Bool8Type as Bool8Type, BooleanArray as BooleanArray, BooleanScalar as BooleanScalar, Buffer as Buffer, BufferOutputStream as BufferOutputStream, BufferReader as BufferReader, BufferedInputStream as BufferedInputStream, BufferedOutputStream as BufferedOutputStream, BuildInfo as BuildInfo, CacheOptions as CacheOptions, ChunkedArray as ChunkedArray, Codec as Codec, CompressedInputStream as CompressedInputStream, CompressedOutputStream as CompressedOutputStream, CppBuildInfo as CppBuildInfo, DataType as DataType, Date32Array as Date32Array, Date32Scalar as Date32Scalar, Date64Array as Date64Array, Date64Scalar as Date64Scalar, Decimal128Array as Decimal128Array, Decimal128Scalar as Decimal128Scalar, Decimal128Type as Decimal128Type, Decimal256Array as Decimal256Array, Decimal256Scalar as Decimal256Scalar, Decimal256Type as Decimal256Type, Decimal32Array as Decimal32Array, Decimal32Scalar as Decimal32Scalar, Decimal32Type as Decimal32Type, Decimal64Array as Decimal64Array, Decimal64Scalar as Decimal64Scalar, Decimal64Type as Decimal64Type, DenseUnionType as DenseUnionType, Device as Device, DeviceAllocationType as DeviceAllocationType, DictionaryArray as DictionaryArray, DictionaryMemo as DictionaryMemo, DictionaryScalar as DictionaryScalar, DictionaryType as DictionaryType, DoubleArray as DoubleArray, DoubleScalar as DoubleScalar, DurationArray as DurationArray, DurationScalar as DurationScalar, DurationType as DurationType, ExtensionArray as ExtensionArray, ExtensionScalar as ExtensionScalar, ExtensionType as ExtensionType, Field as Field, FixedShapeTensorArray as FixedShapeTensorArray, FixedShapeTensorScalar as FixedShapeTensorScalar, FixedShapeTensorType as FixedShapeTensorType, FixedSizeBinaryArray as FixedSizeBinaryArray, FixedSizeBinaryScalar as FixedSizeBinaryScalar, FixedSizeBinaryType as FixedSizeBinaryType, FixedSizeBufferWriter as FixedSizeBufferWriter, FixedSizeListArray as FixedSizeListArray, FixedSizeListScalar as FixedSizeListScalar, FixedSizeListType as FixedSizeListType, FloatArray as FloatArray, FloatScalar as FloatScalar, FloatingPointArray as FloatingPointArray, HalfFloatArray as HalfFloatArray, HalfFloatScalar as HalfFloatScalar, Int16Array as Int16Array, Int16Scalar as Int16Scalar, Int32Array as Int32Array, Int32Scalar as Int32Scalar, Int64Array as Int64Array, Int64Scalar as Int64Scalar, Int8Array as Int8Array, Int8Scalar as Int8Scalar, IntegerArray as IntegerArray, JsonArray as JsonArray, JsonScalar as JsonScalar, JsonType as JsonType, KeyValueMetadata as KeyValueMetadata, LargeBinaryArray as LargeBinaryArray, LargeBinaryScalar as LargeBinaryScalar, LargeListArray as LargeListArray, LargeListScalar as LargeListScalar, LargeListType as LargeListType, LargeListViewArray as LargeListViewArray, LargeListViewScalar as LargeListViewScalar, LargeListViewType as LargeListViewType, LargeStringArray as LargeStringArray, LargeStringScalar as LargeStringScalar, ListArray as ListArray, ListScalar as ListScalar, ListType as ListType, ListViewArray as ListViewArray, ListViewScalar as ListViewScalar, ListViewType as ListViewType, LoggingMemoryPool as LoggingMemoryPool, MapArray as MapArray, MapScalar as MapScalar, MapType as MapType, MemoryManager as MemoryManager, MemoryMappedFile as MemoryMappedFile, MemoryPool as MemoryPool, MockOutputStream as MockOutputStream, MonthDayNano as MonthDayNano, MonthDayNanoIntervalArray as MonthDayNanoIntervalArray, MonthDayNanoIntervalScalar as MonthDayNanoIntervalScalar, NA as NA, NativeFile as NativeFile, NullArray as NullArray, NullScalar as NullScalar, NumericArray as NumericArray, OSFile as OSFile, OpaqueArray as OpaqueArray, OpaqueScalar as OpaqueScalar, OpaqueType as OpaqueType, ProxyMemoryPool as ProxyMemoryPool, PythonFile as PythonFile, RecordBatch as RecordBatch, RecordBatchReader as RecordBatchReader, ResizableBuffer as ResizableBuffer, RunEndEncodedArray as RunEndEncodedArray, RunEndEncodedScalar as RunEndEncodedScalar, RunEndEncodedType as RunEndEncodedType, RuntimeInfo as RuntimeInfo, Scalar as Scalar, Schema as Schema, SparseCOOTensor as SparseCOOTensor, SparseCSCMatrix as SparseCSCMatrix, SparseCSFTensor as SparseCSFTensor, SparseCSRMatrix as SparseCSRMatrix, SparseUnionType as SparseUnionType, StringArray as StringArray, StringScalar as StringScalar, StringViewArray as StringViewArray, StringViewScalar as StringViewScalar, StructArray as StructArray, StructScalar as StructScalar, StructType as StructType, Table as Table, TableGroupBy as TableGroupBy, Tensor as Tensor, Time32Array as Time32Array, Time32Scalar as Time32Scalar, Time32Type as Time32Type, Time64Array as Time64Array, Time64Scalar as Time64Scalar, Time64Type as Time64Type, TimestampArray as TimestampArray, TimestampScalar as TimestampScalar, TimestampType as TimestampType, TransformInputStream as TransformInputStream, UInt16Array as UInt16Array, UInt16Scalar as UInt16Scalar, UInt32Array as UInt32Array, UInt32Scalar as UInt32Scalar, UInt64Array as UInt64Array, UInt64Scalar as UInt64Scalar, UInt8Array as UInt8Array, UInt8Scalar as UInt8Scalar, UnionArray as UnionArray, UnionScalar as UnionScalar, UnionType as UnionType, UnknownExtensionType as UnknownExtensionType, UuidArray as UuidArray, UuidScalar as UuidScalar, UuidType as UuidType, VersionInfo as VersionInfo, allocate_buffer as allocate_buffer, arange as arange, array as array, binary as binary, binary_view as binary_view, bool8 as bool8, bool_ as bool_, build_info as build_info, chunked_array as chunked_array, compress as compress, concat_arrays as concat_arrays, concat_batches as concat_batches, concat_tables as concat_tables, cpp_build_info as cpp_build_info, cpp_version as cpp_version, cpp_version_info as cpp_version_info, cpu_count as cpu_count, create_memory_map as create_memory_map, date32 as date32, date64 as date64, decimal128 as decimal128, decimal256 as decimal256, decimal32 as decimal32, decimal64 as decimal64, decompress as decompress, default_cpu_memory_manager as default_cpu_memory_manager, default_memory_pool as default_memory_pool, dense_union as dense_union, dictionary as dictionary, duration as duration, enable_signal_handlers as enable_signal_handlers, field as field, fixed_shape_tensor as fixed_shape_tensor, float16 as float16, float32 as float32, float64 as float64, foreign_buffer as foreign_buffer, from_numpy_dtype as from_numpy_dtype, infer_type as infer_type, input_stream as input_stream, int16 as int16, int32 as int32, int64 as int64, int8 as int8, io_thread_count as io_thread_count, is_opentelemetry_enabled as is_opentelemetry_enabled, jemalloc_memory_pool as jemalloc_memory_pool, jemalloc_set_decay_ms as jemalloc_set_decay_ms, json_ as json_, large_binary as large_binary, large_list as large_list, large_list_view as large_list_view, large_string as large_string, large_utf8 as large_utf8, list_ as list_, list_view as list_view, log_memory_allocations as log_memory_allocations, logging_memory_pool as logging_memory_pool, map_ as map_, memory_map as memory_map, mimalloc_memory_pool as mimalloc_memory_pool, month_day_nano_interval as month_day_nano_interval, null as null, nulls as nulls, opaque as opaque, output_stream as output_stream, proxy_memory_pool as proxy_memory_pool, py_buffer as py_buffer, record_batch as record_batch, register_extension_type as register_extension_type, repeat as repeat, run_end_encoded as run_end_encoded, runtime_info as runtime_info, scalar as scalar, schema as schema, set_cpu_count as set_cpu_count, set_io_thread_count as set_io_thread_count, set_memory_pool as set_memory_pool, set_timezone_db_path as set_timezone_db_path, sparse_union as sparse_union, string as string, string_view as string_view, struct as struct, supported_memory_backends as supported_memory_backends, system_memory_pool as system_memory_pool, table as table, time32 as time32, time64 as time64, timestamp as timestamp, total_allocated_bytes as total_allocated_bytes, transcoding_input_stream as transcoding_input_stream, type_for_alias as type_for_alias, uint16 as uint16, uint32 as uint32, uint64 as uint64, uint8 as uint8, unify_schemas as unify_schemas, union as union, unregister_extension_type as unregister_extension_type, utf8 as utf8, uuid as uuid from pyarrow.util import _deprecate_api as _deprecate_api, _deprecate_class as _deprecate_class diff --git a/python/pyarrow-stubs/pyarrow/_acero.pyi b/python/pyarrow-stubs/pyarrow/_acero.pyi index ba6b37dbc95e..3050fe2faf58 100644 --- a/python/pyarrow-stubs/pyarrow/_acero.pyi +++ b/python/pyarrow-stubs/pyarrow/_acero.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_acero.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_azurefs.pyi b/python/pyarrow-stubs/pyarrow/_azurefs.pyi index 367472e95887..2283329f838f 100644 --- a/python/pyarrow-stubs/pyarrow/_azurefs.pyi +++ b/python/pyarrow-stubs/pyarrow/_azurefs.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_azurefs.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_compute.pyi b/python/pyarrow-stubs/pyarrow/_compute.pyi index 0a6a506da3fd..e2d54ad8ff2d 100644 --- a/python/pyarrow-stubs/pyarrow/_compute.pyi +++ b/python/pyarrow-stubs/pyarrow/_compute.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_compute.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi b/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi index 322bd8a07773..a83cc5dc1e77 100644 --- a/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi +++ b/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from _typeshed import Incomplete function_doc_additions: Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_csv.pyi b/python/pyarrow-stubs/pyarrow/_csv.pyi index acacdc4922c7..4ff752b897fd 100644 --- a/python/pyarrow-stubs/pyarrow/_csv.pyi +++ b/python/pyarrow-stubs/pyarrow/_csv.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_csv.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_cuda.pyi b/python/pyarrow-stubs/pyarrow/_cuda.pyi index 0bba62778e79..cd5f02af7b2e 100644 --- a/python/pyarrow-stubs/pyarrow/_cuda.pyi +++ b/python/pyarrow-stubs/pyarrow/_cuda.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_cuda.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_dataset.pyi b/python/pyarrow-stubs/pyarrow/_dataset.pyi index d020e2a8ed98..968d4161017c 100644 --- a/python/pyarrow-stubs/pyarrow/_dataset.pyi +++ b/python/pyarrow-stubs/pyarrow/_dataset.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_dataset.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi b/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi index 199e17e188bc..ae314a813cf8 100644 --- a/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi +++ b/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_dataset_orc.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi b/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi index d0102528dfe4..74e65df5d312 100644 --- a/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi +++ b/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_dataset_parquet.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi b/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi index e4c8500826d6..5ce489a1914f 100644 --- a/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi +++ b/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_dataset_parquet_encryption.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_feather.pyi b/python/pyarrow-stubs/pyarrow/_feather.pyi index 699764c9d026..06ccd55777de 100644 --- a/python/pyarrow-stubs/pyarrow/_feather.pyi +++ b/python/pyarrow-stubs/pyarrow/_feather.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_feather.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_flight.pyi b/python/pyarrow-stubs/pyarrow/_flight.pyi index 612030dc5892..27bb47f89be8 100644 --- a/python/pyarrow-stubs/pyarrow/_flight.pyi +++ b/python/pyarrow-stubs/pyarrow/_flight.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_flight.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_fs.pyi b/python/pyarrow-stubs/pyarrow/_fs.pyi index 1609c31f6b37..69337de47447 100644 --- a/python/pyarrow-stubs/pyarrow/_fs.pyi +++ b/python/pyarrow-stubs/pyarrow/_fs.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_fs.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_gcsfs.pyi b/python/pyarrow-stubs/pyarrow/_gcsfs.pyi index 99067b959485..e056dbf687f6 100644 --- a/python/pyarrow-stubs/pyarrow/_gcsfs.pyi +++ b/python/pyarrow-stubs/pyarrow/_gcsfs.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_gcsfs.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_hdfs.pyi b/python/pyarrow-stubs/pyarrow/_hdfs.pyi index 9d526298fa5e..b45038111f61 100644 --- a/python/pyarrow-stubs/pyarrow/_hdfs.pyi +++ b/python/pyarrow-stubs/pyarrow/_hdfs.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_hdfs.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_json.pyi b/python/pyarrow-stubs/pyarrow/_json.pyi index 64b0c2de30ad..4264fdaf35b7 100644 --- a/python/pyarrow-stubs/pyarrow/_json.pyi +++ b/python/pyarrow-stubs/pyarrow/_json.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_json.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_orc.pyi b/python/pyarrow-stubs/pyarrow/_orc.pyi index 0880ad15ffed..dafb6f54f4f8 100644 --- a/python/pyarrow-stubs/pyarrow/_orc.pyi +++ b/python/pyarrow-stubs/pyarrow/_orc.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_orc.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_parquet.pyi b/python/pyarrow-stubs/pyarrow/_parquet.pyi index 494f6ea9a9d2..ec09ead3d918 100644 --- a/python/pyarrow-stubs/pyarrow/_parquet.pyi +++ b/python/pyarrow-stubs/pyarrow/_parquet.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_parquet.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi b/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi index c14ce445549c..0c835d889e62 100644 --- a/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi +++ b/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_parquet_encryption.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi b/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi index e5f8211a4959..ca27b2108ae0 100644 --- a/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi +++ b/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_pyarrow_cpp_tests.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_s3fs.pyi b/python/pyarrow-stubs/pyarrow/_s3fs.pyi index c40968b1a95e..f1219553325d 100644 --- a/python/pyarrow-stubs/pyarrow/_s3fs.pyi +++ b/python/pyarrow-stubs/pyarrow/_s3fs.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_s3fs.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_substrait.pyi b/python/pyarrow-stubs/pyarrow/_substrait.pyi index f68dae186cbc..8916775be1a8 100644 --- a/python/pyarrow-stubs/pyarrow/_substrait.pyi +++ b/python/pyarrow-stubs/pyarrow/_substrait.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/_substrait.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/acero.pyi b/python/pyarrow-stubs/pyarrow/acero.pyi index 81f825799fc7..3e7b3736f5b0 100644 --- a/python/pyarrow-stubs/pyarrow/acero.pyi +++ b/python/pyarrow-stubs/pyarrow/acero.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from pyarrow._acero import AggregateNodeOptions as AggregateNodeOptions, AsofJoinNodeOptions as AsofJoinNodeOptions, Declaration as Declaration, ExecNodeOptions as ExecNodeOptions, FilterNodeOptions as FilterNodeOptions, HashJoinNodeOptions as HashJoinNodeOptions, OrderByNodeOptions as OrderByNodeOptions, ProjectNodeOptions as ProjectNodeOptions, RecordBatchReaderSourceNodeOptions as RecordBatchReaderSourceNodeOptions, TableSourceNodeOptions as TableSourceNodeOptions from pyarrow._dataset import ScanNodeOptions as ScanNodeOptions from pyarrow.compute import Expression as Expression, field as field diff --git a/python/pyarrow-stubs/pyarrow/benchmark.pyi b/python/pyarrow-stubs/pyarrow/benchmark.pyi index 0d2a20d9ae7c..68aa3ab3db99 100644 --- a/python/pyarrow-stubs/pyarrow/benchmark.pyi +++ b/python/pyarrow-stubs/pyarrow/benchmark.pyi @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from pyarrow.lib import benchmark_PandasObjectIsNull as benchmark_PandasObjectIsNull diff --git a/python/pyarrow-stubs/pyarrow/cffi.pyi b/python/pyarrow-stubs/pyarrow/cffi.pyi index 3bf775cb5cd1..69456bd3fc53 100644 --- a/python/pyarrow-stubs/pyarrow/cffi.pyi +++ b/python/pyarrow-stubs/pyarrow/cffi.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from _typeshed import Incomplete c_source: str diff --git a/python/pyarrow-stubs/pyarrow/compute.pyi b/python/pyarrow-stubs/pyarrow/compute.pyi index dad6b78caaad..f3232c6a4f49 100644 --- a/python/pyarrow-stubs/pyarrow/compute.pyi +++ b/python/pyarrow-stubs/pyarrow/compute.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pyarrow.lib from _typeshed import Incomplete from pyarrow._compute import ArraySortOptions as ArraySortOptions, AssumeTimezoneOptions as AssumeTimezoneOptions, CastOptions as CastOptions, CountOptions as CountOptions, CumulativeOptions as CumulativeOptions, CumulativeSumOptions as CumulativeSumOptions, DayOfWeekOptions as DayOfWeekOptions, DictionaryEncodeOptions as DictionaryEncodeOptions, ElementWiseAggregateOptions as ElementWiseAggregateOptions, Expression as Expression, ExtractRegexOptions as ExtractRegexOptions, ExtractRegexSpanOptions as ExtractRegexSpanOptions, FilterOptions as FilterOptions, Function as Function, FunctionOptions as FunctionOptions, FunctionRegistry as FunctionRegistry, HashAggregateFunction as HashAggregateFunction, HashAggregateKernel as HashAggregateKernel, IndexOptions as IndexOptions, InversePermutationOptions as InversePermutationOptions, JoinOptions as JoinOptions, Kernel as Kernel, ListFlattenOptions as ListFlattenOptions, ListSliceOptions as ListSliceOptions, MakeStructOptions as MakeStructOptions, MapLookupOptions as MapLookupOptions, MatchSubstringOptions as MatchSubstringOptions, ModeOptions as ModeOptions, NullOptions as NullOptions, PadOptions as PadOptions, PairwiseOptions as PairwiseOptions, PartitionNthOptions as PartitionNthOptions, PivotWiderOptions as PivotWiderOptions, QuantileOptions as QuantileOptions, RandomOptions as RandomOptions, RankOptions as RankOptions, RankQuantileOptions as RankQuantileOptions, ReplaceSliceOptions as ReplaceSliceOptions, ReplaceSubstringOptions as ReplaceSubstringOptions, RoundBinaryOptions as RoundBinaryOptions, RoundOptions as RoundOptions, RoundTemporalOptions as RoundTemporalOptions, RoundToMultipleOptions as RoundToMultipleOptions, RunEndEncodeOptions as RunEndEncodeOptions, ScalarAggregateFunction as ScalarAggregateFunction, ScalarAggregateKernel as ScalarAggregateKernel, ScalarAggregateOptions as ScalarAggregateOptions, ScalarFunction as ScalarFunction, ScalarKernel as ScalarKernel, ScatterOptions as ScatterOptions, SearchSortedOptions as SearchSortedOptions, SelectKOptions as SelectKOptions, SetLookupOptions as SetLookupOptions, SkewOptions as SkewOptions, SliceOptions as SliceOptions, SortOptions as SortOptions, SplitOptions as SplitOptions, SplitPatternOptions as SplitPatternOptions, StrftimeOptions as StrftimeOptions, StrptimeOptions as StrptimeOptions, StructFieldOptions as StructFieldOptions, TDigestOptions as TDigestOptions, TakeOptions as TakeOptions, TrimOptions as TrimOptions, UdfContext as UdfContext, Utf8NormalizeOptions as Utf8NormalizeOptions, VarianceOptions as VarianceOptions, VectorFunction as VectorFunction, VectorKernel as VectorKernel, WeekOptions as WeekOptions, WinsorizeOptions as WinsorizeOptions, ZeroFillOptions as ZeroFillOptions, call_function as call_function, call_tabular_function as call_tabular_function, function_registry as function_registry, get_function as get_function, list_functions as list_functions, register_aggregate_function as register_aggregate_function, register_scalar_function as register_scalar_function, register_tabular_function as register_tabular_function, register_vector_function as register_vector_function diff --git a/python/pyarrow-stubs/pyarrow/csv.pyi b/python/pyarrow-stubs/pyarrow/csv.pyi index 2c1757380f1b..ead7186863df 100644 --- a/python/pyarrow-stubs/pyarrow/csv.pyi +++ b/python/pyarrow-stubs/pyarrow/csv.pyi @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from pyarrow._csv import CSVStreamingReader as CSVStreamingReader, CSVWriter as CSVWriter, ConvertOptions as ConvertOptions, ISO8601 as ISO8601, InvalidRow as InvalidRow, ParseOptions as ParseOptions, ReadOptions as ReadOptions, WriteOptions as WriteOptions, open_csv as open_csv, read_csv as read_csv, write_csv as write_csv diff --git a/python/pyarrow-stubs/pyarrow/cuda.pyi b/python/pyarrow-stubs/pyarrow/cuda.pyi index fe0377327475..d09469c99319 100644 --- a/python/pyarrow-stubs/pyarrow/cuda.pyi +++ b/python/pyarrow-stubs/pyarrow/cuda.pyi @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from pyarrow._cuda import BufferReader as BufferReader, BufferWriter as BufferWriter, Context as Context, CudaBuffer as CudaBuffer, HostBuffer as HostBuffer, IpcMemHandle as IpcMemHandle, new_host_buffer as new_host_buffer, read_message as read_message, read_record_batch as read_record_batch, serialize_record_batch as serialize_record_batch diff --git a/python/pyarrow-stubs/pyarrow/dataset.pyi b/python/pyarrow-stubs/pyarrow/dataset.pyi index 4019efd3e088..8b3272168662 100644 --- a/python/pyarrow-stubs/pyarrow/dataset.pyi +++ b/python/pyarrow-stubs/pyarrow/dataset.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from pyarrow._dataset import CsvFileFormat as CsvFileFormat, CsvFragmentScanOptions as CsvFragmentScanOptions, Dataset as Dataset, DatasetFactory as DatasetFactory, DirectoryPartitioning as DirectoryPartitioning, FeatherFileFormat as FeatherFileFormat, FileFormat as FileFormat, FileFragment as FileFragment, FileSystemDataset as FileSystemDataset, FileSystemDatasetFactory as FileSystemDatasetFactory, FileSystemFactoryOptions as FileSystemFactoryOptions, FileWriteOptions as FileWriteOptions, FilenamePartitioning as FilenamePartitioning, Fragment as Fragment, FragmentScanOptions as FragmentScanOptions, HivePartitioning as HivePartitioning, InMemoryDataset as InMemoryDataset, IpcFileFormat as IpcFileFormat, IpcFileWriteOptions as IpcFileWriteOptions, JsonFileFormat as JsonFileFormat, JsonFragmentScanOptions as JsonFragmentScanOptions, Partitioning as Partitioning, PartitioningFactory as PartitioningFactory, Scanner as Scanner, TaggedRecordBatch as TaggedRecordBatch, UnionDataset as UnionDataset, UnionDatasetFactory as UnionDatasetFactory, WrittenFile as WrittenFile, _filesystemdataset_write as _filesystemdataset_write, get_partition_keys as get_partition_keys from pyarrow._dataset_orc import OrcFileFormat as OrcFileFormat from pyarrow._dataset_parquet import ParquetDatasetFactory as ParquetDatasetFactory, ParquetFactoryOptions as ParquetFactoryOptions, ParquetFileFormat as ParquetFileFormat, ParquetFileFragment as ParquetFileFragment, ParquetFileWriteOptions as ParquetFileWriteOptions, ParquetFragmentScanOptions as ParquetFragmentScanOptions, ParquetReadOptions as ParquetReadOptions, RowGroupInfo as RowGroupInfo diff --git a/python/pyarrow-stubs/pyarrow/feather.pyi b/python/pyarrow-stubs/pyarrow/feather.pyi index 65ab238475a1..ea0053a5765b 100644 --- a/python/pyarrow-stubs/pyarrow/feather.pyi +++ b/python/pyarrow-stubs/pyarrow/feather.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from _typeshed import Incomplete from pyarrow import _feather as _feather from pyarrow._feather import FeatherError as FeatherError diff --git a/python/pyarrow-stubs/pyarrow/flight.pyi b/python/pyarrow-stubs/pyarrow/flight.pyi index 7f3b30863f9f..42ff65f0df41 100644 --- a/python/pyarrow-stubs/pyarrow/flight.pyi +++ b/python/pyarrow-stubs/pyarrow/flight.pyi @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from pyarrow._flight import Action as Action, ActionType as ActionType, BasicAuth as BasicAuth, CallInfo as CallInfo, CertKeyPair as CertKeyPair, ClientAuthHandler as ClientAuthHandler, ClientMiddleware as ClientMiddleware, ClientMiddlewareFactory as ClientMiddlewareFactory, DescriptorType as DescriptorType, FlightCallOptions as FlightCallOptions, FlightCancelledError as FlightCancelledError, FlightClient as FlightClient, FlightDataStream as FlightDataStream, FlightDescriptor as FlightDescriptor, FlightEndpoint as FlightEndpoint, FlightError as FlightError, FlightInfo as FlightInfo, FlightInternalError as FlightInternalError, FlightMetadataReader as FlightMetadataReader, FlightMetadataWriter as FlightMetadataWriter, FlightMethod as FlightMethod, FlightServerBase as FlightServerBase, FlightServerError as FlightServerError, FlightStreamChunk as FlightStreamChunk, FlightStreamReader as FlightStreamReader, FlightStreamWriter as FlightStreamWriter, FlightTimedOutError as FlightTimedOutError, FlightUnauthenticatedError as FlightUnauthenticatedError, FlightUnauthorizedError as FlightUnauthorizedError, FlightUnavailableError as FlightUnavailableError, FlightWriteSizeExceededError as FlightWriteSizeExceededError, GeneratorStream as GeneratorStream, Location as Location, MetadataRecordBatchReader as MetadataRecordBatchReader, MetadataRecordBatchWriter as MetadataRecordBatchWriter, RecordBatchStream as RecordBatchStream, Result as Result, SchemaResult as SchemaResult, ServerAuthHandler as ServerAuthHandler, ServerCallContext as ServerCallContext, ServerMiddleware as ServerMiddleware, ServerMiddlewareFactory as ServerMiddlewareFactory, Ticket as Ticket, TracingServerMiddlewareFactory as TracingServerMiddlewareFactory, connect as connect diff --git a/python/pyarrow-stubs/pyarrow/fs.pyi b/python/pyarrow-stubs/pyarrow/fs.pyi index b498d5a167d1..78e8e3f3afda 100644 --- a/python/pyarrow-stubs/pyarrow/fs.pyi +++ b/python/pyarrow-stubs/pyarrow/fs.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from _typeshed import Incomplete from pyarrow._azurefs import AzureFileSystem as AzureFileSystem from pyarrow._fs import FileInfo as FileInfo, FileSelector as FileSelector, FileSystem as FileSystem, FileSystemHandler as FileSystemHandler, FileType as FileType, LocalFileSystem as LocalFileSystem, PyFileSystem as PyFileSystem, SubTreeFileSystem as SubTreeFileSystem, _MockFileSystem as _MockFileSystem, _copy_files as _copy_files, _copy_files_selector as _copy_files_selector diff --git a/python/pyarrow-stubs/pyarrow/gandiva.pyi b/python/pyarrow-stubs/pyarrow/gandiva.pyi index 3b7ca3be7e1d..13d0aa18c25f 100644 --- a/python/pyarrow-stubs/pyarrow/gandiva.pyi +++ b/python/pyarrow-stubs/pyarrow/gandiva.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/gandiva.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi b/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi index ffa012116d98..3d04612d9651 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from .from_dataframe import from_dataframe as from_dataframe diff --git a/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi b/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi index 471da201a818..9abec869d2db 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import enum import pyarrow as pa from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/interchange/column.pyi b/python/pyarrow-stubs/pyarrow/interchange/column.pyi index fbbce73a9536..d0d32f193b8b 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/column.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/column.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import enum import pyarrow as pa from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi b/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi index 5c46eb9e754c..6f4740b494d2 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pyarrow as pa from _typeshed import Incomplete from pyarrow.interchange.column import _PyArrowColumn as _PyArrowColumn diff --git a/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi b/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi index 8c3ab6757686..0062bd275b00 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pyarrow as pa from pyarrow.interchange.column import ColumnBuffers as ColumnBuffers, ColumnNullType as ColumnNullType, Dtype as Dtype, DtypeKind as DtypeKind from typing import Any diff --git a/python/pyarrow-stubs/pyarrow/ipc.pyi b/python/pyarrow-stubs/pyarrow/ipc.pyi index ce305f4df1c2..07ddda7bb164 100644 --- a/python/pyarrow-stubs/pyarrow/ipc.pyi +++ b/python/pyarrow-stubs/pyarrow/ipc.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pyarrow.lib as lib from _typeshed import Incomplete from pyarrow.lib import Alignment as Alignment, IpcReadOptions as IpcReadOptions, IpcWriteOptions as IpcWriteOptions, Message as Message, MessageReader as MessageReader, MetadataVersion as MetadataVersion, ReadStats as ReadStats, RecordBatchReader as RecordBatchReader, WriteStats as WriteStats, _ReadPandasMixin as _ReadPandasMixin, get_record_batch_size as get_record_batch_size, get_tensor_size as get_tensor_size, read_message as read_message, read_record_batch as read_record_batch, read_schema as read_schema, read_tensor as read_tensor, write_tensor as write_tensor diff --git a/python/pyarrow-stubs/pyarrow/json.pyi b/python/pyarrow-stubs/pyarrow/json.pyi index 1f4769cff92e..4a5e98125a51 100644 --- a/python/pyarrow-stubs/pyarrow/json.pyi +++ b/python/pyarrow-stubs/pyarrow/json.pyi @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from pyarrow._json import ParseOptions as ParseOptions, ReadOptions as ReadOptions, open_json as open_json, read_json as read_json diff --git a/python/pyarrow-stubs/pyarrow/jvm.pyi b/python/pyarrow-stubs/pyarrow/jvm.pyi index 56e7e88f0ecc..a7c3cb64e4fb 100644 --- a/python/pyarrow-stubs/pyarrow/jvm.pyi +++ b/python/pyarrow-stubs/pyarrow/jvm.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from _typeshed import Incomplete class _JvmBufferNanny: diff --git a/python/pyarrow-stubs/pyarrow/lib.pyi b/python/pyarrow-stubs/pyarrow/lib.pyi index 357830b43bba..8f73a5ad7067 100644 --- a/python/pyarrow-stubs/pyarrow/lib.pyi +++ b/python/pyarrow-stubs/pyarrow/lib.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Generated by stubgen-pyx from python/pyarrow/lib.pyx from _typeshed import Incomplete diff --git a/python/pyarrow-stubs/pyarrow/orc.pyi b/python/pyarrow-stubs/pyarrow/orc.pyi index 67d572bcfc53..9b1abb8b2b3c 100644 --- a/python/pyarrow-stubs/pyarrow/orc.pyi +++ b/python/pyarrow-stubs/pyarrow/orc.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import types from _typeshed import Incomplete from pyarrow.fs import _resolve_filesystem_and_path as _resolve_filesystem_and_path diff --git a/python/pyarrow-stubs/pyarrow/pandas_compat.pyi b/python/pyarrow-stubs/pyarrow/pandas_compat.pyi index 6b80ab3e0d94..9e04065d39f7 100644 --- a/python/pyarrow-stubs/pyarrow/pandas_compat.pyi +++ b/python/pyarrow-stubs/pyarrow/pandas_compat.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from _typeshed import Incomplete from pyarrow.lib import _pandas_api as _pandas_api, frombytes as frombytes, is_threading_enabled as is_threading_enabled diff --git a/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi b/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi index bb67a43fa4e5..14d5874f5c46 100644 --- a/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi +++ b/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from .core import * diff --git a/python/pyarrow-stubs/pyarrow/parquet/core.pyi b/python/pyarrow-stubs/pyarrow/parquet/core.pyi index 16b1265c5fd6..1fe5b27ea5a6 100644 --- a/python/pyarrow-stubs/pyarrow/parquet/core.pyi +++ b/python/pyarrow-stubs/pyarrow/parquet/core.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import types from _typeshed import Incomplete from pyarrow._parquet import ColumnChunkMetaData as ColumnChunkMetaData, ColumnSchema as ColumnSchema, FileDecryptionProperties as FileDecryptionProperties, FileEncryptionProperties as FileEncryptionProperties, FileMetaData as FileMetaData, ParquetLogicalType as ParquetLogicalType, ParquetReader as ParquetReader, ParquetSchema as ParquetSchema, RowGroupMetaData as RowGroupMetaData, SortingColumn as SortingColumn, Statistics as Statistics diff --git a/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi b/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi index 4b18b68cd1de..04211f468f6e 100644 --- a/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi +++ b/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from pyarrow._parquet_encryption import CryptoFactory as CryptoFactory, DecryptionConfiguration as DecryptionConfiguration, EncryptionConfiguration as EncryptionConfiguration, KmsClient as KmsClient, KmsConnectionConfig as KmsConnectionConfig, create_decryption_properties as create_decryption_properties, create_encryption_properties as create_encryption_properties diff --git a/python/pyarrow-stubs/pyarrow/substrait.pyi b/python/pyarrow-stubs/pyarrow/substrait.pyi index ff74e7100fb3..bf0b5922e1d8 100644 --- a/python/pyarrow-stubs/pyarrow/substrait.pyi +++ b/python/pyarrow-stubs/pyarrow/substrait.pyi @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from pyarrow._substrait import BoundExpressions as BoundExpressions, SubstraitSchema as SubstraitSchema, deserialize_expressions as deserialize_expressions, deserialize_schema as deserialize_schema, get_supported_functions as get_supported_functions, run_query as run_query, serialize_expressions as serialize_expressions, serialize_schema as serialize_schema diff --git a/python/pyarrow-stubs/pyarrow/types.pyi b/python/pyarrow-stubs/pyarrow/types.pyi index 9ea7daf6651e..863d12a3e370 100644 --- a/python/pyarrow-stubs/pyarrow/types.pyi +++ b/python/pyarrow-stubs/pyarrow/types.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from _typeshed import Incomplete from enum import IntEnum from pyarrow.lib import is_boolean_value as is_boolean_value, is_float_value as is_float_value, is_integer_value as is_integer_value diff --git a/python/pyarrow-stubs/pyarrow/util.pyi b/python/pyarrow-stubs/pyarrow/util.pyi index 640bb7f7646a..9cc1eedabab6 100644 --- a/python/pyarrow-stubs/pyarrow/util.pyi +++ b/python/pyarrow-stubs/pyarrow/util.pyi @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + _DEPR_MSG: str def doc(*docstrings, **params): ... From 7275d3416ffd974b7a3d238f8d72200d265e3c62 Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Fri, 9 Oct 2026 18:14:21 +0100 Subject: [PATCH 6/8] [Python] Annotate the stubs from the sources and the runtime scripts/add_types_to_stubs.py without traces: return types read from the Cython return statements, parameter and return types from the numpydoc docstrings and from defaults, module and class attributes from their runtime values, namedtuples as NamedTuple classes, leaked Cython types spelled as Python passes them, explicit re-exports, classes ordered after their bases, imports first and once, and Self from __enter__. --- python/pyarrow-stubs/pyarrow/__init__.pyi | 58 +- python/pyarrow-stubs/pyarrow/_acero.pyi | 100 +- python/pyarrow-stubs/pyarrow/_azurefs.pyi | 21 +- python/pyarrow-stubs/pyarrow/_compute.pyi | 856 ++- .../pyarrow/_compute_docstrings.pyi | 5 +- python/pyarrow-stubs/pyarrow/_csv.pyi | 447 +- python/pyarrow-stubs/pyarrow/_dataset.pyi | 675 ++- python/pyarrow-stubs/pyarrow/_dataset_orc.pyi | 26 +- .../pyarrow/_dataset_parquet.pyi | 312 +- .../pyarrow/_dataset_parquet_encryption.pyi | 31 +- python/pyarrow-stubs/pyarrow/_feather.pyi | 35 +- python/pyarrow-stubs/pyarrow/_flight.pyi | 765 ++- python/pyarrow-stubs/pyarrow/_fs.pyi | 287 +- python/pyarrow-stubs/pyarrow/_gcsfs.pyi | 35 +- python/pyarrow-stubs/pyarrow/_hdfs.pyi | 28 +- python/pyarrow-stubs/pyarrow/_json.pyi | 111 +- python/pyarrow-stubs/pyarrow/_orc.pyi | 119 +- python/pyarrow-stubs/pyarrow/_parquet.pyi | 647 +- .../pyarrow/_parquet_encryption.pyi | 232 +- .../pyarrow/_pyarrow_cpp_tests.pyi | 24 +- python/pyarrow-stubs/pyarrow/_s3fs.pyi | 57 +- python/pyarrow-stubs/pyarrow/_substrait.pyi | 72 +- python/pyarrow-stubs/pyarrow/acero.pyi | 42 +- python/pyarrow-stubs/pyarrow/benchmark.pyi | 5 + python/pyarrow-stubs/pyarrow/cffi.pyi | 5 +- python/pyarrow-stubs/pyarrow/compute.pyi | 1183 +++- python/pyarrow-stubs/pyarrow/csv.pyi | 5 + python/pyarrow-stubs/pyarrow/dataset.pyi | 59 +- python/pyarrow-stubs/pyarrow/feather.pyi | 39 +- python/pyarrow-stubs/pyarrow/flight.pyi | 5 + python/pyarrow-stubs/pyarrow/fs.pyi | 100 +- .../pyarrow/interchange/__init__.pyi | 5 + .../pyarrow/interchange/buffer.pyi | 28 +- .../pyarrow/interchange/column.pyi | 69 +- .../pyarrow/interchange/dataframe.pyi | 56 +- .../pyarrow/interchange/from_dataframe.pyi | 49 +- python/pyarrow-stubs/pyarrow/ipc.pyi | 58 +- python/pyarrow-stubs/pyarrow/json.pyi | 5 + python/pyarrow-stubs/pyarrow/jvm.pyi | 53 +- python/pyarrow-stubs/pyarrow/lib.pyi | 5355 ++++++++++++----- python/pyarrow-stubs/pyarrow/orc.pyi | 124 +- .../pyarrow-stubs/pyarrow/pandas_compat.pyi | 142 +- .../pyarrow/parquet/__init__.pyi | 5 + python/pyarrow-stubs/pyarrow/parquet/core.pyi | 188 +- .../pyarrow/parquet/encryption.pyi | 5 + python/pyarrow-stubs/pyarrow/substrait.pyi | 5 + python/pyarrow-stubs/pyarrow/types.pyi | 220 +- python/pyarrow-stubs/pyarrow/util.pyi | 60 +- 48 files changed, 9356 insertions(+), 3457 deletions(-) diff --git a/python/pyarrow-stubs/pyarrow/__init__.pyi b/python/pyarrow-stubs/pyarrow/__init__.pyi index 00007a0e7567..b617162a9377 100644 --- a/python/pyarrow-stubs/pyarrow/__init__.pyi +++ b/python/pyarrow-stubs/pyarrow/__init__.pyi @@ -15,21 +15,51 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete from pyarrow.ipc import Message as Message, MessageReader as MessageReader, MetadataVersion as MetadataVersion, RecordBatchFileReader as RecordBatchFileReader, RecordBatchFileWriter as RecordBatchFileWriter, RecordBatchStreamReader as RecordBatchStreamReader, RecordBatchStreamWriter as RecordBatchStreamWriter, deserialize_pandas as deserialize_pandas, serialize_pandas as serialize_pandas from pyarrow.lib import Array as Array, ArrowCancelled as ArrowCancelled, ArrowCapacityError as ArrowCapacityError, ArrowException as ArrowException, ArrowIOError as ArrowIOError, ArrowIndexError as ArrowIndexError, ArrowInvalid as ArrowInvalid, ArrowKeyError as ArrowKeyError, ArrowMemoryError as ArrowMemoryError, ArrowNotImplementedError as ArrowNotImplementedError, ArrowSerializationError as ArrowSerializationError, ArrowTypeError as ArrowTypeError, BaseExtensionType as BaseExtensionType, BinaryArray as BinaryArray, BinaryScalar as BinaryScalar, BinaryViewArray as BinaryViewArray, BinaryViewScalar as BinaryViewScalar, Bool8Array as Bool8Array, Bool8Scalar as Bool8Scalar, Bool8Type as Bool8Type, BooleanArray as BooleanArray, BooleanScalar as BooleanScalar, Buffer as Buffer, BufferOutputStream as BufferOutputStream, BufferReader as BufferReader, BufferedInputStream as BufferedInputStream, BufferedOutputStream as BufferedOutputStream, BuildInfo as BuildInfo, CacheOptions as CacheOptions, ChunkedArray as ChunkedArray, Codec as Codec, CompressedInputStream as CompressedInputStream, CompressedOutputStream as CompressedOutputStream, CppBuildInfo as CppBuildInfo, DataType as DataType, Date32Array as Date32Array, Date32Scalar as Date32Scalar, Date64Array as Date64Array, Date64Scalar as Date64Scalar, Decimal128Array as Decimal128Array, Decimal128Scalar as Decimal128Scalar, Decimal128Type as Decimal128Type, Decimal256Array as Decimal256Array, Decimal256Scalar as Decimal256Scalar, Decimal256Type as Decimal256Type, Decimal32Array as Decimal32Array, Decimal32Scalar as Decimal32Scalar, Decimal32Type as Decimal32Type, Decimal64Array as Decimal64Array, Decimal64Scalar as Decimal64Scalar, Decimal64Type as Decimal64Type, DenseUnionType as DenseUnionType, Device as Device, DeviceAllocationType as DeviceAllocationType, DictionaryArray as DictionaryArray, DictionaryMemo as DictionaryMemo, DictionaryScalar as DictionaryScalar, DictionaryType as DictionaryType, DoubleArray as DoubleArray, DoubleScalar as DoubleScalar, DurationArray as DurationArray, DurationScalar as DurationScalar, DurationType as DurationType, ExtensionArray as ExtensionArray, ExtensionScalar as ExtensionScalar, ExtensionType as ExtensionType, Field as Field, FixedShapeTensorArray as FixedShapeTensorArray, FixedShapeTensorScalar as FixedShapeTensorScalar, FixedShapeTensorType as FixedShapeTensorType, FixedSizeBinaryArray as FixedSizeBinaryArray, FixedSizeBinaryScalar as FixedSizeBinaryScalar, FixedSizeBinaryType as FixedSizeBinaryType, FixedSizeBufferWriter as FixedSizeBufferWriter, FixedSizeListArray as FixedSizeListArray, FixedSizeListScalar as FixedSizeListScalar, FixedSizeListType as FixedSizeListType, FloatArray as FloatArray, FloatScalar as FloatScalar, FloatingPointArray as FloatingPointArray, HalfFloatArray as HalfFloatArray, HalfFloatScalar as HalfFloatScalar, Int16Array as Int16Array, Int16Scalar as Int16Scalar, Int32Array as Int32Array, Int32Scalar as Int32Scalar, Int64Array as Int64Array, Int64Scalar as Int64Scalar, Int8Array as Int8Array, Int8Scalar as Int8Scalar, IntegerArray as IntegerArray, JsonArray as JsonArray, JsonScalar as JsonScalar, JsonType as JsonType, KeyValueMetadata as KeyValueMetadata, LargeBinaryArray as LargeBinaryArray, LargeBinaryScalar as LargeBinaryScalar, LargeListArray as LargeListArray, LargeListScalar as LargeListScalar, LargeListType as LargeListType, LargeListViewArray as LargeListViewArray, LargeListViewScalar as LargeListViewScalar, LargeListViewType as LargeListViewType, LargeStringArray as LargeStringArray, LargeStringScalar as LargeStringScalar, ListArray as ListArray, ListScalar as ListScalar, ListType as ListType, ListViewArray as ListViewArray, ListViewScalar as ListViewScalar, ListViewType as ListViewType, LoggingMemoryPool as LoggingMemoryPool, MapArray as MapArray, MapScalar as MapScalar, MapType as MapType, MemoryManager as MemoryManager, MemoryMappedFile as MemoryMappedFile, MemoryPool as MemoryPool, MockOutputStream as MockOutputStream, MonthDayNano as MonthDayNano, MonthDayNanoIntervalArray as MonthDayNanoIntervalArray, MonthDayNanoIntervalScalar as MonthDayNanoIntervalScalar, NA as NA, NativeFile as NativeFile, NullArray as NullArray, NullScalar as NullScalar, NumericArray as NumericArray, OSFile as OSFile, OpaqueArray as OpaqueArray, OpaqueScalar as OpaqueScalar, OpaqueType as OpaqueType, ProxyMemoryPool as ProxyMemoryPool, PythonFile as PythonFile, RecordBatch as RecordBatch, RecordBatchReader as RecordBatchReader, ResizableBuffer as ResizableBuffer, RunEndEncodedArray as RunEndEncodedArray, RunEndEncodedScalar as RunEndEncodedScalar, RunEndEncodedType as RunEndEncodedType, RuntimeInfo as RuntimeInfo, Scalar as Scalar, Schema as Schema, SparseCOOTensor as SparseCOOTensor, SparseCSCMatrix as SparseCSCMatrix, SparseCSFTensor as SparseCSFTensor, SparseCSRMatrix as SparseCSRMatrix, SparseUnionType as SparseUnionType, StringArray as StringArray, StringScalar as StringScalar, StringViewArray as StringViewArray, StringViewScalar as StringViewScalar, StructArray as StructArray, StructScalar as StructScalar, StructType as StructType, Table as Table, TableGroupBy as TableGroupBy, Tensor as Tensor, Time32Array as Time32Array, Time32Scalar as Time32Scalar, Time32Type as Time32Type, Time64Array as Time64Array, Time64Scalar as Time64Scalar, Time64Type as Time64Type, TimestampArray as TimestampArray, TimestampScalar as TimestampScalar, TimestampType as TimestampType, TransformInputStream as TransformInputStream, UInt16Array as UInt16Array, UInt16Scalar as UInt16Scalar, UInt32Array as UInt32Array, UInt32Scalar as UInt32Scalar, UInt64Array as UInt64Array, UInt64Scalar as UInt64Scalar, UInt8Array as UInt8Array, UInt8Scalar as UInt8Scalar, UnionArray as UnionArray, UnionScalar as UnionScalar, UnionType as UnionType, UnknownExtensionType as UnknownExtensionType, UuidArray as UuidArray, UuidScalar as UuidScalar, UuidType as UuidType, VersionInfo as VersionInfo, allocate_buffer as allocate_buffer, arange as arange, array as array, binary as binary, binary_view as binary_view, bool8 as bool8, bool_ as bool_, build_info as build_info, chunked_array as chunked_array, compress as compress, concat_arrays as concat_arrays, concat_batches as concat_batches, concat_tables as concat_tables, cpp_build_info as cpp_build_info, cpp_version as cpp_version, cpp_version_info as cpp_version_info, cpu_count as cpu_count, create_memory_map as create_memory_map, date32 as date32, date64 as date64, decimal128 as decimal128, decimal256 as decimal256, decimal32 as decimal32, decimal64 as decimal64, decompress as decompress, default_cpu_memory_manager as default_cpu_memory_manager, default_memory_pool as default_memory_pool, dense_union as dense_union, dictionary as dictionary, duration as duration, enable_signal_handlers as enable_signal_handlers, field as field, fixed_shape_tensor as fixed_shape_tensor, float16 as float16, float32 as float32, float64 as float64, foreign_buffer as foreign_buffer, from_numpy_dtype as from_numpy_dtype, infer_type as infer_type, input_stream as input_stream, int16 as int16, int32 as int32, int64 as int64, int8 as int8, io_thread_count as io_thread_count, is_opentelemetry_enabled as is_opentelemetry_enabled, jemalloc_memory_pool as jemalloc_memory_pool, jemalloc_set_decay_ms as jemalloc_set_decay_ms, json_ as json_, large_binary as large_binary, large_list as large_list, large_list_view as large_list_view, large_string as large_string, large_utf8 as large_utf8, list_ as list_, list_view as list_view, log_memory_allocations as log_memory_allocations, logging_memory_pool as logging_memory_pool, map_ as map_, memory_map as memory_map, mimalloc_memory_pool as mimalloc_memory_pool, month_day_nano_interval as month_day_nano_interval, null as null, nulls as nulls, opaque as opaque, output_stream as output_stream, proxy_memory_pool as proxy_memory_pool, py_buffer as py_buffer, record_batch as record_batch, register_extension_type as register_extension_type, repeat as repeat, run_end_encoded as run_end_encoded, runtime_info as runtime_info, scalar as scalar, schema as schema, set_cpu_count as set_cpu_count, set_io_thread_count as set_io_thread_count, set_memory_pool as set_memory_pool, set_timezone_db_path as set_timezone_db_path, sparse_union as sparse_union, string as string, string_view as string_view, struct as struct, supported_memory_backends as supported_memory_backends, system_memory_pool as system_memory_pool, table as table, time32 as time32, time64 as time64, timestamp as timestamp, total_allocated_bytes as total_allocated_bytes, transcoding_input_stream as transcoding_input_stream, type_for_alias as type_for_alias, uint16 as uint16, uint32 as uint32, uint64 as uint64, uint8 as uint8, unify_schemas as unify_schemas, union as union, unregister_extension_type as unregister_extension_type, utf8 as utf8, uuid as uuid from pyarrow.util import _deprecate_api as _deprecate_api, _deprecate_class as _deprecate_class - -def parse_git(root, **kwargs): ... -def show_versions() -> None: ... -def _module_is_available(module): ... -def _filesystem_is_available(fs): ... -def have_libhdfs(): ... -def show_info() -> None: ... -def get_include(): ... -def _get_pkg_config_executable(): ... -def _has_pkg_config(pkgname): ... -def _read_pkg_config_variable(pkgname, cli_args): ... -def get_libraries(): ... -def create_library_symlinks(): ... -def get_library_dirs(): ... from pyarrow.lib import _NULL as NULL + +def parse_git(root, **kwargs): + ... + +def show_versions() -> None: + ... + +def _module_is_available(module): + ... + +def _filesystem_is_available(fs): + ... + +def have_libhdfs(): + ... + +def show_info() -> None: + ... + +def get_include(): + ... + +def _get_pkg_config_executable(): + ... + +def _has_pkg_config(pkgname): + ... + +def _read_pkg_config_variable(pkgname, cli_args): + ... + +def get_libraries(): + ... + +def create_library_symlinks(): + ... + +def get_library_dirs(): + ... diff --git a/python/pyarrow-stubs/pyarrow/_acero.pyi b/python/pyarrow-stubs/pyarrow/_acero.pyi index 3050fe2faf58..5f3901e2cdf3 100644 --- a/python/pyarrow-stubs/pyarrow/_acero.pyi +++ b/python/pyarrow-stubs/pyarrow/_acero.pyi @@ -14,74 +14,118 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_acero.pyx -from _typeshed import Incomplete +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from pyarrow.lib import _Weakrefable +from _typeshed import Incomplete from pyarrow._compute import Expression -from pyarrow.includes.common import * -from pyarrow.includes.libarrow import * -from pyarrow.includes.libarrow_acero import * from pyarrow.lib import * from pyarrow.lib import RecordBatchReader, Table - class ExecNodeOptions(_Weakrefable): __slots__ = () class _TableSourceNodeOptions(ExecNodeOptions): - def _set_options(self, table: Table): ... + + def _set_options(self, table: Table) -> None: + ... class TableSourceNodeOptions(_TableSourceNodeOptions): - def __init__(self, table: Table): ... + + def __init__(self, table: Table): + ... class _RecordBatchReaderSourceNodeOptions(ExecNodeOptions): - def _set_options(self, reader: RecordBatchReader): ... + + def _set_options(self, reader: RecordBatchReader) -> None: + ... class RecordBatchReaderSourceNodeOptions(_RecordBatchReaderSourceNodeOptions): - def __init__(self, reader: RecordBatchReader): ... + + def __init__(self, reader: RecordBatchReader): + ... class _FilterNodeOptions(ExecNodeOptions): - def _set_options(self, filter_expression: Expression): ... + + def _set_options(self, filter_expression: Expression) -> None: + ... class FilterNodeOptions(_FilterNodeOptions): - def __init__(self, filter_expression: Expression): ... + + def __init__(self, filter_expression: Expression): + ... class _ProjectNodeOptions(ExecNodeOptions): - def _set_options(self, expressions, names=None): ... + + def _set_options(self, expressions, names=None) -> None: + ... class ProjectNodeOptions(_ProjectNodeOptions): - def __init__(self, expressions, names=None): ... + + def __init__(self, expressions, names=None): + ... class _AggregateNodeOptions(ExecNodeOptions): - def _set_options(self, aggregates, keys=None): ... + + def _set_options(self, aggregates, keys=None) -> None: + ... class AggregateNodeOptions(_AggregateNodeOptions): - def __init__(self, aggregates, keys=None): ... + + def __init__(self, aggregates, keys=None): + ... class _OrderByNodeOptions(ExecNodeOptions): - def _set_options(self, sort_keys, null_placement): ... + + def _set_options(self, sort_keys, null_placement) -> None: + ... class OrderByNodeOptions(_OrderByNodeOptions): - def __init__(self, sort_keys=(), *, null_placement=None): ... + + def __init__(self, sort_keys=(), *, null_placement=None): + ... class _HashJoinNodeOptions(ExecNodeOptions): - def _set_options(self, join_type, left_keys, right_keys, left_output=None, right_output=None, output_suffix_for_left='', output_suffix_for_right='', filter_expression: Expression | None=None): ... + + def _set_options(self, join_type, left_keys, right_keys, left_output=None, right_output=None, output_suffix_for_left: str='', output_suffix_for_right: str='', filter_expression: Expression | None=None) -> None: + ... class HashJoinNodeOptions(_HashJoinNodeOptions): - def __init__(self, join_type, left_keys, right_keys, left_output=None, right_output=None, output_suffix_for_left='', output_suffix_for_right='', filter_expression=None): ... + + def __init__(self, join_type, left_keys, right_keys, left_output=None, right_output=None, output_suffix_for_left: str='', output_suffix_for_right: str='', filter_expression=None): + ... class _AsofJoinNodeOptions(ExecNodeOptions): - def _set_options(self, left_on, left_by, right_on, right_by, tolerance): ... + + def _set_options(self, left_on, left_by, right_on, right_by, tolerance) -> None: + ... class AsofJoinNodeOptions(_AsofJoinNodeOptions): - def __init__(self, left_on, left_by, right_on, right_by, tolerance): ... + + def __init__(self, left_on, left_by, right_on, right_by, tolerance): + ... class Declaration(_Weakrefable): - def __init__(self, factory_name, options: ExecNodeOptions, inputs=None): ... + + def __init__(self, factory_name, options: ExecNodeOptions, inputs=None): + ... + @staticmethod - def from_sequence(decls): ... - def __str__(self): ... - def __repr__(self): ... - def to_table(self, use_threads: bool=True): ... - def to_reader(self, use_threads: bool=True): ... + def from_sequence(decls: list[Declaration]) -> Declaration: + ... + + def __str__(self) -> str: + ... + + def __repr__(self) -> str: + ... + + def to_table(self, use_threads: bool=True) -> pyarrow.Table: + ... + + def to_reader(self, use_threads: bool=True) -> pyarrow.RecordBatchReader: + ... diff --git a/python/pyarrow-stubs/pyarrow/_azurefs.pyi b/python/pyarrow-stubs/pyarrow/_azurefs.pyi index 2283329f838f..e9da70ac4d20 100644 --- a/python/pyarrow-stubs/pyarrow/_azurefs.pyi +++ b/python/pyarrow-stubs/pyarrow/_azurefs.pyi @@ -14,16 +14,23 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_azurefs.pyx -from _typeshed import Incomplete +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete from pyarrow._fs import FileSystem -from pyarrow.includes.libarrow_fs import * - class AzureFileSystem(FileSystem): - def __init__(self, account_name, *, account_key=None, blob_storage_authority=None, blob_storage_scheme=None, client_id=None, client_secret=None, dfs_storage_authority=None, dfs_storage_scheme=None, sas_token=None, tenant_id=None): ... + + def __init__(self, account_name, *, account_key=None, blob_storage_authority=None, blob_storage_scheme=None, client_id=None, client_secret=None, dfs_storage_authority=None, dfs_storage_scheme=None, sas_token=None, tenant_id=None): + ... + @staticmethod - def _reconstruct(kwargs): ... - def __reduce__(self): ... + def _reconstruct(kwargs) -> AzureFileSystem: + ... + + def __reduce__(self) -> tuple: + ... diff --git a/python/pyarrow-stubs/pyarrow/_compute.pyi b/python/pyarrow-stubs/pyarrow/_compute.pyi index e2d54ad8ff2d..cb4715d597b6 100644 --- a/python/pyarrow-stubs/pyarrow/_compute.pyi +++ b/python/pyarrow-stubs/pyarrow/_compute.pyi @@ -14,551 +14,963 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_compute.pyx -from _typeshed import Incomplete +import os +import re +import pyarrow +from typing import IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from pyarrow.lib import _Weakrefable +from _typeshed import Incomplete from collections import namedtuple from typing import Any, Callable - import pyarrow.lib as lib -from pyarrow.includes.common import * -from pyarrow.includes.libarrow import * from pyarrow.lib import * from typing_extensions import TypeAlias +__pas: Incomplete +_substrait_msg: str +SUPPORTED_INPUT_ARR_TYPES: tuple[Incomplete, ...] + +class FunctionDoc(NamedTuple): + summary: Incomplete + description: Incomplete + arg_names: Incomplete + options_class: Incomplete + options_required: Incomplete +_deserialize: Incomplete + +def _pas(): + ... + +def _forbid_instantiation(klass, subclasses_instead: bool=True): + ... + +def function_registry(): + ... + +def get_function(name: str): + ... + +def list_functions(): + ... + +def call_function(name: str, args: list, options=None, memory_pool: pyarrow.MemoryPool | None=None, length: int | None=None): + ... + +def _raise_invalid_function_option(value, description, *, exception_class=ValueError): + ... + +def _skip_nulls_doc() -> str: + ... -__pas = None -_substrait_msg = 'The pyarrow installation is not built with support for Substrait.' -SUPPORTED_INPUT_ARR_TYPES = (list, tuple) -FunctionDoc = namedtuple('FunctionDoc', ('summary', 'description', 'arg_names', 'options_class', 'options_required')) -_deserialize = Expression._deserialize +def _min_count_doc(*, default) -> str: + ... + +def _get_udf_context(memory_pool, batch_length): + ... + +def register_scalar_function(func: Callable[..., Any], function_name: str, function_doc: dict, in_types, out_type: pyarrow.DataType, func_registry: FunctionRegistry | None=None): + ... + +def register_vector_function(func: Callable[..., Any], function_name: str, function_doc: dict, in_types, out_type: pyarrow.DataType, func_registry: FunctionRegistry | None=None): + ... + +def register_aggregate_function(func: Callable[..., Any], function_name: str, function_doc: dict, in_types, out_type: pyarrow.DataType, func_registry: FunctionRegistry | None=None): + ... + +def register_tabular_function(func: Callable[..., Any], function_name: str, function_doc: dict, in_types, out_type, func_registry: FunctionRegistry | None=None): + ... + +def _register_user_defined_function(register_func, func: Callable[..., Any], function_name: str, function_doc: dict, in_types, out_type: pyarrow.DataType, func_registry: FunctionRegistry | None=None) -> None: + ... + +def call_tabular_function(function_name: str, args: Iterable[Any] | None=None, func_registry: FunctionRegistry | None=None) -> pyarrow.RecordBatchReader: + ... class Kernel(_Weakrefable): - def __init__(self): ... + + def __init__(self): + ... class ScalarKernel(Kernel): - def __repr__(self): ... + + def __repr__(self) -> str: + ... class VectorKernel(Kernel): - def __repr__(self): ... + + def __repr__(self) -> str: + ... class ScalarAggregateKernel(Kernel): - def __repr__(self): ... + + def __repr__(self) -> str: + ... class HashAggregateKernel(Kernel): - def __repr__(self): ... + + def __repr__(self) -> str: + ... class Function(_Weakrefable): _kind_map: Incomplete - def __init__(self): ... - def __repr__(self): ... - def __reduce__(self): ... + def __init__(self): + ... + + def __repr__(self) -> str: + ... + + def __reduce__(self) -> tuple: + ... + @property - def name(self): ... + def name(self) -> str: + ... + @property - def arity(self): ... + def arity(self): + ... + @property - def kind(self): ... + def kind(self): + ... + @property - def _doc(self): ... + def _doc(self): + ... + @property - def num_kernels(self): ... - def call(self, args, options: FunctionOptions | None=None, memory_pool: MemoryPool | None=None, length=None): ... + def num_kernels(self): + ... + + def call(self, args: Iterable[Any], options: FunctionOptions | None=None, memory_pool: MemoryPool | None=None, length: int | None=None): + ... class ScalarFunction(Function): + @property - def kernels(self): ... + def kernels(self) -> list: + ... class VectorFunction(Function): + @property - def kernels(self): ... + def kernels(self) -> list: + ... class ScalarAggregateFunction(Function): + @property - def kernels(self): ... + def kernels(self) -> list: + ... class HashAggregateFunction(Function): + @property - def kernels(self): ... + def kernels(self) -> list: + ... class MetaFunction(Function): + @property - def kernels(self): ... + def kernels(self) -> list: + ... class FunctionRegistry(_Weakrefable): - def __init__(self): ... - def list_functions(self): ... - def get_function(self, name): ... + + def __init__(self): + ... + + def list_functions(self) -> list: + ... + + def get_function(self, name: str): + ... class FunctionOptions(_Weakrefable): __slots__ = () - def serialize(self): ... + def serialize(self) -> pyarrow.Buffer: + ... + @staticmethod - def deserialize(buf: Buffer): ... - def __repr__(self): ... - def __eq__(self, other: FunctionOptions): ... + def deserialize(buf: Buffer): + ... + + def __repr__(self) -> str: + ... + + def __eq__(self, other: FunctionOptions): + ... class _CastOptions(FunctionOptions): - def _set_options(self, target_type: DataType, allow_int_overflow, allow_time_truncate, allow_time_overflow, allow_decimal_truncate, allow_float_truncate, allow_invalid_utf8): ... - def _set_type(self, target_type=None): ... - def _set_safe(self): ... - def _set_unsafe(self): ... - def is_safe(self): ... + + def _set_options(self, target_type: DataType, allow_int_overflow, allow_time_truncate, allow_time_overflow, allow_decimal_truncate, allow_float_truncate, allow_invalid_utf8) -> None: + ... + + def _set_type(self, target_type=None) -> None: + ... + + def _set_safe(self) -> None: + ... + + def _set_unsafe(self) -> None: + ... + + def is_safe(self) -> bool: + ... + @property - def allow_int_overflow(self): ... + def allow_int_overflow(self) -> None: + ... + @allow_int_overflow.setter - def allow_int_overflow(self, flag: c_bool): ... + def allow_int_overflow(self, flag: bool) -> None: + ... + @property - def allow_time_truncate(self): ... + def allow_time_truncate(self) -> None: + ... + @allow_time_truncate.setter - def allow_time_truncate(self, flag: c_bool): ... + def allow_time_truncate(self, flag: bool) -> None: + ... + @property - def allow_time_overflow(self): ... + def allow_time_overflow(self) -> None: + ... + @allow_time_overflow.setter - def allow_time_overflow(self, flag: c_bool): ... + def allow_time_overflow(self, flag: bool) -> None: + ... + @property - def allow_decimal_truncate(self): ... + def allow_decimal_truncate(self) -> None: + ... + @allow_decimal_truncate.setter - def allow_decimal_truncate(self, flag: c_bool): ... + def allow_decimal_truncate(self, flag: bool) -> None: + ... + @property - def allow_float_truncate(self): ... + def allow_float_truncate(self) -> None: + ... + @allow_float_truncate.setter - def allow_float_truncate(self, flag: c_bool): ... + def allow_float_truncate(self, flag: bool) -> None: + ... + @property - def allow_invalid_utf8(self): ... + def allow_invalid_utf8(self) -> None: + ... + @allow_invalid_utf8.setter - def allow_invalid_utf8(self, flag: c_bool): ... + def allow_invalid_utf8(self, flag: bool) -> None: + ... class CastOptions(_CastOptions): - def __init__(self, target_type=None, *, allow_int_overflow=None, allow_time_truncate=None, allow_time_overflow=None, allow_decimal_truncate=None, allow_float_truncate=None, allow_invalid_utf8=None): ... + + def __init__(self, target_type=None, *, allow_int_overflow=None, allow_time_truncate=None, allow_time_overflow=None, allow_decimal_truncate=None, allow_float_truncate=None, allow_invalid_utf8=None): + ... + @staticmethod - def safe(target_type=None): ... + def safe(target_type=None) -> CastOptions: + ... + @staticmethod - def unsafe(target_type=None): ... + def unsafe(target_type=None) -> CastOptions: + ... class _ElementWiseAggregateOptions(FunctionOptions): - def _set_options(self, skip_nulls): ... + + def _set_options(self, skip_nulls) -> None: + ... class ElementWiseAggregateOptions(_ElementWiseAggregateOptions): - def __init__(self, *, skip_nulls=True): ... + + def __init__(self, *, skip_nulls: bool=True): + ... class _RoundOptions(FunctionOptions): - def _set_options(self, ndigits, round_mode): ... + + def _set_options(self, ndigits, round_mode) -> None: + ... class RoundOptions(_RoundOptions): - def __init__(self, ndigits=0, round_mode='half_to_even'): ... + + def __init__(self, ndigits: int=0, round_mode: str='half_to_even'): + ... class _RoundBinaryOptions(FunctionOptions): - def _set_options(self, round_mode): ... + + def _set_options(self, round_mode) -> None: + ... class RoundBinaryOptions(_RoundBinaryOptions): - def __init__(self, round_mode='half_to_even'): ... + + def __init__(self, round_mode: str='half_to_even'): + ... class _RoundTemporalOptions(FunctionOptions): - def _set_options(self, multiple, unit, week_starts_monday, ceil_is_strictly_greater, calendar_based_origin): ... + + def _set_options(self, multiple, unit, week_starts_monday, ceil_is_strictly_greater, calendar_based_origin) -> None: + ... class RoundTemporalOptions(_RoundTemporalOptions): - def __init__(self, multiple=1, unit='day', *, week_starts_monday=True, ceil_is_strictly_greater=False, calendar_based_origin=False): ... + + def __init__(self, multiple: int=1, unit: str='day', *, week_starts_monday: bool=True, ceil_is_strictly_greater: bool=False, calendar_based_origin: bool=False): + ... class _RoundToMultipleOptions(FunctionOptions): - def _set_options(self, multiple, round_mode): ... + + def _set_options(self, multiple, round_mode) -> None: + ... class RoundToMultipleOptions(_RoundToMultipleOptions): - def __init__(self, multiple=1.0, round_mode='half_to_even'): ... + + def __init__(self, multiple: float=1.0, round_mode: str='half_to_even'): + ... class _JoinOptions(FunctionOptions): _null_handling_map: Incomplete - def _set_options(self, null_handling, null_replacement): ... + def _set_options(self, null_handling, null_replacement) -> None: + ... class JoinOptions(_JoinOptions): - def __init__(self, null_handling='emit_null', null_replacement=''): ... + + def __init__(self, null_handling: str='emit_null', null_replacement: str=''): + ... class _MatchSubstringOptions(FunctionOptions): - def _set_options(self, pattern, ignore_case): ... + + def _set_options(self, pattern, ignore_case) -> None: + ... class MatchSubstringOptions(_MatchSubstringOptions): - def __init__(self, pattern, *, ignore_case=False): ... + + def __init__(self, pattern, *, ignore_case: bool=False): + ... class _PadOptions(FunctionOptions): - def _set_options(self, width, padding, lean_left_on_odd_padding): ... + + def _set_options(self, width, padding, lean_left_on_odd_padding) -> None: + ... class PadOptions(_PadOptions): - def __init__(self, width, padding=' ', lean_left_on_odd_padding=True): ... + + def __init__(self, width, padding: str=' ', lean_left_on_odd_padding: bool=True): + ... class _ZeroFillOptions(FunctionOptions): - def _set_options(self, width, padding): ... + + def _set_options(self, width, padding) -> None: + ... class ZeroFillOptions(_ZeroFillOptions): - def __init__(self, width, padding='0'): ... + + def __init__(self, width, padding: str='0'): + ... class _TrimOptions(FunctionOptions): - def _set_options(self, characters): ... + + def _set_options(self, characters) -> None: + ... class TrimOptions(_TrimOptions): - def __init__(self, characters): ... + + def __init__(self, characters): + ... class _ReplaceSubstringOptions(FunctionOptions): - def _set_options(self, pattern, replacement, max_replacements): ... + + def _set_options(self, pattern, replacement, max_replacements) -> None: + ... class ReplaceSubstringOptions(_ReplaceSubstringOptions): - def __init__(self, pattern, replacement, *, max_replacements=None): ... + + def __init__(self, pattern, replacement, *, max_replacements=None): + ... class _ExtractRegexOptions(FunctionOptions): - def _set_options(self, pattern): ... + + def _set_options(self, pattern) -> None: + ... class ExtractRegexOptions(_ExtractRegexOptions): - def __init__(self, pattern): ... + + def __init__(self, pattern): + ... class _ExtractRegexSpanOptions(FunctionOptions): - def _set_options(self, pattern): ... + + def _set_options(self, pattern) -> None: + ... class ExtractRegexSpanOptions(_ExtractRegexSpanOptions): - def __init__(self, pattern): ... + + def __init__(self, pattern): + ... class _SliceOptions(FunctionOptions): - def _set_options(self, start, stop, step): ... + + def _set_options(self, start, stop, step) -> None: + ... class SliceOptions(_SliceOptions): - def __init__(self, start, stop=None, step=1): ... + + def __init__(self, start, stop=None, step: int=1): + ... class _ListSliceOptions(FunctionOptions): - def _set_options(self, start, stop=None, step=1, return_fixed_size_list=None): ... + + def _set_options(self, start, stop=None, step: int=1, return_fixed_size_list=None): + ... class ListSliceOptions(_ListSliceOptions): - def __init__(self, start, stop=None, step=1, return_fixed_size_list=None): ... + + def __init__(self, start, stop=None, step: int=1, return_fixed_size_list=None): + ... class _ReplaceSliceOptions(FunctionOptions): - def _set_options(self, start, stop, replacement): ... + + def _set_options(self, start, stop, replacement) -> None: + ... class ReplaceSliceOptions(_ReplaceSliceOptions): - def __init__(self, start, stop, replacement): ... + + def __init__(self, start, stop, replacement): + ... class _FilterOptions(FunctionOptions): _null_selection_map: Incomplete - def _set_options(self, null_selection_behavior): ... + def _set_options(self, null_selection_behavior) -> None: + ... class FilterOptions(_FilterOptions): - def __init__(self, null_selection_behavior='drop'): ... + + def __init__(self, null_selection_behavior: str='drop'): + ... class _DictionaryEncodeOptions(FunctionOptions): _null_encoding_map: Incomplete - def _set_options(self, null_encoding): ... + def _set_options(self, null_encoding) -> None: + ... class DictionaryEncodeOptions(_DictionaryEncodeOptions): - def __init__(self, null_encoding='mask'): ... + + def __init__(self, null_encoding: str='mask'): + ... class _RunEndEncodeOptions(FunctionOptions): - def _set_options(self, run_end_type): ... + + def _set_options(self, run_end_type) -> None: + ... class RunEndEncodeOptions(_RunEndEncodeOptions): - def __init__(self, run_end_type=lib.int32()): ... + + def __init__(self, run_end_type=lib.int32()): + ... class _InversePermutationOptions(FunctionOptions): - def _set_options(self, max_index=-1, output_type=None): ... + + def _set_options(self, max_index=-1, output_type=None) -> None: + ... class InversePermutationOptions(_InversePermutationOptions): - def __init__(self, max_index=-1, output_type=None): ... + + def __init__(self, max_index=-1, output_type=None): + ... class _ScatterOptions(FunctionOptions): - def _set_options(self, max_index): ... + + def _set_options(self, max_index) -> None: + ... class ScatterOptions(_ScatterOptions): - def __init__(self, max_index=-1): ... + + def __init__(self, max_index=-1): + ... class _TakeOptions(FunctionOptions): - def _set_options(self, boundscheck): ... + + def _set_options(self, boundscheck) -> None: + ... class TakeOptions(_TakeOptions): - def __init__(self, *, boundscheck=True): ... + + def __init__(self, *, boundscheck: bool=True): + ... class _MakeStructOptions(FunctionOptions): - def _set_options(self, field_names, field_nullability, field_metadata): ... + + def _set_options(self, field_names, field_nullability, field_metadata) -> None: + ... class MakeStructOptions(_MakeStructOptions): - def __init__(self, field_names=(), *, field_nullability=None, field_metadata=None): ... + + def __init__(self, field_names=(), *, field_nullability=None, field_metadata=None): + ... class _StructFieldOptions(FunctionOptions): - def _set_options(self, indices): ... + + def _set_options(self, indices) -> None: + ... class StructFieldOptions(_StructFieldOptions): - def __init__(self, indices): ... + + def __init__(self, indices): + ... class _ScalarAggregateOptions(FunctionOptions): - def _set_options(self, skip_nulls, min_count): ... + + def _set_options(self, skip_nulls, min_count) -> None: + ... class ScalarAggregateOptions(_ScalarAggregateOptions): - def __init__(self, *, skip_nulls=True, min_count=1): ... + + def __init__(self, *, skip_nulls: bool=True, min_count: int=1): + ... class _CountOptions(FunctionOptions): _mode_map: Incomplete - def _set_options(self, mode): ... + def _set_options(self, mode) -> None: + ... class CountOptions(_CountOptions): - def __init__(self, mode='only_valid'): ... + + def __init__(self, mode: str='only_valid'): + ... class _IndexOptions(FunctionOptions): - def _set_options(self, scalar): ... + + def _set_options(self, scalar) -> None: + ... class IndexOptions(_IndexOptions): - def __init__(self, value): ... + + def __init__(self, value): + ... class _MapLookupOptions(FunctionOptions): _occurrence_map: Incomplete - def _set_options(self, query_key, occurrence): ... + def _set_options(self, query_key, occurrence) -> None: + ... class MapLookupOptions(_MapLookupOptions): - def __init__(self, query_key, occurrence): ... + + def __init__(self, query_key, occurrence): + ... class _ModeOptions(FunctionOptions): - def _set_options(self, n, skip_nulls, min_count): ... + + def _set_options(self, n, skip_nulls, min_count) -> None: + ... class ModeOptions(_ModeOptions): - def __init__(self, n=1, *, skip_nulls=True, min_count=0): ... + + def __init__(self, n: int=1, *, skip_nulls: bool=True, min_count: int=0): + ... class _SetLookupOptions(FunctionOptions): - def _set_options(self, value_set, skip_nulls: c_bool): ... + + def _set_options(self, value_set, skip_nulls: bool) -> None: + ... class SetLookupOptions(_SetLookupOptions): - def __init__(self, value_set, *, skip_nulls=False): ... + + def __init__(self, value_set, *, skip_nulls: bool=False): + ... class _StrptimeOptions(FunctionOptions): _unit_map: Incomplete - def _set_options(self, format, unit, error_is_null): ... + def _set_options(self, format, unit, error_is_null) -> None: + ... class StrptimeOptions(_StrptimeOptions): - def __init__(self, format, unit, error_is_null=False): ... + + def __init__(self, format, unit, error_is_null: bool=False): + ... class _StrftimeOptions(FunctionOptions): - def _set_options(self, format, locale): ... + + def _set_options(self, format, locale) -> None: + ... class StrftimeOptions(_StrftimeOptions): - def __init__(self, format='%Y-%m-%dT%H:%M:%S', locale='C'): ... + + def __init__(self, format: str='%Y-%m-%dT%H:%M:%S', locale: str='C'): + ... class _DayOfWeekOptions(FunctionOptions): - def _set_options(self, count_from_zero, week_start): ... + + def _set_options(self, count_from_zero, week_start) -> None: + ... class DayOfWeekOptions(_DayOfWeekOptions): - def __init__(self, *, count_from_zero=True, week_start=1): ... + + def __init__(self, *, count_from_zero: bool=True, week_start: int=1): + ... class _WeekOptions(FunctionOptions): - def _set_options(self, week_starts_monday, count_from_zero, first_week_is_fully_in_year): ... + + def _set_options(self, week_starts_monday, count_from_zero, first_week_is_fully_in_year) -> None: + ... class WeekOptions(_WeekOptions): - def __init__(self, *, week_starts_monday=True, count_from_zero=False, first_week_is_fully_in_year=False): ... + + def __init__(self, *, week_starts_monday: bool=True, count_from_zero: bool=False, first_week_is_fully_in_year: bool=False): + ... class _AssumeTimezoneOptions(FunctionOptions): _ambiguous_map: Incomplete _nonexistent_map: Incomplete - def _set_options(self, timezone, ambiguous, nonexistent): ... + def _set_options(self, timezone, ambiguous, nonexistent) -> None: + ... class AssumeTimezoneOptions(_AssumeTimezoneOptions): - def __init__(self, timezone, *, ambiguous='raise', nonexistent='raise'): ... + + def __init__(self, timezone, *, ambiguous: str='raise', nonexistent: str='raise'): + ... class _NullOptions(FunctionOptions): - def _set_options(self, nan_is_null): ... + + def _set_options(self, nan_is_null) -> None: + ... class NullOptions(_NullOptions): - def __init__(self, *, nan_is_null=False): ... + + def __init__(self, *, nan_is_null: bool=False): + ... class _VarianceOptions(FunctionOptions): - def _set_options(self, ddof, skip_nulls, min_count): ... + + def _set_options(self, ddof, skip_nulls, min_count) -> None: + ... class VarianceOptions(_VarianceOptions): - def __init__(self, *, ddof=0, skip_nulls=True, min_count=0): ... + + def __init__(self, *, ddof: int=0, skip_nulls: bool=True, min_count: int=0): + ... class _SkewOptions(FunctionOptions): - def _set_options(self, skip_nulls, biased, min_count): ... + + def _set_options(self, skip_nulls, biased, min_count) -> None: + ... class SkewOptions(_SkewOptions): - def __init__(self, *, skip_nulls=True, biased=True, min_count=0): ... + + def __init__(self, *, skip_nulls: bool=True, biased: bool=True, min_count: int=0): + ... class _SplitOptions(FunctionOptions): - def _set_options(self, max_splits, reverse): ... + + def _set_options(self, max_splits, reverse) -> None: + ... class SplitOptions(_SplitOptions): - def __init__(self, *, max_splits=None, reverse=False): ... + + def __init__(self, *, max_splits=None, reverse: bool=False): + ... class _SplitPatternOptions(FunctionOptions): - def _set_options(self, pattern, max_splits, reverse): ... + + def _set_options(self, pattern, max_splits, reverse) -> None: + ... class SplitPatternOptions(_SplitPatternOptions): - def __init__(self, pattern, *, max_splits=None, reverse=False): ... + + def __init__(self, pattern, *, max_splits=None, reverse: bool=False): + ... class _PartitionNthOptions(FunctionOptions): - def _set_options(self, pivot, null_placement): ... + + def _set_options(self, pivot, null_placement) -> None: + ... class PartitionNthOptions(_PartitionNthOptions): - def __init__(self, pivot, *, null_placement='at_end'): ... + + def __init__(self, pivot, *, null_placement: str='at_end'): + ... class _WinsorizeOptions(FunctionOptions): - def _set_options(self, lower_limit, upper_limit): ... + + def _set_options(self, lower_limit, upper_limit) -> None: + ... class WinsorizeOptions(_WinsorizeOptions): - def __init__(self, lower_limit, upper_limit): ... + + def __init__(self, lower_limit, upper_limit): + ... class _CumulativeOptions(FunctionOptions): - def _set_options(self, start, skip_nulls): ... + + def _set_options(self, start, skip_nulls) -> None: + ... class CumulativeOptions(_CumulativeOptions): - def __init__(self, start=None, *, skip_nulls=False): ... + + def __init__(self, start=None, *, skip_nulls: bool=False): + ... class CumulativeSumOptions(_CumulativeOptions): - def __init__(self, start=None, *, skip_nulls=False): ... + + def __init__(self, start=None, *, skip_nulls: bool=False): + ... class _PairwiseOptions(FunctionOptions): - def _set_options(self, period): ... + + def _set_options(self, period) -> None: + ... class PairwiseOptions(_PairwiseOptions): - def __init__(self, period=1): ... + + def __init__(self, period: int=1): + ... class _ListFlattenOptions(FunctionOptions): - def _set_options(self, recursive): ... + + def _set_options(self, recursive) -> None: + ... class ListFlattenOptions(_ListFlattenOptions): - def __init__(self, recursive=False): ... + + def __init__(self, recursive: bool=False): + ... class _ArraySortOptions(FunctionOptions): - def _set_options(self, order, null_placement): ... + + def _set_options(self, order, null_placement) -> None: + ... class ArraySortOptions(_ArraySortOptions): - def __init__(self, order='ascending', *, null_placement='at_end'): ... + + def __init__(self, order: str='ascending', *, null_placement: str='at_end'): + ... class _SearchSortedOptions(FunctionOptions): - def _set_options(self, side): ... + + def _set_options(self, side) -> None: + ... class SearchSortedOptions(_SearchSortedOptions): - def __init__(self, side='left'): ... + + def __init__(self, side: str='left'): + ... class _SortOptions(FunctionOptions): - def _set_options(self, sort_keys, null_placement): ... + + def _set_options(self, sort_keys, null_placement) -> None: + ... class SortOptions(_SortOptions): - def __init__(self, sort_keys=(), *, null_placement=None): ... + + def __init__(self, sort_keys=(), *, null_placement=None): + ... class _SelectKOptions(FunctionOptions): - def _set_options(self, k, sort_keys): ... + + def _set_options(self, k, sort_keys) -> None: + ... class SelectKOptions(_SelectKOptions): - def __init__(self, k, sort_keys): ... + + def __init__(self, k, sort_keys): + ... class _QuantileOptions(FunctionOptions): _interp_map: Incomplete - def _set_options(self, quantiles, interp, skip_nulls, min_count): ... + def _set_options(self, quantiles, interp, skip_nulls, min_count) -> None: + ... class QuantileOptions(_QuantileOptions): - def __init__(self, q=0.5, *, interpolation='linear', skip_nulls=True, min_count=0): ... + + def __init__(self, q: float=0.5, *, interpolation: str='linear', skip_nulls: bool=True, min_count: int=0): + ... class _TDigestOptions(FunctionOptions): - def _set_options(self, quantiles, delta, buffer_size, skip_nulls, min_count): ... + + def _set_options(self, quantiles, delta, buffer_size, skip_nulls, min_count) -> None: + ... class TDigestOptions(_TDigestOptions): - def __init__(self, q=0.5, *, delta=100, buffer_size=500, skip_nulls=True, min_count=0): ... + + def __init__(self, q: float=0.5, *, delta: int=100, buffer_size: int=500, skip_nulls: bool=True, min_count: int=0): + ... class _Utf8NormalizeOptions(FunctionOptions): _form_map: Incomplete - def _set_options(self, form): ... + def _set_options(self, form) -> None: + ... class Utf8NormalizeOptions(_Utf8NormalizeOptions): - def __init__(self, form): ... + + def __init__(self, form): + ... class _RandomOptions(FunctionOptions): - def _set_options(self, initializer): ... + + def _set_options(self, initializer) -> None: + ... class RandomOptions(_RandomOptions): - def __init__(self, *, initializer='system'): ... + + def __init__(self, *, initializer: str='system'): + ... class _RankOptions(FunctionOptions): _tiebreaker_map: Incomplete - def _set_options(self, sort_keys, null_placement, tiebreaker): ... + def _set_options(self, sort_keys, null_placement, tiebreaker) -> None: + ... class RankOptions(_RankOptions): - def __init__(self, sort_keys='ascending', *, null_placement=None, tiebreaker='first'): ... + + def __init__(self, sort_keys: str='ascending', *, null_placement=None, tiebreaker: str='first'): + ... class _RankQuantileOptions(FunctionOptions): - def _set_options(self, sort_keys, null_placement): ... + + def _set_options(self, sort_keys, null_placement) -> None: + ... class RankQuantileOptions(_RankQuantileOptions): - def __init__(self, sort_keys='ascending', *, null_placement=None): ... + + def __init__(self, sort_keys: str='ascending', *, null_placement=None): + ... class _PivotWiderOptions(FunctionOptions): - def _set_options(self, key_names, unexpected_key_behavior): ... + + def _set_options(self, key_names, unexpected_key_behavior) -> None: + ... class PivotWiderOptions(_PivotWiderOptions): - def __init__(self, key_names, *, unexpected_key_behavior='ignore'): ... + + def __init__(self, key_names, *, unexpected_key_behavior: str='ignore'): + ... class Expression(_Weakrefable): - def __init__(self): ... - def equals(self, other: Expression): ... - def __str__(self): ... - def __repr__(self): ... + + def __init__(self): + ... + + def equals(self, other: Expression) -> bool: + ... + + def __str__(self) -> str: + ... + + def __repr__(self) -> str: + ... + @staticmethod - def from_substrait(message: object): ... - def to_substrait(self, schema: Schema, allow_arrow_extensions: c_bool=False): ... + def from_substrait(message: object) -> Expression: + ... + + def to_substrait(self, schema: Schema, allow_arrow_extensions: bool=False) -> pyarrow.Buffer: + ... + @staticmethod - def _deserialize(buffer: Buffer): ... - def __reduce__(self): ... + def _deserialize(buffer: Buffer) -> Expression: + ... + + def __reduce__(self) -> tuple: + ... + @staticmethod - def _call(function_name: str, arguments: list[Any], options: FunctionOptions | None=None): ... - def __bool__(self): ... - def __invert__(self): ... - def __and__(self: Expression, other): ... - def __rand__(self: Expression, other): ... - def __or__(self: Expression, other): ... - def __ror__(self: Expression, other): ... - def __add__(self: Expression, other): ... - def __radd__(self: Expression, other): ... - def __mul__(self: Expression, other): ... - def __rmul__(self: Expression, other): ... - def __sub__(self: Expression, other): ... - def __rsub__(self: Expression, other): ... - def __truediv__(self: Expression, other): ... - def __rtruediv__(self: Expression, other): ... - def is_valid(self): ... - def is_null(self, nan_is_null: bool=False): ... - def is_nan(self): ... - def cast(self, type=None, safe=None, options=None): ... - def isin(self, values): ... + def _call(function_name: str, arguments: list[Any], options: FunctionOptions | None=None) -> Expression: + ... + + def __bool__(self): + ... + + def __invert__(self): + ... + + def __and__(self: Expression, other): + ... + + def __rand__(self: Expression, other): + ... + + def __or__(self: Expression, other): + ... + + def __ror__(self: Expression, other): + ... + + def __add__(self: Expression, other): + ... + + def __radd__(self: Expression, other): + ... + + def __mul__(self: Expression, other): + ... + + def __rmul__(self: Expression, other): + ... + + def __sub__(self: Expression, other): + ... + + def __rsub__(self: Expression, other): + ... + + def __truediv__(self: Expression, other): + ... + + def __rtruediv__(self: Expression, other): + ... + + def is_valid(self) -> Expression: + ... + + def is_null(self, nan_is_null: bool=False) -> Expression: + ... + + def is_nan(self) -> Expression: + ... + + def cast(self, type: pyarrow.DataType | None=None, safe: bool | None=None, options: CastOptions | None=None) -> Expression: + ... + + def isin(self, values: pyarrow.Array | Iterable[Any]) -> Expression: + ... + @staticmethod - def _field(name_or_idx): ... + def _field(name_or_idx) -> Expression: + ... + @staticmethod - def _nested_field(names: tuple[Any, ...]): ... + def _nested_field(names: tuple[Any, ...]) -> Expression: + ... + @staticmethod - def _scalar(value): ... + def _scalar(value) -> Expression: + ... class UdfContext(_Weakrefable): - def __init__(self): ... + + def __init__(self): + ... + @property - def batch_length(self): ... + def batch_length(self): + ... + @property - def memory_pool(self): ... - -class RegisterUdf(_Weakrefable): ... - -def _pas(): ... -def _forbid_instantiation(klass, subclasses_instead=True): ... -def function_registry(): ... -def get_function(name): ... -def list_functions(): ... -def call_function(name, args, options=None, memory_pool=None, length=None): ... -def _raise_invalid_function_option(value, description, *, exception_class=ValueError): ... -def _skip_nulls_doc(): ... -def _min_count_doc(*, default): ... -def _get_udf_context(memory_pool, batch_length): ... -def register_scalar_function(func, function_name, function_doc, in_types, out_type, func_registry=None): ... -def register_vector_function(func, function_name, function_doc, in_types, out_type, func_registry=None): ... -def register_aggregate_function(func, function_name, function_doc, in_types, out_type, func_registry=None): ... -def register_tabular_function(func, function_name, function_doc, in_types, out_type, func_registry=None): ... -def _register_user_defined_function(register_func, func, function_name, function_doc, in_types, out_type, func_registry=None): ... -def call_tabular_function(function_name, args=None, func_registry=None): ... + def memory_pool(self): + ... + +class RegisterUdf(_Weakrefable): + ... diff --git a/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi b/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi index a83cc5dc1e77..3038ba72b538 100644 --- a/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi +++ b/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi @@ -15,6 +15,9 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence from _typeshed import Incomplete - function_doc_additions: Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_csv.pyi b/python/pyarrow-stubs/pyarrow/_csv.pyi index 4ff752b897fd..a8bcb022b59f 100644 --- a/python/pyarrow-stubs/pyarrow/_csv.pyi +++ b/python/pyarrow-stubs/pyarrow/_csv.pyi @@ -14,20 +14,36 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_csv.pyx -from _typeshed import Incomplete +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +import pyarrow.csv +from pyarrow.lib import __name__ as __name__ +from _typeshed import Incomplete from collections import namedtuple +from pyarrow.lib import MemoryPool, RecordBatchReader, Schema, _CRecordBatchWriter, _Weakrefable + +class _InvalidRow(NamedTuple): + expected_columns: Incomplete + actual_columns: Incomplete + number: Incomplete + text: Incomplete +ISO8601: _ISO8601 -from pyarrow.includes.common import * -from pyarrow.includes.libarrow import * -from pyarrow.includes.libarrow_python import * -from pyarrow.lib import (MemoryPool, RecordBatchReader, Schema, - _CRecordBatchWriter, _Weakrefable) +def read_csv(input_file, read_options: pyarrow.csv.ReadOptions | None=None, parse_options: pyarrow.csv.ParseOptions | None=None, convert_options: pyarrow.csv.ConvertOptions | None=None, memory_pool: MemoryPool | None=None) -> pyarrow.Table: + ... -_InvalidRow = namedtuple('_InvalidRow', ('expected_columns', 'actual_columns', 'number', 'text'), module=__name__) -ISO8601 = _ISO8601() +def open_csv(input_file, read_options: pyarrow.csv.ReadOptions | None=None, parse_options: pyarrow.csv.ParseOptions | None=None, convert_options: pyarrow.csv.ConvertOptions | None=None, memory_pool: MemoryPool | None=None) -> CSVStreamingReader: + ... + +def _raise_invalid_function_option(value, description, *, exception_class=ValueError): + ... + +def write_csv(data: pyarrow.RecordBatch | pyarrow.Table, output_file, write_options: pyarrow.csv.WriteOptions | None=None, memory_pool: MemoryPool | None=None) -> None: + ... class InvalidRow(_InvalidRow): __slots__ = () @@ -36,202 +52,399 @@ class ReadOptions(_Weakrefable): __slots__ = () encoding: object - def __init__(self, *, use_threads=None, block_size=None, skip_rows=None, skip_rows_after_names=None, column_names=None, autogenerate_column_names=None, encoding='utf8'): ... + def __init__(self, *, use_threads=None, block_size=None, skip_rows=None, skip_rows_after_names=None, column_names=None, autogenerate_column_names=None, encoding: str='utf8'): + ... + @property - def use_threads(self): ... + def use_threads(self) -> None: + ... + @use_threads.setter - def use_threads(self, value): ... + def use_threads(self, value) -> None: + ... + @property - def block_size(self): ... + def block_size(self) -> None: + ... + @block_size.setter - def block_size(self, value): ... + def block_size(self, value) -> None: + ... + @property - def skip_rows(self): ... + def skip_rows(self) -> None: + ... + @skip_rows.setter - def skip_rows(self, value): ... + def skip_rows(self, value) -> None: + ... + @property - def skip_rows_after_names(self): ... + def skip_rows_after_names(self) -> None: + ... + @skip_rows_after_names.setter - def skip_rows_after_names(self, value): ... + def skip_rows_after_names(self, value) -> None: + ... + @property - def column_names(self): ... + def column_names(self) -> None: + ... + @column_names.setter - def column_names(self, value): ... + def column_names(self, value) -> None: + ... + @property - def autogenerate_column_names(self): ... + def autogenerate_column_names(self) -> None: + ... + @autogenerate_column_names.setter - def autogenerate_column_names(self, value): ... - def validate(self): ... - def equals(self, other: ReadOptions): ... - def __getstate__(self): ... - def __setstate__(self, state): ... - def __eq__(self, other): ... - def _repr_base(self): ... - def __repr__(self): ... - def __str__(self): ... + def autogenerate_column_names(self, value) -> None: + ... + + def validate(self) -> None: + ... + + def equals(self, other: ReadOptions) -> bool: + ... + + def __getstate__(self) -> tuple: + ... + + def __setstate__(self, state) -> None: + ... + + def __eq__(self, other): + ... + + def _repr_base(self) -> str: + ... + + def __repr__(self) -> str: + ... + + def __str__(self) -> str: + ... class ParseOptions(_Weakrefable): __slots__ = () - def __init__(self, *, delimiter=None, quote_char=None, double_quote=None, escape_char=None, newlines_in_values=None, ignore_empty_lines=None, invalid_row_handler=None): ... + def __init__(self, *, delimiter=None, quote_char=None, double_quote=None, escape_char=None, newlines_in_values=None, ignore_empty_lines=None, invalid_row_handler=None): + ... + @property - def delimiter(self): ... + def delimiter(self) -> None: + ... + @delimiter.setter - def delimiter(self, value): ... + def delimiter(self, value) -> None: + ... + @property - def quote_char(self): ... + def quote_char(self) -> None: + ... + @quote_char.setter - def quote_char(self, value): ... + def quote_char(self, value) -> None: + ... + @property - def double_quote(self): ... + def double_quote(self) -> None: + ... + @double_quote.setter - def double_quote(self, value): ... + def double_quote(self, value) -> None: + ... + @property - def escape_char(self): ... + def escape_char(self) -> None: + ... + @escape_char.setter - def escape_char(self, value): ... + def escape_char(self, value) -> None: + ... + @property - def newlines_in_values(self): ... + def newlines_in_values(self) -> None: + ... + @newlines_in_values.setter - def newlines_in_values(self, value): ... + def newlines_in_values(self, value) -> None: + ... + @property - def ignore_empty_lines(self): ... + def ignore_empty_lines(self) -> None: + ... + @property - def invalid_row_handler(self): ... + def invalid_row_handler(self) -> None: + ... + @invalid_row_handler.setter - def invalid_row_handler(self, value): ... + def invalid_row_handler(self, value) -> None: + ... + @ignore_empty_lines.setter - def ignore_empty_lines(self, value): ... - def validate(self): ... - def equals(self, other: ParseOptions): ... - def __getstate__(self): ... - def __setstate__(self, state): ... - def __eq__(self, other): ... - def _repr_base(self): ... - def __repr__(self): ... - def __str__(self): ... + def ignore_empty_lines(self, value) -> None: + ... + + def validate(self) -> None: + ... + + def equals(self, other: ParseOptions) -> bool: + ... + + def __getstate__(self) -> tuple: + ... + + def __setstate__(self, state) -> None: + ... + + def __eq__(self, other): + ... + + def _repr_base(self) -> str: + ... + + def __repr__(self) -> str: + ... + + def __str__(self) -> str: + ... class _ISO8601(_Weakrefable): __slots__ = () - def __str__(self): ... - def __eq__(self, other): ... + def __str__(self) -> str: + ... + + def __eq__(self, other) -> bool: + ... class ConvertOptions(_Weakrefable): __slots__ = () - def __init__(self, *, check_utf8=None, column_types=None, default_column_type=None, null_values=None, true_values=None, false_values=None, decimal_point=None, strings_can_be_null=None, quoted_strings_can_be_null=None, include_columns=None, include_missing_columns=None, auto_dict_encode=None, auto_dict_max_cardinality=None, timestamp_parsers=None): ... + def __init__(self, *, check_utf8=None, column_types=None, default_column_type=None, null_values=None, true_values=None, false_values=None, decimal_point=None, strings_can_be_null=None, quoted_strings_can_be_null=None, include_columns=None, include_missing_columns=None, auto_dict_encode=None, auto_dict_max_cardinality=None, timestamp_parsers=None): + ... + @property - def check_utf8(self): ... + def check_utf8(self) -> None: + ... + @check_utf8.setter - def check_utf8(self, value): ... + def check_utf8(self, value) -> None: + ... + @property - def strings_can_be_null(self): ... + def strings_can_be_null(self) -> None: + ... + @strings_can_be_null.setter - def strings_can_be_null(self, value): ... + def strings_can_be_null(self, value) -> None: + ... + @property - def quoted_strings_can_be_null(self): ... + def quoted_strings_can_be_null(self) -> None: + ... + @quoted_strings_can_be_null.setter - def quoted_strings_can_be_null(self, value): ... + def quoted_strings_can_be_null(self, value) -> None: + ... + @property - def column_types(self): ... + def column_types(self) -> None: + ... + @column_types.setter - def column_types(self, value): ... + def column_types(self, value) -> None: + ... + @property - def default_column_type(self): ... + def default_column_type(self) -> None: + ... + @default_column_type.setter - def default_column_type(self, value): ... + def default_column_type(self, value) -> None: + ... + @property - def null_values(self): ... + def null_values(self) -> None: + ... + @null_values.setter - def null_values(self, value): ... + def null_values(self, value) -> None: + ... + @property - def true_values(self): ... + def true_values(self) -> None: + ... + @true_values.setter - def true_values(self, value): ... + def true_values(self, value) -> None: + ... + @property - def false_values(self): ... + def false_values(self) -> None: + ... + @false_values.setter - def false_values(self, value): ... + def false_values(self, value) -> None: + ... + @property - def decimal_point(self): ... + def decimal_point(self) -> None: + ... + @decimal_point.setter - def decimal_point(self, value): ... + def decimal_point(self, value) -> None: + ... + @property - def auto_dict_encode(self): ... + def auto_dict_encode(self) -> None: + ... + @auto_dict_encode.setter - def auto_dict_encode(self, value): ... + def auto_dict_encode(self, value) -> None: + ... + @property - def auto_dict_max_cardinality(self): ... + def auto_dict_max_cardinality(self) -> None: + ... + @auto_dict_max_cardinality.setter - def auto_dict_max_cardinality(self, value): ... + def auto_dict_max_cardinality(self, value) -> None: + ... + @property - def include_columns(self): ... + def include_columns(self) -> None: + ... + @include_columns.setter - def include_columns(self, value): ... + def include_columns(self, value) -> None: + ... + @property - def include_missing_columns(self): ... + def include_missing_columns(self) -> None: + ... + @include_missing_columns.setter - def include_missing_columns(self, value): ... + def include_missing_columns(self, value) -> None: + ... + @property - def timestamp_parsers(self): ... + def timestamp_parsers(self) -> None: + ... + @timestamp_parsers.setter - def timestamp_parsers(self, value): ... - def validate(self): ... - def equals(self, other: ConvertOptions): ... - def __getstate__(self): ... - def __setstate__(self, state): ... - def __eq__(self, other): ... - def _repr_base(self): ... - def __repr__(self): ... - def __str__(self): ... + def timestamp_parsers(self, value) -> None: + ... + + def validate(self) -> None: + ... + + def equals(self, other: ConvertOptions) -> bool: + ... + + def __getstate__(self) -> tuple: + ... + + def __setstate__(self, state) -> None: + ... + + def __eq__(self, other): + ... + + def _repr_base(self) -> str: + ... + + def __repr__(self) -> str: + ... + + def __str__(self) -> str: + ... class CSVStreamingReader(RecordBatchReader): schema: Schema - def __init__(self): ... + def __init__(self): + ... class WriteOptions(_Weakrefable): __slots__ = () - def __init__(self, *, include_header=None, batch_size=None, delimiter=None, eol=None, null_string=None, quoting_style=None, quoting_header=None): ... + def __init__(self, *, include_header=None, batch_size=None, delimiter=None, eol=None, null_string=None, quoting_style=None, quoting_header=None): + ... + @property - def include_header(self): ... + def include_header(self) -> None: + ... + @include_header.setter - def include_header(self, value): ... + def include_header(self, value) -> None: + ... + @property - def batch_size(self): ... + def batch_size(self) -> None: + ... + @batch_size.setter - def batch_size(self, value): ... + def batch_size(self, value) -> None: + ... + @property - def delimiter(self): ... + def delimiter(self) -> None: + ... + @delimiter.setter - def delimiter(self, value): ... + def delimiter(self, value) -> None: + ... + @property - def eol(self): ... + def eol(self) -> None: + ... + @eol.setter - def eol(self, value): ... + def eol(self, value) -> None: + ... + @property - def null_string(self): ... + def null_string(self) -> None: + ... + @null_string.setter - def null_string(self, value): ... + def null_string(self, value) -> None: + ... + @property - def quoting_style(self): ... + def quoting_style(self) -> None: + ... + @quoting_style.setter - def quoting_style(self, value): ... + def quoting_style(self, value) -> None: + ... + @property - def quoting_header(self): ... + def quoting_header(self) -> None: + ... + @quoting_header.setter - def quoting_header(self, value): ... - def validate(self): ... - def _repr_base(self): ... - def __repr__(self): ... - def __str__(self): ... + def quoting_header(self, value) -> None: + ... + + def validate(self) -> None: + ... + + def _repr_base(self) -> str: + ... + + def __repr__(self) -> str: + ... + + def __str__(self) -> str: + ... class CSVWriter(_CRecordBatchWriter): - def __init__(self, sink, schema: Schema, *, write_options: WriteOptions | None=None, memory_pool: MemoryPool | None=None): ... -def read_csv(input_file, read_options=None, parse_options=None, convert_options=None, memory_pool: MemoryPool | None=None): ... -def open_csv(input_file, read_options=None, parse_options=None, convert_options=None, memory_pool: MemoryPool | None=None): ... -def _raise_invalid_function_option(value, description, *, exception_class=ValueError): ... -def write_csv(data, output_file, write_options=None, memory_pool: MemoryPool | None=None): ... + def __init__(self, sink, schema: Schema, *, write_options: WriteOptions | None=None, memory_pool: MemoryPool | None=None): + ... diff --git a/python/pyarrow-stubs/pyarrow/_dataset.pyi b/python/pyarrow-stubs/pyarrow/_dataset.pyi index 968d4161017c..73831c6c9c94 100644 --- a/python/pyarrow-stubs/pyarrow/_dataset.pyi +++ b/python/pyarrow-stubs/pyarrow/_dataset.pyi @@ -14,332 +14,651 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_dataset.pyx -from _typeshed import Incomplete +import os +import re +import pyarrow +from typing import Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +import pyarrow.compute +import pyarrow.dataset +import pyarrow.fs +from pyarrow.lib import _Weakrefable +from _typeshed import Incomplete import collections from typing import Any - from pyarrow._acero import ExecNodeOptions from pyarrow._compute import Expression -from pyarrow._csv import (ConvertOptions, ParseOptions, ReadOptions, - WriteOptions) +from pyarrow._csv import ConvertOptions, ParseOptions, ReadOptions, WriteOptions from pyarrow._fs import FileSystem from pyarrow._json import ParseOptions as JsonParseOptions from pyarrow._json import ReadOptions as JsonReadOptions -from pyarrow.includes.common import * -from pyarrow.includes.libarrow_dataset import * from pyarrow.lib import * +_DEFAULT_BATCH_SIZE: int +_DEFAULT_BATCH_READAHEAD: int +_DEFAULT_FRAGMENT_READAHEAD: int +_orc_fileformat: Incomplete +_orc_imported: bool +_dataset_pq: bool + +def _get_orc_fileformat() -> None: + ... + +def _get_parquet_classes() -> None: + ... + +def _get_parquet_symbol(name): + ... + +def _constructor_directory_partitioning_factory(*args): + ... + +def _constructor_hive_partitioning_factory(*args): + ... + +def _constructor_filename_partitioning_factory(*args): + ... -_DEFAULT_BATCH_SIZE = 2 ** 17 -_DEFAULT_BATCH_READAHEAD = 16 -_DEFAULT_FRAGMENT_READAHEAD = 4 -_orc_fileformat = None -_orc_imported = False -_dataset_pq = False +def get_partition_keys(partition_expression: Expression) -> dict: + ... + +def _filesystemdataset_write(data: Scanner, base_dir: object, basename_template: str, filesystem: FileSystem, partitioning: Partitioning, preserve_order: bool, file_options: FileWriteOptions, max_partitions: int, file_visitor: object, existing_data_behavior: str, max_open_files: int, max_rows_per_file: int, min_rows_per_group: int, max_rows_per_group: int, create_dir: bool) -> None: + ... class Dataset(_Weakrefable): _scan_options: dict[Any, Any] - def __init__(self): ... - @property - def partition_expression(self): ... - def replace_schema(self, schema: Schema): ... - def get_fragments(self, filter: Expression | None=None): ... - def _get_fragments(self, filter: Expression): ... - def _scanner_options(self, options): ... - def scanner(self, columns: object | None=None, filter: object | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... - def to_batches(self, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... - def to_table(self, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... - def take(self, indices: object, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... - def head(self, num_rows: int, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... - def count_rows(self, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... - @property - def schema(self): ... - def filter(self, expression): ... - def sort_by(self, sorting, **kwargs): ... - def join(self, right_dataset, keys, right_keys=None, join_type='left outer', left_suffix=None, right_suffix=None, coalesce_keys=True, use_threads=True): ... - def join_asof(self, right_dataset, on, by, tolerance, right_on=None, right_by=None): ... + def __init__(self): + ... + + @property + def partition_expression(self) -> pyarrow.compute.Expression: + ... + + def replace_schema(self, schema: Schema) -> Dataset: + ... + + def get_fragments(self, filter: Expression | None=None) -> Iterator[Fragment]: + ... + + def _get_fragments(self, filter: Expression) -> Iterator[Fragment]: + ... + + def _scanner_options(self, options): + ... + + def scanner(self, columns: object | None=None, filter: object | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> Scanner: + ... + + def to_batches(self, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> Iterator[pyarrow.RecordBatch]: + ... + + def to_table(self, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> pyarrow.Table: + ... + + def take(self, indices: object, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> pyarrow.Table: + ... + + def head(self, num_rows: int, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> pyarrow.Table: + ... + + def count_rows(self, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> int: + ... + + @property + def schema(self) -> pyarrow.Schema: + ... + + def filter(self, expression: pyarrow.compute.Expression) -> Dataset: + ... + + def sort_by(self, sorting, **kwargs) -> InMemoryDataset: + ... + + def join(self, right_dataset, keys: str | list[str], right_keys: str | list[str] | None=None, join_type: str='left outer', left_suffix: str | None=None, right_suffix: str | None=None, coalesce_keys: bool=True, use_threads: bool=True) -> InMemoryDataset: + ... + + def join_asof(self, right_dataset, on: str, by: str | list[str], tolerance: int, right_on: str | list[str] | None=None, right_by: str | list[str] | None=None) -> InMemoryDataset: + ... class InMemoryDataset(Dataset): - def __init__(self, source, schema: Schema | None=None): ... + + def __init__(self, source, schema: Schema | None=None): + ... class UnionDataset(Dataset): - def __init__(self, schema: Schema, children): ... - def __reduce__(self): ... + + def __init__(self, schema: Schema, children): + ... + + def __reduce__(self) -> tuple: + ... + @property - def children(self): ... + def children(self) -> list: + ... class FileSystemDataset(Dataset): - def __init__(self, fragments, schema: Schema, format: FileFormat, filesystem: FileSystem | None=None, root_partition=None): ... + + def __init__(self, fragments, schema: Schema, format: FileFormat, filesystem: FileSystem | None=None, root_partition=None): + ... + @property - def filesystem(self): ... + def filesystem(self) -> pyarrow.fs.FileSystem: + ... + @property - def partitioning(self): ... - def __reduce__(self): ... + def partitioning(self) -> None | Partitioning: + ... + + def __reduce__(self) -> tuple: + ... + @classmethod - def from_paths(cls, paths, schema=None, format=None, filesystem=None, partitions=None, root_partition=None): ... + def from_paths(cls, paths: list[str], schema: pyarrow.Schema | None=None, format: FileFormat | None=None, filesystem: pyarrow.fs.FileSystem | None=None, partitions: list[pyarrow.compute.Expression] | None=None, root_partition: pyarrow.compute.Expression | None=None) -> FileSystemDataset: + ... + @property - def files(self): ... + def files(self) -> list: + ... + @property - def format(self): ... + def format(self) -> FileFormat: + ... class FileWriteOptions(_Weakrefable): - def __init__(self): ... + + def __init__(self): + ... + @property - def format(self): ... + def format(self) -> FileFormat: + ... class FileFormat(_Weakrefable): - def __init__(self): ... - def inspect(self, file, filesystem=None): ... - def make_fragment(self, file, filesystem=None, partition_expression: Expression | None=None, *, file_size=None): ... - def make_write_options(self): ... + + def __init__(self): + ... + + def inspect(self, file: IO[Any] | os.PathLike[str] | str, filesystem=None) -> pyarrow.Schema: + ... + + def make_fragment(self, file: IO[Any] | os.PathLike[str] | str, filesystem=None, partition_expression: Expression | None=None, *, file_size: int | None=None) -> Fragment: + ... + + def make_write_options(self) -> FileWriteOptions: + ... + @property - def default_extname(self): ... + def default_extname(self) -> str: + ... + @property - def default_fragment_scan_options(self): ... + def default_fragment_scan_options(self) -> None: + ... + @default_fragment_scan_options.setter - def default_fragment_scan_options(self, options: FragmentScanOptions): ... - def __eq__(self, other): ... + def default_fragment_scan_options(self, options: FragmentScanOptions) -> None: + ... + + def __eq__(self, other): + ... class Fragment(_Weakrefable): - def __init__(self): ... + + def __init__(self): + ... + @property - def physical_schema(self): ... + def physical_schema(self) -> pyarrow.Schema: + ... + @property - def partition_expression(self): ... - def scanner(self, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... - def to_batches(self, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... - def to_table(self, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... - def take(self, indices: object, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... - def head(self, num_rows: int, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... - def count_rows(self, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + def partition_expression(self) -> pyarrow.compute.Expression: + ... + + def scanner(self, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> Scanner: + ... + + def to_batches(self, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> Iterator[pyarrow.RecordBatch]: + ... + + def to_table(self, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> pyarrow.Table: + ... + + def take(self, indices: object, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> pyarrow.Table: + ... + + def head(self, num_rows: int, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> pyarrow.Table: + ... + + def count_rows(self, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> int: + ... class FileFragment(Fragment): - def __repr__(self): ... - def __reduce__(self): ... - def open(self): ... + + def __repr__(self) -> str: + ... + + def __reduce__(self) -> tuple: + ... + + def open(self): + ... + @property - def path(self): ... + def path(self) -> str: + ... + @property - def filesystem(self): ... + def filesystem(self) -> None | pyarrow.fs.FileSystem: + ... + @property - def buffer(self): ... + def buffer(self) -> None | pyarrow.Buffer: + ... + @property - def format(self): ... + def format(self) -> FileFormat: + ... class FragmentScanOptions(_Weakrefable): - def __init__(self): ... + + def __init__(self): + ... + @property - def type_name(self): ... - def __eq__(self, other): ... + def type_name(self) -> str: + ... + + def __eq__(self, other): + ... class IpcFileWriteOptions(FileWriteOptions): - def __init__(self): ... + + def __init__(self): + ... + @property - def write_options(self): ... + def write_options(self) -> None: + ... + @write_options.setter - def write_options(self, write_options: IpcWriteOptions): ... + def write_options(self, write_options: IpcWriteOptions) -> None: + ... class IpcFileFormat(FileFormat): - def __init__(self): ... - def equals(self, other: IpcFileFormat): ... - def make_write_options(self, **kwargs): ... + + def __init__(self): + ... + + def equals(self, other: IpcFileFormat) -> bool: + ... + + def make_write_options(self, **kwargs) -> IpcFileWriteOptions: + ... + @property - def default_extname(self): ... - def __reduce__(self): ... + def default_extname(self) -> str: + ... + + def __reduce__(self) -> tuple: + ... class FeatherFileFormat(IpcFileFormat): + @property - def default_extname(self): ... + def default_extname(self) -> str: + ... class CsvFileFormat(FileFormat): __slots__ = () _read_options_py: ReadOptions - def __init__(self, parse_options: ParseOptions | None=None, default_fragment_scan_options=None, convert_options: ConvertOptions | None=None, read_options: ReadOptions | None=None): ... - def make_write_options(self, **kwargs): ... + def __init__(self, parse_options: ParseOptions | None=None, default_fragment_scan_options=None, convert_options: ConvertOptions | None=None, read_options: ReadOptions | None=None): + ... + + def make_write_options(self, **kwargs) -> CsvFileWriteOptions: + ... + @property - def parse_options(self): ... + def parse_options(self) -> None: + ... + @parse_options.setter - def parse_options(self, parse_options: ParseOptions): ... - def equals(self, other: CsvFileFormat): ... - def __reduce__(self): ... - def __repr__(self): ... + def parse_options(self, parse_options: ParseOptions) -> None: + ... + + def equals(self, other: CsvFileFormat) -> bool: + ... + + def __reduce__(self) -> tuple: + ... + + def __repr__(self) -> str: + ... class CsvFragmentScanOptions(FragmentScanOptions): __slots__ = () - def __init__(self, convert_options: ConvertOptions | None=None, read_options: ReadOptions | None=None): ... + def __init__(self, convert_options: ConvertOptions | None=None, read_options: ReadOptions | None=None): + ... + @property - def convert_options(self): ... + def convert_options(self) -> None: + ... + @convert_options.setter - def convert_options(self, convert_options: ConvertOptions): ... + def convert_options(self, convert_options: ConvertOptions) -> None: + ... + @property - def read_options(self): ... + def read_options(self) -> None: + ... + @read_options.setter - def read_options(self, read_options: ReadOptions): ... - def equals(self, other: CsvFragmentScanOptions): ... - def __reduce__(self): ... + def read_options(self, read_options: ReadOptions) -> None: + ... + + def equals(self, other: CsvFragmentScanOptions) -> bool: + ... + + def __reduce__(self) -> tuple: + ... class CsvFileWriteOptions(FileWriteOptions): - def __init__(self): ... + + def __init__(self): + ... + @property - def write_options(self): ... + def write_options(self) -> None: + ... + @write_options.setter - def write_options(self, write_options: WriteOptions): ... + def write_options(self, write_options: WriteOptions) -> None: + ... class JsonFileFormat(FileFormat): __slots__ = () - def __init__(self, default_fragment_scan_options=None, parse_options: JsonParseOptions | None=None, read_options: JsonReadOptions | None=None): ... - def equals(self, other: JsonFileFormat): ... - def __reduce__(self): ... - def __repr__(self): ... + def __init__(self, default_fragment_scan_options=None, parse_options: JsonParseOptions | None=None, read_options: JsonReadOptions | None=None): + ... + + def equals(self, other: JsonFileFormat) -> bool: + ... + + def __reduce__(self) -> tuple: + ... + + def __repr__(self) -> str: + ... class JsonFragmentScanOptions(FragmentScanOptions): __slots__ = () - def __init__(self, parse_options: JsonParseOptions | None=None, read_options: JsonReadOptions | None=None): ... + def __init__(self, parse_options: JsonParseOptions | None=None, read_options: JsonReadOptions | None=None): + ... + @property - def parse_options(self): ... + def parse_options(self) -> None: + ... + @parse_options.setter - def parse_options(self, parse_options: JsonParseOptions): ... + def parse_options(self, parse_options: JsonParseOptions) -> None: + ... + @property - def read_options(self): ... + def read_options(self) -> None: + ... + @read_options.setter - def read_options(self, read_options: JsonReadOptions): ... - def equals(self, other: JsonFragmentScanOptions): ... - def __reduce__(self): ... + def read_options(self, read_options: JsonReadOptions) -> None: + ... + + def equals(self, other: JsonFragmentScanOptions) -> bool: + ... + + def __reduce__(self) -> tuple: + ... class Partitioning(_Weakrefable): - def __init__(self): ... - def __eq__(self, other): ... - def parse(self, path): ... - def format(self, expr): ... + + def __init__(self): + ... + + def __eq__(self, other): + ... + + def parse(self, path: str) -> pyarrow.compute.Expression: + ... + + def format(self, expr: pyarrow.dataset.Expression) -> tuple: + ... + @property - def schema(self): ... + def schema(self) -> pyarrow.Schema: + ... class PartitioningFactory(_Weakrefable): - def __init__(self): ... - def __reduce__(self): ... + + def __init__(self): + ... + + def __reduce__(self) -> tuple: + ... + @property - def type_name(self): ... + def type_name(self) -> str: + ... class KeyValuePartitioning(Partitioning): - def __init__(self): ... - def __reduce__(self): ... + + def __init__(self): + ... + + def __reduce__(self) -> tuple: + ... + @property - def dictionaries(self): ... + def dictionaries(self) -> list: + ... class DirectoryPartitioning(KeyValuePartitioning): - def __init__(self, schema: Schema, dictionaries=None, segment_encoding='uri'): ... + + def __init__(self, schema: Schema, dictionaries=None, segment_encoding: str='uri'): + ... + @staticmethod - def discover(field_names=None, infer_dictionary=False, max_partition_dictionary_size=0, schema=None, segment_encoding='uri'): ... + def discover(field_names: list[str] | None=None, infer_dictionary: bool=False, max_partition_dictionary_size: int=0, schema: pyarrow.Schema | None=None, segment_encoding: str='uri') -> PartitioningFactory: + ... class HivePartitioning(KeyValuePartitioning): - def __init__(self, schema: Schema, dictionaries=None, null_fallback='__HIVE_DEFAULT_PARTITION__', segment_encoding='uri'): ... - def __reduce__(self): ... + + def __init__(self, schema: Schema, dictionaries=None, null_fallback: str='__HIVE_DEFAULT_PARTITION__', segment_encoding: str='uri'): + ... + + def __reduce__(self) -> tuple: + ... + @staticmethod - def discover(infer_dictionary=False, max_partition_dictionary_size=0, null_fallback='__HIVE_DEFAULT_PARTITION__', schema=None, segment_encoding='uri'): ... + def discover(infer_dictionary: bool=False, max_partition_dictionary_size: int=0, null_fallback: str='__HIVE_DEFAULT_PARTITION__', schema: pyarrow.Schema | None=None, segment_encoding: str='uri') -> PartitioningFactory: + ... class FilenamePartitioning(KeyValuePartitioning): - def __init__(self, schema: Schema, dictionaries=None, segment_encoding='uri'): ... + + def __init__(self, schema: Schema, dictionaries=None, segment_encoding: str='uri'): + ... + @staticmethod - def discover(field_names=None, infer_dictionary=False, schema=None, segment_encoding='uri'): ... + def discover(field_names: list[str] | None=None, infer_dictionary: bool=False, schema: pyarrow.Schema | None=None, segment_encoding: str='uri') -> PartitioningFactory: + ... class DatasetFactory(_Weakrefable): - def __init__(self): ... + + def __init__(self): + ... + @property - def root_partition(self): ... + def root_partition(self) -> None: + ... + @root_partition.setter - def root_partition(self, expr: Expression): ... - def inspect_schemas(self): ... - def inspect(self, *, promote_options='default', fragments=None): ... - def finish(self, schema: Schema | None=None): ... + def root_partition(self, expr: Expression) -> None: + ... + + def inspect_schemas(self) -> list: + ... + + def inspect(self, *, promote_options: str='default', fragments: int | None=None) -> pyarrow.Schema: + ... + + def finish(self, schema: Schema | None=None) -> Dataset: + ... class FileSystemFactoryOptions(_Weakrefable): __slots__ = () - def __init__(self, partition_base_dir=None, partitioning=None, exclude_invalid_files=None, selector_ignore_prefixes: list[Any] | None=None): ... + def __init__(self, partition_base_dir=None, partitioning=None, exclude_invalid_files=None, selector_ignore_prefixes: list[Any] | None=None): + ... + @property - def partitioning(self): ... + def partitioning(self) -> None: + ... + @partitioning.setter - def partitioning(self, value: Partitioning): ... + def partitioning(self, value: Partitioning) -> None: + ... + @property - def partitioning_factory(self): ... + def partitioning_factory(self) -> None: + ... + @partitioning_factory.setter - def partitioning_factory(self, value: PartitioningFactory): ... + def partitioning_factory(self, value: PartitioningFactory) -> None: + ... + @property - def partition_base_dir(self): ... + def partition_base_dir(self) -> None: + ... + @partition_base_dir.setter - def partition_base_dir(self, value): ... + def partition_base_dir(self, value) -> None: + ... + @property - def exclude_invalid_files(self): ... + def exclude_invalid_files(self) -> None: + ... + @exclude_invalid_files.setter - def exclude_invalid_files(self, value: bool): ... + def exclude_invalid_files(self, value: bool) -> None: + ... + @property - def selector_ignore_prefixes(self): ... + def selector_ignore_prefixes(self) -> None: + ... + @selector_ignore_prefixes.setter - def selector_ignore_prefixes(self, values): ... + def selector_ignore_prefixes(self, values) -> None: + ... class FileSystemDatasetFactory(DatasetFactory): - def __init__(self, filesystem: FileSystem, paths_or_selector, format: FileFormat, options: FileSystemFactoryOptions | None=None): ... + + def __init__(self, filesystem: FileSystem, paths_or_selector, format: FileFormat, options: FileSystemFactoryOptions | None=None): + ... class UnionDatasetFactory(DatasetFactory): - def __init__(self, factories: list[Any]): ... + + def __init__(self, factories: list[Any]): + ... class RecordBatchIterator(_Weakrefable): - def __init__(self): ... - def __iter__(self): ... - def __next__(self): ... -class TaggedRecordBatch(collections.namedtuple('TaggedRecordBatch', ['record_batch', 'fragment'])): ... + def __init__(self): + ... + + def __iter__(self) -> Self: + ... + + def __next__(self) -> pyarrow.RecordBatch: + ... + +class TaggedRecordBatch(NamedTuple): + record_batch: Incomplete + fragment: Incomplete + ... class TaggedRecordBatchIterator(_Weakrefable): - def __init__(self): ... - def __iter__(self): ... - def __next__(self): ... + + def __init__(self): + ... + + def __iter__(self) -> Self: + ... + + def __next__(self) -> TaggedRecordBatch: + ... class Scanner(_Weakrefable): - def __init__(self): ... + + def __init__(self): + ... + @staticmethod - def from_dataset(dataset: Dataset, *, columns: object | None=None, filter: object | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + def from_dataset(dataset: Dataset, *, columns: object | None=None, filter: object | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> Scanner: + ... + @staticmethod - def from_fragment(fragment: Fragment, *, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + def from_fragment(fragment: Fragment, *, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> Scanner: + ... + @staticmethod - def from_batches(source, *, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None): ... + def from_batches(source, *, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> Scanner: + ... + @property - def dataset_schema(self): ... + def dataset_schema(self) -> pyarrow.Schema: + ... + @property - def projected_schema(self): ... - def to_batches(self): ... - def scan_batches(self): ... - def to_table(self): ... - def take(self, indices: object): ... - def head(self, num_rows: int): ... - def count_rows(self): ... - def to_reader(self): ... + def projected_schema(self) -> pyarrow.Schema: + ... + + def to_batches(self) -> Iterator[pyarrow.RecordBatch]: + ... + + def scan_batches(self) -> TaggedRecordBatchIterator: + ... + + def to_table(self) -> pyarrow.Table: + ... + + def take(self, indices: object) -> pyarrow.Table: + ... + + def head(self, num_rows: int) -> pyarrow.Table: + ... + + def count_rows(self) -> int: + ... + + def to_reader(self) -> pyarrow.RecordBatchReader: + ... class WrittenFile(_Weakrefable): path: str metadata: object size: int - def __init__(self, path, metadata, size): ... + def __init__(self, path, metadata, size): + ... class _ScanNodeOptions(ExecNodeOptions): - def _set_options(self, dataset: Dataset, scan_options: dict[Any, Any]): ... + + def _set_options(self, dataset: Dataset, scan_options: dict[Any, Any]) -> None: + ... class ScanNodeOptions(_ScanNodeOptions): - def __init__(self, dataset: Dataset, **kwargs): ... - -def _get_orc_fileformat(): ... -def _get_parquet_classes(): ... -def _get_parquet_symbol(name): ... -def _constructor_directory_partitioning_factory(*args): ... -def _constructor_hive_partitioning_factory(*args): ... -def _constructor_filename_partitioning_factory(*args): ... -def get_partition_keys(partition_expression: Expression): ... -def _filesystemdataset_write(data: Scanner, base_dir: object, basename_template: str, filesystem: FileSystem, partitioning: Partitioning, preserve_order: bool, file_options: FileWriteOptions, max_partitions: int, file_visitor: object, existing_data_behavior: str, max_open_files: int, max_rows_per_file: int, min_rows_per_group: int, max_rows_per_group: int, create_dir: bool): ... + + def __init__(self, dataset: Dataset, **kwargs): + ... diff --git a/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi b/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi index ae314a813cf8..04b47805112a 100644 --- a/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi +++ b/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi @@ -14,19 +14,27 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_dataset_orc.pyx -from _typeshed import Incomplete +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete from pyarrow._dataset import FileFormat -from pyarrow.includes.libarrow import * -from pyarrow.includes.libarrow_dataset import * from pyarrow.lib import * - class OrcFileFormat(FileFormat): - def __init__(self): ... - def equals(self, other: OrcFileFormat): ... + + def __init__(self): + ... + + def equals(self, other: OrcFileFormat) -> bool: + ... + @property - def default_extname(self): ... - def __reduce__(self): ... + def default_extname(self) -> str: + ... + + def __reduce__(self) -> tuple: + ... diff --git a/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi b/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi index 74e65df5d312..bb433ddff5db 100644 --- a/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi +++ b/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi @@ -14,165 +14,303 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_dataset_parquet.pyx -from _typeshed import Incomplete +import os +import re +import pyarrow +from typing import Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +import pyarrow.dataset +import pyarrow.parquet +from pyarrow.lib import _Weakrefable +from _typeshed import Incomplete from typing import Any - from pyarrow._compute import Expression -from pyarrow._dataset import (CacheOptions, DatasetFactory, FileFormat, - FileFragment, FileWriteOptions, - FragmentScanOptions, Partitioning, - PartitioningFactory) +from pyarrow._dataset import CacheOptions, DatasetFactory, FileFormat, FileFragment, FileWriteOptions, FragmentScanOptions, Partitioning, PartitioningFactory from pyarrow._fs import FileSystem -from pyarrow.includes.libarrow import * -from pyarrow.includes.libarrow_dataset import * -from pyarrow.includes.libarrow_dataset_parquet import * from pyarrow.lib import * from typing_extensions import TypeAlias - +parquet_encryption_enabled: bool class ParquetFileFormat(FileFormat): - def __init__(self, read_options=None, default_fragment_scan_options=None, **kwargs): ... + + def __init__(self, read_options=None, default_fragment_scan_options=None, **kwargs): + ... + @property - def read_options(self): ... - def make_write_options(self, **kwargs): ... - def equals(self, other: ParquetFileFormat): ... + def read_options(self) -> ParquetReadOptions: + ... + + def make_write_options(self, **kwargs) -> pyarrow.dataset.FileWriteOptions: + ... + + def equals(self, other: ParquetFileFormat) -> bool: + ... + @property - def default_extname(self): ... - def __reduce__(self): ... - def __repr__(self): ... - def make_fragment(self, file, filesystem=None, partition_expression: Expression | None=None, row_groups=None, *, file_size=None): ... + def default_extname(self) -> str: + ... + + def __reduce__(self) -> tuple: + ... + + def __repr__(self) -> str: + ... + + def make_fragment(self, file: IO[Any] | os.PathLike[str] | str, filesystem=None, partition_expression: Expression | None=None, row_groups: Iterable[Any] | None=None, *, file_size: int | None=None) -> pyarrow.dataset.Fragment: + ... class RowGroupInfo: - def __init__(self, id, metadata, schema): ... + + def __init__(self, id, metadata, schema): + ... + @property - def num_rows(self): ... + def num_rows(self): + ... + @property - def total_byte_size(self): ... + def total_byte_size(self): + ... + @property - def statistics(self): ... - def __repr__(self): ... - def __eq__(self, other): ... + def statistics(self) -> dict: + ... + + def __repr__(self) -> str: + ... + + def __eq__(self, other) -> bool: + ... class ParquetFileFragment(FileFragment): - def __reduce__(self): ... - def ensure_complete_metadata(self): ... + + def __reduce__(self) -> tuple: + ... + + def ensure_complete_metadata(self) -> None: + ... + @property - def row_groups(self): ... + def row_groups(self) -> list: + ... + @property - def metadata(self): ... + def metadata(self) -> pyarrow.parquet.FileMetaData: + ... + @property - def num_row_groups(self): ... - def split_by_row_group(self, filter: Expression | None=None, schema: Schema | None=None): ... - def subset(self, filter: Expression | None=None, schema: Schema | None=None, row_group_ids: object | None=None): ... + def num_row_groups(self): + ... + + def split_by_row_group(self, filter: Expression | None=None, schema: Schema | None=None) -> list: + ... + + def subset(self, filter: Expression | None=None, schema: Schema | None=None, row_group_ids: object | None=None) -> pyarrow.dataset.Fragment: + ... class ParquetReadOptions(_Weakrefable): dictionary_columns: set[Any] - _coerce_int96_timestamp_unit: TimeUnit - _binary_type: Type - _list_type: Type + _coerce_int96_timestamp_unit: int + _binary_type: int + _list_type: int + + def __init__(self, dictionary_columns=None, coerce_int96_timestamp_unit=None, binary_type=None, list_type=None): + ... - def __init__(self, dictionary_columns=None, coerce_int96_timestamp_unit=None, binary_type=None, list_type=None): ... @property - def binary_type(self): ... + def binary_type(self) -> None: + ... + @binary_type.setter - def binary_type(self, ty): ... + def binary_type(self, ty) -> None: + ... + @property - def list_type(self): ... + def list_type(self) -> None: + ... + @list_type.setter - def list_type(self, ty): ... + def list_type(self, ty) -> None: + ... + @property - def coerce_int96_timestamp_unit(self): ... + def coerce_int96_timestamp_unit(self) -> None: + ... + @coerce_int96_timestamp_unit.setter - def coerce_int96_timestamp_unit(self, unit): ... - def equals(self, other: ParquetReadOptions): ... - def __eq__(self, other): ... - def __repr__(self): ... + def coerce_int96_timestamp_unit(self, unit) -> None: + ... + + def equals(self, other: ParquetReadOptions) -> bool: + ... + + def __eq__(self, other): + ... + + def __repr__(self) -> str: + ... class ParquetFileWriteOptions(FileWriteOptions): - def update(self, **kwargs): ... - def _set_properties(self): ... - def _set_arrow_properties(self): ... - def _set_encryption_config(self): ... - def __repr__(self): ... + + def update(self, **kwargs) -> None: + ... + + def _set_properties(self) -> None: + ... + + def _set_arrow_properties(self) -> None: + ... + + def _set_encryption_config(self) -> None: + ... + + def __repr__(self) -> str: + ... class ParquetFragmentScanOptions(FragmentScanOptions): __slots__ = () - def __init__(self, *, use_buffered_stream: bool=False, buffer_size=8192, pre_buffer: bool=True, cache_options=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, decryption_config=None, decryption_properties=None, page_checksum_verification: bool=False, arrow_extensions_enabled: bool=False): ... + def __init__(self, *, use_buffered_stream: bool=False, buffer_size: int=8192, pre_buffer: bool=True, cache_options=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, decryption_config=None, decryption_properties=None, page_checksum_verification: bool=False, arrow_extensions_enabled: bool=False): + ... + @property - def use_buffered_stream(self): ... + def use_buffered_stream(self) -> None: + ... + @use_buffered_stream.setter - def use_buffered_stream(self, use_buffered_stream: bool): ... + def use_buffered_stream(self, use_buffered_stream: bool) -> None: + ... + @property - def buffer_size(self): ... + def buffer_size(self) -> None: + ... + @buffer_size.setter - def buffer_size(self, buffer_size): ... + def buffer_size(self, buffer_size) -> None: + ... + @property - def pre_buffer(self): ... + def pre_buffer(self) -> None: + ... + @pre_buffer.setter - def pre_buffer(self, pre_buffer: bool): ... + def pre_buffer(self, pre_buffer: bool) -> None: + ... + @property - def cache_options(self): ... + def cache_options(self) -> None: + ... + @cache_options.setter - def cache_options(self, options: CacheOptions): ... + def cache_options(self, options: CacheOptions) -> None: + ... + @property - def thrift_string_size_limit(self): ... + def thrift_string_size_limit(self) -> None: + ... + @thrift_string_size_limit.setter - def thrift_string_size_limit(self, size): ... + def thrift_string_size_limit(self, size) -> None: + ... + @property - def thrift_container_size_limit(self): ... + def thrift_container_size_limit(self) -> None: + ... + @thrift_container_size_limit.setter - def thrift_container_size_limit(self, size): ... + def thrift_container_size_limit(self, size) -> None: + ... + @property - def schema_depth_limit(self): ... + def schema_depth_limit(self) -> None: + ... + @schema_depth_limit.setter - def schema_depth_limit(self, limit): ... + def schema_depth_limit(self, limit) -> None: + ... + @property - def decryption_properties(self): ... + def decryption_properties(self) -> None: + ... + @decryption_properties.setter - def decryption_properties(self, config): ... + def decryption_properties(self, config) -> None: + ... + @property - def parquet_decryption_config(self): ... + def parquet_decryption_config(self) -> None: + ... + @parquet_decryption_config.setter - def parquet_decryption_config(self, config): ... + def parquet_decryption_config(self, config) -> None: + ... + @property - def page_checksum_verification(self): ... + def page_checksum_verification(self) -> None: + ... + @page_checksum_verification.setter - def page_checksum_verification(self, page_checksum_verification: bool): ... + def page_checksum_verification(self, page_checksum_verification: bool) -> None: + ... + @property - def arrow_extensions_enabled(self): ... + def arrow_extensions_enabled(self) -> None: + ... + @arrow_extensions_enabled.setter - def arrow_extensions_enabled(self, arrow_extensions_enabled: bool): ... - def equals(self, other: ParquetFragmentScanOptions): ... + def arrow_extensions_enabled(self, arrow_extensions_enabled: bool) -> None: + ... + + def equals(self, other: ParquetFragmentScanOptions) -> bool: + ... + @staticmethod - def _reconstruct(kwargs): ... - def __reduce__(self): ... + def _reconstruct(kwargs) -> ParquetFragmentScanOptions: + ... + + def __reduce__(self) -> tuple: + ... class ParquetFactoryOptions(_Weakrefable): __slots__ = () - def __init__(self, partition_base_dir=None, partitioning=None, validate_column_chunk_paths=False): ... + def __init__(self, partition_base_dir=None, partitioning=None, validate_column_chunk_paths: bool=False): + ... + @property - def partitioning(self): ... + def partitioning(self) -> None: + ... + @partitioning.setter - def partitioning(self, value: Partitioning): ... + def partitioning(self, value: Partitioning) -> None: + ... + @property - def partitioning_factory(self): ... + def partitioning_factory(self) -> None: + ... + @partitioning_factory.setter - def partitioning_factory(self, value: PartitioningFactory): ... + def partitioning_factory(self, value: PartitioningFactory) -> None: + ... + @property - def partition_base_dir(self): ... + def partition_base_dir(self) -> None: + ... + @partition_base_dir.setter - def partition_base_dir(self, value): ... + def partition_base_dir(self, value) -> None: + ... + @property - def validate_column_chunk_paths(self): ... + def validate_column_chunk_paths(self) -> None: + ... + @validate_column_chunk_paths.setter - def validate_column_chunk_paths(self, value): ... + def validate_column_chunk_paths(self, value) -> None: + ... class ParquetDatasetFactory(DatasetFactory): - def __init__(self, metadata_path, filesystem: FileSystem, format: FileFormat, options: ParquetFactoryOptions | None=None): ... -parquet_encryption_enabled: bool + def __init__(self, metadata_path, filesystem: FileSystem, format: FileFormat, options: ParquetFactoryOptions | None=None): + ... diff --git a/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi b/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi index 5ce489a1914f..ad4c42b7492c 100644 --- a/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi +++ b/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi @@ -14,26 +14,35 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_dataset_parquet_encryption.pyx -from _typeshed import Incomplete -from pyarrow._dataset_parquet import (ParquetFileWriteOptions, - ParquetFragmentScanOptions) +import pyarrow.parquet +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from pyarrow.lib import _Weakrefable +from _typeshed import Incomplete +from pyarrow._dataset_parquet import ParquetFileWriteOptions, ParquetFragmentScanOptions from pyarrow._parquet_encryption import * -from pyarrow.includes.libarrow_dataset_parquet import * +def set_encryption_config(opts: ParquetFileWriteOptions, config: ParquetEncryptionConfig) -> None: + ... + +def set_decryption_properties(opts: ParquetFragmentScanOptions, config: pyarrow.parquet.FileDecryptionProperties) -> None: + ... + +def set_decryption_config(opts: ParquetFragmentScanOptions, config: ParquetDecryptionConfig) -> None: + ... class ParquetEncryptionConfig(_Weakrefable): __slots__ = () - def __init__(self, crypto_factory: CryptoFactory, kms_connection_config: KmsConnectionConfig, encryption_config: EncryptionConfiguration): ... + def __init__(self, crypto_factory: CryptoFactory, kms_connection_config: KmsConnectionConfig, encryption_config: EncryptionConfiguration): + ... class ParquetDecryptionConfig(_Weakrefable): __slots__ = () - def __init__(self, crypto_factory: CryptoFactory, kms_connection_config: KmsConnectionConfig, decryption_config: DecryptionConfiguration): ... - -def set_encryption_config(opts: ParquetFileWriteOptions, config: ParquetEncryptionConfig): ... -def set_decryption_properties(opts: ParquetFragmentScanOptions, config: FileDecryptionProperties): ... -def set_decryption_config(opts: ParquetFragmentScanOptions, config: ParquetDecryptionConfig): ... + def __init__(self, crypto_factory: CryptoFactory, kms_connection_config: KmsConnectionConfig, decryption_config: DecryptionConfiguration): + ... diff --git a/python/pyarrow-stubs/pyarrow/_feather.pyi b/python/pyarrow-stubs/pyarrow/_feather.pyi index 06ccd55777de..9fd838070f61 100644 --- a/python/pyarrow-stubs/pyarrow/_feather.pyi +++ b/python/pyarrow-stubs/pyarrow/_feather.pyi @@ -14,24 +14,35 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_feather.pyx -from _typeshed import Incomplete -from pyarrow.includes.common import * -from pyarrow.includes.libarrow import * -from pyarrow.includes.libarrow_feather import * +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete from pyarrow.lib import Table, _Weakrefable +def write_feather(table: Table, dest: object, compression=None, compression_level=None, chunksize=None, version: int=2) -> None: + ... -class FeatherError(Exception): ... +class FeatherError(Exception): + ... class FeatherReader(_Weakrefable): - def __init__(self, source, use_memory_map: c_bool, use_threads: c_bool): ... + + def __init__(self, source, use_memory_map: bool, use_threads: bool): + ... + @property - def version(self): ... - def read(self): ... - def read_indices(self, indices): ... - def read_names(self, names): ... + def version(self): + ... + + def read(self) -> pyarrow.Table: + ... + + def read_indices(self, indices) -> pyarrow.Table: + ... -def write_feather(table: Table, dest: object, compression=None, compression_level=None, chunksize=None, version=2): ... + def read_names(self, names) -> pyarrow.Table: + ... diff --git a/python/pyarrow-stubs/pyarrow/_flight.pyi b/python/pyarrow-stubs/pyarrow/_flight.pyi index 27bb47f89be8..225c8ed34711 100644 --- a/python/pyarrow-stubs/pyarrow/_flight.pyi +++ b/python/pyarrow-stubs/pyarrow/_flight.pyi @@ -14,95 +14,176 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_flight.pyx -from _typeshed import Incomplete +import os +import pyarrow +from typing import Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +import pyarrow.ipc +from pyarrow.lib import _CRecordBatchWriter, _Weakrefable +from _typeshed import Incomplete import collections import enum import re from typing import Any - -from pyarrow.includes.libarrow_flight import * -from pyarrow.ipc import _ReadPandasMixin +from pyarrow.ipc import _ReadPandasMixin as _ReadPandasMixin from pyarrow.lib import * -from pyarrow.lib import ArrowCancelled, ArrowException, ArrowInvalid +from pyarrow.lib import ArrowCancelled as ArrowCancelled, ArrowException as ArrowException, ArrowInvalid as ArrowInvalid +_FLIGHT_SERVER_ERROR_REGEX: re.Pattern[str] + +class _CertKeyPair(NamedTuple): + cert: Incomplete + key: Incomplete + +class _ActionType(NamedTuple): + type: Incomplete + description: Incomplete -_FLIGHT_SERVER_ERROR_REGEX = re.compile('Flight RPC failed with message: (.*). Detail: Python exception: (.*)', re.DOTALL) -_CertKeyPair = collections.namedtuple('_CertKeyPair', ['cert', 'key']) -_ActionType = collections.namedtuple('_ActionType', ['type', 'description']) -_CallInfo = collections.namedtuple('_CallInfo', ['method']) +class _CallInfo(NamedTuple): + method: Incomplete + +def _munge_grpc_python_error(message): + ... + +def connect(location: str | tuple | Location, **kwargs) -> FlightClient: + ... class FlightCallOptions(_Weakrefable): - def __init__(self, timeout=None, write_options=None, headers=None, read_options: IpcReadOptions | None=None): ... + + def __init__(self, timeout: float | None=None, write_options: pyarrow.ipc.IpcWriteOptions | None=None, headers=None, read_options: IpcReadOptions | None=None): + ... + @property - def timeout(self): ... + def timeout(self): + ... + @property - def headers(self): ... + def headers(self): + ... + @property - def read_options(self): ... + def read_options(self): + ... + @property - def write_options(self): ... - def __repr__(self): ... + def write_options(self): + ... + + def __repr__(self) -> str: + ... -class CertKeyPair(_CertKeyPair): ... +class CertKeyPair(_CertKeyPair): + ... class FlightError(Exception): - def __init__(self, message='', extra_info=b''): ... -class FlightInternalError(FlightError, ArrowException): ... + def __init__(self, message: str='', extra_info: bytes=b''): + ... + +class FlightInternalError(FlightError, ArrowException): + ... -class FlightTimedOutError(FlightError, ArrowException): ... +class FlightTimedOutError(FlightError, ArrowException): + ... -class FlightCancelledError(FlightError, ArrowCancelled): ... +class FlightCancelledError(FlightError, ArrowCancelled): + ... -class FlightServerError(FlightError, ArrowException): ... +class FlightServerError(FlightError, ArrowException): + ... -class FlightUnauthenticatedError(FlightError, ArrowException): ... +class FlightUnauthenticatedError(FlightError, ArrowException): + ... -class FlightUnauthorizedError(FlightError, ArrowException): ... +class FlightUnauthorizedError(FlightError, ArrowException): + ... -class FlightUnavailableError(FlightError, ArrowException): ... +class FlightUnavailableError(FlightError, ArrowException): + ... class FlightWriteSizeExceededError(ArrowInvalid): - def __init__(self, message, limit, actual): ... + + def __init__(self, message, limit, actual): + ... class Action(_Weakrefable): - def __init__(self, action_type, buf): ... + + def __init__(self, action_type: bytes | str, buf): + ... + @property - def type(self): ... + def type(self) -> str: + ... + @property - def body(self): ... - def serialize(self): ... + def body(self) -> pyarrow.Buffer: + ... + + def serialize(self): + ... + @classmethod - def deserialize(cls, serialized): ... - def __eq__(self, other: Action): ... - def __repr__(self): ... + def deserialize(cls, serialized) -> Action: + ... + + def __eq__(self, other: Action) -> bool: + ... + + def __repr__(self) -> str: + ... class ActionType(_ActionType): - def make_action(self, buf): ... + + def make_action(self, buf) -> Action: + ... class Result(_Weakrefable): - def __init__(self, buf): ... + + def __init__(self, buf): + ... + @property - def body(self): ... - def serialize(self): ... + def body(self) -> pyarrow.Buffer: + ... + + def serialize(self): + ... + @classmethod - def deserialize(cls, serialized): ... - def __eq__(self, other: Result): ... - def __repr__(self): ... + def deserialize(cls, serialized) -> Result: + ... + + def __eq__(self, other: Result) -> bool: + ... + + def __repr__(self) -> str: + ... class BasicAuth(_Weakrefable): - def __init__(self, username=None, password=None): ... + + def __init__(self, username: str | None=None, password: str | None=None): + ... + @property - def username(self): ... + def username(self): + ... + @property - def password(self): ... + def password(self): + ... + @staticmethod - def deserialize(serialized): ... - def serialize(self): ... - def __eq__(self, other: BasicAuth): ... - def __repr__(self): ... + def deserialize(serialized) -> BasicAuth: + ... + + def serialize(self): + ... + + def __eq__(self, other: BasicAuth) -> bool: + ... + + def __repr__(self) -> str: + ... class DescriptorType(enum.Enum): UNKNOWN = 0 @@ -122,260 +203,546 @@ class FlightMethod(enum.Enum): DO_EXCHANGE = 9 class FlightDescriptor(_Weakrefable): - def __init__(self): ... + + def __init__(self): + ... + @staticmethod - def for_path(*path): ... + def for_path(*path) -> FlightDescriptor: + ... + @staticmethod - def for_command(command): ... + def for_command(command) -> FlightDescriptor: + ... + @property - def descriptor_type(self): ... + def descriptor_type(self): + ... + @property - def command(self): ... + def command(self): + ... + @property - def path(self): ... - def __repr__(self): ... - def serialize(self): ... + def path(self): + ... + + def __repr__(self) -> str: + ... + + def serialize(self): + ... + @classmethod - def deserialize(cls, serialized): ... - def __eq__(self, other: FlightDescriptor): ... + def deserialize(cls, serialized) -> FlightDescriptor: + ... + + def __eq__(self, other: FlightDescriptor) -> bool: + ... class Ticket(_Weakrefable): - def __init__(self, ticket): ... + + def __init__(self, ticket): + ... + @property - def ticket(self): ... - def serialize(self): ... + def ticket(self): + ... + + def serialize(self): + ... + @classmethod - def deserialize(cls, serialized): ... - def __eq__(self, other: Ticket): ... - def __repr__(self): ... + def deserialize(cls, serialized) -> Ticket: + ... + + def __eq__(self, other: Ticket) -> bool: + ... + + def __repr__(self) -> str: + ... class Location(_Weakrefable): - def __init__(self, uri): ... - def __repr__(self): ... + + def __init__(self, uri): + ... + + def __repr__(self) -> str: + ... + @property - def uri(self): ... - def equals(self, other: Location): ... - def __eq__(self, other): ... + def uri(self): + ... + + def equals(self, other: Location) -> bool: + ... + + def __eq__(self, other): + ... + @staticmethod - def for_grpc_tcp(host, port): ... + def for_grpc_tcp(host, port) -> Location: + ... + @staticmethod - def for_grpc_tls(host, port): ... + def for_grpc_tls(host, port) -> Location: + ... + @staticmethod - def for_grpc_unix(path): ... + def for_grpc_unix(path) -> Location: + ... class FlightEndpoint(_Weakrefable): - def __init__(self, ticket, locations, expiration_time=None, app_metadata=''): ... + + def __init__(self, ticket: Ticket | bytes, locations, expiration_time: pyarrow.TimestampScalar | None=None, app_metadata: bytes | str=''): + ... + @property - def ticket(self): ... + def ticket(self) -> Ticket: + ... + @property - def locations(self): ... + def locations(self) -> list: + ... + @property - def expiration_time(self): ... + def expiration_time(self): + ... + @property - def app_metadata(self): ... - def serialize(self): ... + def app_metadata(self): + ... + + def serialize(self): + ... + @classmethod - def deserialize(cls, serialized): ... - def __repr__(self): ... - def __eq__(self, other: FlightEndpoint): ... + def deserialize(cls, serialized) -> FlightEndpoint: + ... + + def __repr__(self) -> str: + ... + + def __eq__(self, other: FlightEndpoint) -> bool: + ... class SchemaResult(_Weakrefable): - def __init__(self, schema: Schema): ... + + def __init__(self, schema: Schema): + ... + @property - def schema(self): ... - def serialize(self): ... + def schema(self) -> pyarrow.Schema: + ... + + def serialize(self): + ... + @classmethod - def deserialize(cls, serialized): ... - def __eq__(self, other: SchemaResult): ... - def __repr__(self): ... + def deserialize(cls, serialized) -> SchemaResult: + ... + + def __eq__(self, other: SchemaResult) -> bool: + ... + + def __repr__(self) -> str: + ... class FlightInfo(_Weakrefable): - def __init__(self, schema: Schema, descriptor: FlightDescriptor, endpoints, total_records=None, total_bytes=None, ordered=False, app_metadata=''): ... + + def __init__(self, schema: Schema, descriptor: FlightDescriptor, endpoints: list[FlightEndpoint], total_records: int | None=None, total_bytes: int | None=None, ordered: bool=False, app_metadata: bytes | str=''): + ... + @property - def total_records(self): ... + def total_records(self): + ... + @property - def total_bytes(self): ... + def total_bytes(self): + ... + @property - def ordered(self): ... + def ordered(self): + ... + @property - def app_metadata(self): ... + def app_metadata(self): + ... + @property - def schema(self): ... + def schema(self) -> None | pyarrow.Schema: + ... + @property - def descriptor(self): ... + def descriptor(self) -> FlightDescriptor: + ... + @property - def endpoints(self): ... - def serialize(self): ... + def endpoints(self) -> list: + ... + + def serialize(self): + ... + @classmethod - def deserialize(cls, serialized): ... - def __eq__(self, other: FlightInfo): ... - def __repr__(self): ... + def deserialize(cls, serialized) -> FlightInfo: + ... + + def __eq__(self, other: FlightInfo) -> bool: + ... + + def __repr__(self) -> str: + ... class FlightStreamChunk(_Weakrefable): + @property - def data(self): ... + def data(self) -> None | pyarrow.RecordBatch: + ... + @property - def app_metadata(self): ... - def __iter__(self): ... - def __repr__(self): ... + def app_metadata(self) -> None | pyarrow.Buffer: + ... + + def __iter__(self): + ... + + def __repr__(self) -> str: + ... class _MetadataRecordBatchReader(_Weakrefable, _ReadPandasMixin): - def __iter__(self): ... - def __next__(self): ... + + def __iter__(self) -> Self: + ... + + def __next__(self): + ... + @property - def schema(self): ... - def read_all(self): ... - def read_chunk(self): ... - def to_reader(self): ... + def schema(self) -> pyarrow.Schema: + ... + + def read_all(self) -> pyarrow.Table: + ... + + def read_chunk(self) -> FlightStreamChunk: + ... + + def to_reader(self) -> pyarrow.RecordBatchReader: + ... + @property - def stats(self): ... + def stats(self) -> pyarrow.ipc.ReadStats: + ... -class MetadataRecordBatchReader(_MetadataRecordBatchReader): ... +class MetadataRecordBatchReader(_MetadataRecordBatchReader): + ... class FlightStreamReader(MetadataRecordBatchReader): - def cancel(self): ... - def read_all(self): ... + + def cancel(self) -> None: + ... + + def read_all(self) -> pyarrow.Table: + ... class MetadataRecordBatchWriter(_CRecordBatchWriter): - def begin(self, schema: Schema, options=None): ... - def write_metadata(self, buf): ... - def write_batch(self, batch: RecordBatch): ... - def write_table(self, table: Table, max_chunksize=None, **kwargs): ... - def close(self): ... - def write_with_metadata(self, batch: RecordBatch, buf): ... + + def begin(self, schema: Schema, options=None) -> None: + ... + + def write_metadata(self, buf) -> None: + ... + + def write_batch(self, batch: RecordBatch) -> None: + ... + + def write_table(self, table: Table, max_chunksize: int | None=None, **kwargs) -> None: + ... + + def close(self) -> None: + ... + + def write_with_metadata(self, batch: RecordBatch, buf: pyarrow.Buffer) -> None: + ... class FlightStreamWriter(MetadataRecordBatchWriter): - def done_writing(self): ... + + def done_writing(self) -> None: + ... class FlightMetadataReader(_Weakrefable): - def read(self): ... + + def read(self) -> None | pyarrow.Buffer: + ... class FlightMetadataWriter(_Weakrefable): - def write(self, message): ... + + def write(self, message: pyarrow.Buffer) -> None: + ... class AsyncioCall: - def __init__(self) -> None: ... - def as_awaitable(self) -> object: ... - def wakeup(self, result_or_exception) -> None: ... + + def __init__(self) -> None: + ... + + def as_awaitable(self) -> object: + ... + + def wakeup(self, result_or_exception) -> None: + ... class AsyncioFlightClient: - def __init__(self, client: FlightClient) -> None: ... - async def get_flight_info(self, descriptor: FlightDescriptor, *, options: FlightCallOptions | None=None): ... + + def __init__(self, client: FlightClient) -> None: + ... + + async def get_flight_info(self, descriptor: FlightDescriptor, *, options: FlightCallOptions | None=None): + ... class FlightClient(_Weakrefable): - def __init__(self, location, *, tls_root_certs=None, cert_chain=None, private_key=None, override_hostname=None, middleware=None, write_size_limit_bytes=None, disable_server_verification=None, generic_options=None): ... + + def __init__(self, location, *, tls_root_certs=None, cert_chain=None, private_key=None, override_hostname=None, middleware=None, write_size_limit_bytes=None, disable_server_verification=None, generic_options=None): + ... + @property - def supports_async(self): ... - def as_async(self) -> None: ... - def wait_for_available(self, timeout=5): ... + def supports_async(self): + ... + + def as_async(self) -> None: + ... + + def wait_for_available(self, timeout: int=5) -> None: + ... + @classmethod - def connect(cls, location, tls_root_certs=None, cert_chain=None, private_key=None, override_hostname=None, disable_server_verification=None): ... - def authenticate(self, auth_handler, options: FlightCallOptions | None=None): ... - def authenticate_basic_token(self, username, password, options: FlightCallOptions | None=None): ... - def list_actions(self, options: FlightCallOptions | None=None): ... - def do_action(self, action, options: FlightCallOptions | None=None): ... - def list_flights(self, criteria: bytes | None=None, options: FlightCallOptions | None=None): ... - def get_flight_info(self, descriptor: FlightDescriptor, options: FlightCallOptions | None=None): ... - def get_schema(self, descriptor: FlightDescriptor, options: FlightCallOptions | None=None): ... - def do_get(self, ticket: Ticket, options: FlightCallOptions | None=None): ... - def do_put(self, descriptor: FlightDescriptor, schema: Schema, options: FlightCallOptions | None=None): ... - def do_exchange(self, descriptor: FlightDescriptor, options: FlightCallOptions | None=None): ... - def close(self): ... - def __del__(self): ... - def __enter__(self): ... - def __exit__(self, exc_type, exc_value, traceback): ... - -class FlightDataStream(_Weakrefable): ... + def connect(cls, location, tls_root_certs=None, cert_chain=None, private_key=None, override_hostname=None, disable_server_verification=None) -> FlightClient: + ... + + def authenticate(self, auth_handler: ClientAuthHandler, options: FlightCallOptions | None=None) -> None: + ... + + def authenticate_basic_token(self, username: str, password: str, options: FlightCallOptions | None=None): + ... + + def list_actions(self, options: FlightCallOptions | None=None) -> list: + ... + + def do_action(self, action: str | tuple | Action, options: FlightCallOptions | None=None): + ... + + def list_flights(self, criteria: bytes | None=None, options: FlightCallOptions | None=None) -> Iterator[FlightInfo]: + ... + + def get_flight_info(self, descriptor: FlightDescriptor, options: FlightCallOptions | None=None) -> FlightInfo: + ... + + def get_schema(self, descriptor: FlightDescriptor, options: FlightCallOptions | None=None) -> SchemaResult: + ... + + def do_get(self, ticket: Ticket, options: FlightCallOptions | None=None) -> FlightStreamReader: + ... + + def do_put(self, descriptor: FlightDescriptor, schema: Schema, options: FlightCallOptions | None=None) -> tuple: + ... + + def do_exchange(self, descriptor: FlightDescriptor, options: FlightCallOptions | None=None) -> tuple: + ... + + def close(self) -> None: + ... + + def __del__(self) -> None: + ... + + def __enter__(self) -> Self: + ... + + def __exit__(self, exc_type, exc_value, traceback) -> None: + ... + +class FlightDataStream(_Weakrefable): + ... class RecordBatchStream(FlightDataStream): - def __init__(self, data_source, options=None): ... + + def __init__(self, data_source: pyarrow.RecordBatchReader | pyarrow.Table, options: pyarrow.ipc.IpcWriteOptions | None=None): + ... class GeneratorStream(FlightDataStream): - def __init__(self, schema, generator, options=None): ... + + def __init__(self, schema: pyarrow.Schema, generator: Iterator[Any] | Iterable[Any], options: pyarrow.ipc.IpcWriteOptions | None=None): + ... class ServerCallContext(_Weakrefable): - def peer_identity(self): ... - def peer(self): ... - def is_cancelled(self): ... - def add_header(self, key, value): ... - def add_trailer(self, key, value): ... - def get_middleware(self, key): ... + + def peer_identity(self) -> bytes: + ... + + def peer(self) -> str: + ... + + def is_cancelled(self): + ... + + def add_header(self, key, value) -> None: + ... + + def add_trailer(self, key, value) -> None: + ... + + def get_middleware(self, key): + ... class ServerAuthReader(_Weakrefable): - def read(self): ... + + def read(self) -> bytes: + ... class ServerAuthSender(_Weakrefable): - def write(self, message): ... + + def write(self, message) -> None: + ... class ClientAuthReader(_Weakrefable): - def read(self): ... + + def read(self) -> bytes: + ... class ClientAuthSender(_Weakrefable): - def write(self, message): ... + + def write(self, message) -> None: + ... class ServerAuthHandler(_Weakrefable): - def authenticate(self, outgoing, incoming): ... - def is_valid(self, token): ... + + def authenticate(self, outgoing: ServerAuthSender, incoming: ServerAuthReader): + ... + + def is_valid(self, token: bytes): + ... class ClientAuthHandler(_Weakrefable): - def authenticate(self, outgoing, incoming): ... - def get_token(self): ... -class CallInfo(_CallInfo): ... + def authenticate(self, outgoing: ClientAuthSender, incoming: ClientAuthReader): + ... + + def get_token(self): + ... + +class CallInfo(_CallInfo): + ... class ClientMiddlewareFactory(_Weakrefable): - def start_call(self, info): ... + + def start_call(self, info: CallInfo) -> None: + ... class ClientMiddleware(_Weakrefable): - def sending_headers(self): ... - def received_headers(self, headers): ... - def call_completed(self, exception): ... + + def sending_headers(self) -> None: + ... + + def received_headers(self, headers: dict) -> None: + ... + + def call_completed(self, exception: ArrowException) -> None: + ... class ServerMiddlewareFactory(_Weakrefable): - def start_call(self, info, headers): ... -class TracingServerMiddlewareFactory(ServerMiddlewareFactory): ... + def start_call(self, info: CallInfo, headers: dict) -> None: + ... + +class TracingServerMiddlewareFactory(ServerMiddlewareFactory): + ... class ServerMiddleware(_Weakrefable): - def sending_headers(self): ... - def call_completed(self, exception): ... + + def sending_headers(self) -> None: + ... + + def call_completed(self, exception: ArrowException) -> None: + ... class TracingServerMiddleware(ServerMiddleware): trace_context: Incomplete __slots__ = ['trace_context'] - def __init__(self, trace_context): ... + def __init__(self, trace_context): + ... class _ServerMiddlewareFactoryWrapper(ServerMiddlewareFactory): - def __init__(self, factories: dict[Any, Any]): ... - def start_call(self, info, headers): ... + + def __init__(self, factories: dict[Any, Any]): + ... + + def start_call(self, info, headers) -> _ServerMiddlewareWrapper | None: + ... class _ServerMiddlewareWrapper(ServerMiddleware): - def __init__(self, middleware: dict[Any, Any]): ... - def sending_headers(self): ... - def call_completed(self, exception): ... + + def __init__(self, middleware: dict[Any, Any]): + ... + + def sending_headers(self): + ... + + def call_completed(self, exception) -> None: + ... class _FlightServerFinalizer(_Weakrefable): - def finalize(self): ... + + def finalize(self) -> None: + ... class FlightServerBase(_Weakrefable): - def __init__(self, location=None, auth_handler=None, tls_certificates=None, verify_client=None, root_certificates=None, middleware=None): ... - @property - def port(self): ... - def list_flights(self, context, criteria): ... - def get_flight_info(self, context, descriptor): ... - def get_schema(self, context, descriptor): ... - def do_put(self, context, descriptor, reader: MetadataRecordBatchReader, writer: FlightMetadataWriter): ... - def do_get(self, context, ticket): ... - def do_exchange(self, context, descriptor, reader, writer): ... - def list_actions(self, context): ... - def do_action(self, context, action): ... - def serve(self): ... - def run(self): ... - def shutdown(self): ... - def wait(self): ... - def __enter__(self): ... - def __exit__(self, exc_type, exc_value, traceback): ... - -def _munge_grpc_python_error(message): ... -def connect(location, **kwargs): ... + + def __init__(self, location=None, auth_handler=None, tls_certificates=None, verify_client=None, root_certificates=None, middleware=None): + ... + + @property + def port(self): + ... + + def list_flights(self, context: ServerCallContext, criteria: bytes) -> Iterator[FlightInfo]: + ... + + def get_flight_info(self, context: ServerCallContext, descriptor: FlightDescriptor) -> FlightInfo: + ... + + def get_schema(self, context: ServerCallContext, descriptor: FlightDescriptor) -> pyarrow.Schema: + ... + + def do_put(self, context: ServerCallContext, descriptor: FlightDescriptor, reader: MetadataRecordBatchReader, writer: FlightMetadataWriter): + ... + + def do_get(self, context: ServerCallContext, ticket: Ticket) -> FlightDataStream: + ... + + def do_exchange(self, context: ServerCallContext, descriptor: FlightDescriptor, reader: MetadataRecordBatchReader, writer: MetadataRecordBatchWriter): + ... + + def list_actions(self, context: ServerCallContext) -> Iterator[ActionType] | tuple: + ... + + def do_action(self, context: ServerCallContext, action: Action) -> Iterator[bytes]: + ... + + def serve(self) -> None: + ... + + def run(self) -> None: + ... + + def shutdown(self) -> None: + ... + + def wait(self) -> None: + ... + + def __enter__(self) -> Self: + ... + + def __exit__(self, exc_type, exc_value, traceback) -> None: + ... diff --git a/python/pyarrow-stubs/pyarrow/_fs.pyi b/python/pyarrow-stubs/pyarrow/_fs.pyi index 69337de47447..eb9b2f68b99c 100644 --- a/python/pyarrow-stubs/pyarrow/_fs.pyi +++ b/python/pyarrow-stubs/pyarrow/_fs.pyi @@ -14,18 +14,27 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_fs.pyx -from _typeshed import Incomplete +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from pyarrow.lib import _Weakrefable +from _typeshed import Incomplete from abc import ABC, abstractmethod from datetime import datetime from enum import IntEnum - -from pyarrow.includes.common import * -from pyarrow.includes.libarrow_fs import * from pyarrow.lib import * +def _file_type_to_string(ty) -> str: + ... + +def _copy_files(source_fs: FileSystem, source_path: str, destination_fs: FileSystem, destination_path: str, chunk_size: int, use_threads: bool) -> None: + ... + +def _copy_files_selector(source_fs: FileSystem, source_sel: FileSelector, destination_fs: FileSystem, destination_base_dir: str, chunk_size: int, use_threads: bool) -> None: + ... class FileType(IntEnum): NotFound = ... @@ -34,124 +43,252 @@ class FileType(IntEnum): Directory = ... class FileInfo(_Weakrefable): - def __init__(self, path, type: FileType=FileType.Unknown, *, mtime=None, mtime_ns=None, size=None): ... - def __repr__(self): ... + + def __init__(self, path, type: FileType=FileType.Unknown, *, mtime=None, mtime_ns=None, size=None): + ... + + def __repr__(self) -> str: + ... + @property - def type(self): ... + def type(self) -> FileType: + ... + @property - def is_file(self): ... + def is_file(self) -> bool: + ... + @property - def path(self): ... + def path(self) -> str: + ... + @property - def base_name(self): ... + def base_name(self) -> str: + ... + @property - def size(self): ... + def size(self) -> int | None: + ... + @property - def extension(self): ... + def extension(self) -> str: + ... + @property - def mtime(self): ... + def mtime(self): + ... + @property - def mtime_ns(self): ... + def mtime_ns(self) -> int | None: + ... class FileSelector(_Weakrefable): - def __init__(self, base_dir, allow_not_found: bool=False, recursive: bool=False): ... + + def __init__(self, base_dir, allow_not_found: bool=False, recursive: bool=False): + ... + @property - def base_dir(self): ... + def base_dir(self) -> None: + ... + @base_dir.setter - def base_dir(self, base_dir): ... + def base_dir(self, base_dir) -> None: + ... + @property - def allow_not_found(self): ... + def allow_not_found(self) -> None: + ... + @allow_not_found.setter - def allow_not_found(self, allow_not_found: bool): ... + def allow_not_found(self, allow_not_found: bool) -> None: + ... + @property - def recursive(self): ... + def recursive(self) -> None: + ... + @recursive.setter - def recursive(self, recursive: bool): ... - def __repr__(self): ... + def recursive(self, recursive: bool) -> None: + ... + + def __repr__(self) -> str: + ... class FileSystem(_Weakrefable): - def __init__(self): ... + + def __init__(self): + ... + @staticmethod - def _from_uri(uri): ... + def _from_uri(uri): + ... + @staticmethod - def _fsspec_from_uri(uri): ... + def _fsspec_from_uri(uri) -> tuple: + ... + @staticmethod - def _native_from_uri(uri): ... + def _native_from_uri(uri) -> tuple: + ... + @staticmethod - def from_uri(uri): ... - def equals(self, other: FileSystem): ... - def __eq__(self, other): ... + def from_uri(uri: str): + ... + + def equals(self, other: FileSystem) -> bool: + ... + + def __eq__(self, other): + ... + @property - def type_name(self): ... - def get_file_info(self, paths_or_selector): ... - def create_dir(self, path, *, recursive: bool=True): ... - def delete_dir(self, path): ... - def delete_dir_contents(self, path, *, accept_root_dir: bool=False, missing_dir_ok: bool=False): ... - def move(self, src, dest): ... - def copy_file(self, src, dest): ... - def delete_file(self, path): ... - def _wrap_input_stream(self, stream, path, compression, buffer_size): ... - def _wrap_output_stream(self, stream, path, compression, buffer_size): ... - def open_input_file(self, path): ... - def open_input_stream(self, path, compression='detect', buffer_size=None): ... - def open_output_stream(self, path, compression='detect', buffer_size=None, metadata=None): ... - def open_append_stream(self, path, compression='detect', buffer_size=None, metadata=None): ... - def normalize_path(self, path): ... + def type_name(self) -> str: + ... + + def get_file_info(self, paths_or_selector) -> FileInfo | list: + ... + + def create_dir(self, path: str, *, recursive: bool=True) -> None: + ... + + def delete_dir(self, path: str) -> None: + ... + + def delete_dir_contents(self, path: str, *, accept_root_dir: bool=False, missing_dir_ok: bool=False) -> None: + ... + + def move(self, src: str, dest: str) -> None: + ... + + def copy_file(self, src: str, dest: str) -> None: + ... + + def delete_file(self, path: str) -> None: + ... + + def _wrap_input_stream(self, stream, path, compression, buffer_size) -> pyarrow.BufferedInputStream: + ... + + def _wrap_output_stream(self, stream, path, compression, buffer_size) -> pyarrow.BufferedOutputStream: + ... + + def open_input_file(self, path: str) -> pyarrow.NativeFile: + ... + + def open_input_stream(self, path: str, compression: str | None='detect', buffer_size: int | None=None) -> pyarrow.NativeFile: + ... + + def open_output_stream(self, path: str, compression: str | None='detect', buffer_size: int | None=None, metadata: dict | None=None) -> pyarrow.NativeFile: + ... + + def open_append_stream(self, path: str, compression: str | None='detect', buffer_size: int | None=None, metadata: dict | None=None) -> pyarrow.NativeFile: + ... + + def normalize_path(self, path: str) -> str: + ... class LocalFileSystem(FileSystem): - def __init__(self, *, use_mmap=False): ... - def __reduce__(self): ... + + def __init__(self, *, use_mmap: bool=False): + ... + + def __reduce__(self) -> tuple: + ... class SubTreeFileSystem(FileSystem): - def __init__(self, base_path, base_fs: FileSystem): ... - def __repr__(self): ... - def __reduce__(self): ... + + def __init__(self, base_path, base_fs: FileSystem): + ... + + def __repr__(self) -> str: + ... + + def __reduce__(self) -> tuple: + ... + @property - def base_path(self): ... + def base_path(self) -> str: + ... + @property - def base_fs(self): ... + def base_fs(self) -> FileSystem: + ... class _MockFileSystem(FileSystem): - def __init__(self, current_time: datetime | None=None): ... + + def __init__(self, current_time: datetime | None=None): + ... class PyFileSystem(FileSystem): - def __init__(self, handler): ... + + def __init__(self, handler): + ... + @property - def handler(self): ... - def __reduce__(self): ... + def handler(self) -> FileSystemHandler: + ... + + def __reduce__(self) -> tuple: + ... class FileSystemHandler(ABC): + @abstractmethod - def get_type_name(self): ... - @abstractmethod - def get_file_info(self, paths): ... + def get_type_name(self) -> None: + ... + @abstractmethod - def get_file_info_selector(self, selector): ... + def get_file_info(self, paths: list[str]) -> None: + ... + @abstractmethod - def create_dir(self, path, recursive): ... + def get_file_info_selector(self, selector: FileSelector) -> None: + ... + @abstractmethod - def delete_dir(self, path): ... + def create_dir(self, path: str, recursive: bool) -> None: + ... + @abstractmethod - def delete_dir_contents(self, path, missing_dir_ok=False): ... + def delete_dir(self, path: str) -> None: + ... + @abstractmethod - def delete_root_dir_contents(self): ... + def delete_dir_contents(self, path: str, missing_dir_ok: bool=False) -> None: + ... + @abstractmethod - def delete_file(self, path): ... + def delete_root_dir_contents(self) -> None: + ... + @abstractmethod - def move(self, src, dest): ... + def delete_file(self, path: str) -> None: + ... + @abstractmethod - def copy_file(self, src, dest): ... + def move(self, src: str, dest: str) -> None: + ... + @abstractmethod - def open_input_stream(self, path): ... + def copy_file(self, src: str, dest: str) -> None: + ... + @abstractmethod - def open_input_file(self, path): ... + def open_input_stream(self, path: str) -> None: + ... + @abstractmethod - def open_output_stream(self, path, metadata): ... + def open_input_file(self, path: str) -> None: + ... + @abstractmethod - def open_append_stream(self, path, metadata): ... + def open_output_stream(self, path: str, metadata: Mapping[Any, Any]) -> None: + ... + @abstractmethod - def normalize_path(self, path): ... + def open_append_stream(self, path: str, metadata: Mapping[Any, Any]) -> None: + ... -def _file_type_to_string(ty): ... -def _copy_files(source_fs: FileSystem, source_path: str, destination_fs: FileSystem, destination_path: str, chunk_size: int, use_threads: c_bool): ... -def _copy_files_selector(source_fs: FileSystem, source_sel: FileSelector, destination_fs: FileSystem, destination_base_dir: str, chunk_size: int, use_threads: c_bool): ... + @abstractmethod + def normalize_path(self, path: str) -> None: + ... diff --git a/python/pyarrow-stubs/pyarrow/_gcsfs.pyi b/python/pyarrow-stubs/pyarrow/_gcsfs.pyi index e056dbf687f6..dfac2d2e323b 100644 --- a/python/pyarrow-stubs/pyarrow/_gcsfs.pyi +++ b/python/pyarrow-stubs/pyarrow/_gcsfs.pyi @@ -14,23 +14,34 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_gcsfs.pyx -from _typeshed import Incomplete +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete from pyarrow._fs import FileSystem -from pyarrow.includes.common import * -from pyarrow.includes.libarrow import * -from pyarrow.includes.libarrow_fs import * - class GcsFileSystem(FileSystem): - def __init__(self, *, anonymous: bool=False, access_token=None, target_service_account=None, credential_token_expiration=None, default_bucket_location='US', scheme=None, endpoint_override=None, default_metadata=None, retry_time_limit=None, project_id=None): ... - def _expiration_datetime_from_options(self): ... + + def __init__(self, *, anonymous: bool=False, access_token=None, target_service_account=None, credential_token_expiration=None, default_bucket_location: str='US', scheme=None, endpoint_override=None, default_metadata=None, retry_time_limit=None, project_id=None): + ... + + def _expiration_datetime_from_options(self): + ... + @staticmethod - def _reconstruct(kwargs): ... - def __reduce__(self): ... + def _reconstruct(kwargs) -> GcsFileSystem: + ... + + def __reduce__(self) -> tuple: + ... + @property - def default_bucket_location(self): ... + def default_bucket_location(self) -> str: + ... + @property - def project_id(self): ... + def project_id(self) -> str: + ... diff --git a/python/pyarrow-stubs/pyarrow/_hdfs.pyi b/python/pyarrow-stubs/pyarrow/_hdfs.pyi index b45038111f61..213c67cfb42b 100644 --- a/python/pyarrow-stubs/pyarrow/_hdfs.pyi +++ b/python/pyarrow-stubs/pyarrow/_hdfs.pyi @@ -14,22 +14,30 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_hdfs.pyx -from _typeshed import Incomplete +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete from pyarrow._fs import FileSystem -from pyarrow.includes.common import * -from pyarrow.includes.libarrow import * -from pyarrow.includes.libarrow_fs import * +def _have_libhdfs() -> bool: + ... class HadoopFileSystem(FileSystem): - def __init__(self, host: str, port: int=8020, *, user: str | None=None, replication: int=3, buffer_size: int=0, default_block_size=None, kerb_ticket=None, extra_conf=None): ... + + def __init__(self, host: str, port: int=8020, *, user: str | None=None, replication: int=3, buffer_size: int=0, default_block_size=None, kerb_ticket=None, extra_conf=None): + ... + @staticmethod - def from_uri(uri): ... + def from_uri(uri: str) -> HadoopFileSystem: + ... + @staticmethod - def _reconstruct(kwargs): ... - def __reduce__(self): ... + def _reconstruct(kwargs) -> HadoopFileSystem: + ... -def _have_libhdfs(): ... + def __reduce__(self) -> tuple: + ... diff --git a/python/pyarrow-stubs/pyarrow/_json.pyi b/python/pyarrow-stubs/pyarrow/_json.pyi index 4264fdaf35b7..f2126427bb07 100644 --- a/python/pyarrow-stubs/pyarrow/_json.pyi +++ b/python/pyarrow-stubs/pyarrow/_json.pyi @@ -14,60 +14,109 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_json.pyx -from _typeshed import Incomplete -from pyarrow.includes.common import * -from pyarrow.includes.libarrow import * +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +import pyarrow.json +from _typeshed import Incomplete from pyarrow.lib import MemoryPool, RecordBatchReader, Schema, _Weakrefable +def read_json(input_file, read_options: pyarrow.json.ReadOptions | None=None, parse_options: pyarrow.json.ParseOptions | None=None, memory_pool: MemoryPool | None=None) -> pyarrow.Table: + ... + +def open_json(input_file, read_options: pyarrow.json.ReadOptions | None=None, parse_options: pyarrow.json.ParseOptions | None=None, memory_pool: MemoryPool | None=None) -> JSONStreamingReader: + ... class ReadOptions(_Weakrefable): __slots__ = () - def __init__(self, use_threads=None, block_size=None): ... + def __init__(self, use_threads=None, block_size=None): + ... + @property - def use_threads(self): ... + def use_threads(self) -> None: + ... + @use_threads.setter - def use_threads(self, value): ... + def use_threads(self, value) -> None: + ... + @property - def block_size(self): ... + def block_size(self) -> None: + ... + @block_size.setter - def block_size(self, value): ... - def __reduce__(self): ... - def equals(self, other: ReadOptions): ... - def __eq__(self, other): ... - def __repr__(self): ... - def __str__(self): ... + def block_size(self, value) -> None: + ... + + def __reduce__(self) -> tuple: + ... + + def equals(self, other: ReadOptions) -> bool: + ... + + def __eq__(self, other): + ... + + def __repr__(self) -> str: + ... + + def __str__(self) -> str: + ... class ParseOptions(_Weakrefable): __slots__ = () - def __init__(self, explicit_schema=None, newlines_in_values=None, unexpected_field_behavior=None): ... - def __reduce__(self): ... + def __init__(self, explicit_schema=None, newlines_in_values=None, unexpected_field_behavior=None): + ... + + def __reduce__(self) -> tuple: + ... + @property - def explicit_schema(self): ... + def explicit_schema(self) -> None: + ... + @explicit_schema.setter - def explicit_schema(self, value): ... + def explicit_schema(self, value) -> None: + ... + @property - def newlines_in_values(self): ... + def newlines_in_values(self) -> None: + ... + @newlines_in_values.setter - def newlines_in_values(self, value): ... + def newlines_in_values(self, value) -> None: + ... + @property - def unexpected_field_behavior(self): ... + def unexpected_field_behavior(self) -> None: + ... + @unexpected_field_behavior.setter - def unexpected_field_behavior(self, value): ... - def equals(self, other: ParseOptions): ... - def __eq__(self, other): ... - def _repr_base(self): ... - def __repr__(self): ... - def __str__(self): ... + def unexpected_field_behavior(self, value) -> None: + ... + + def equals(self, other: ParseOptions) -> bool: + ... + + def __eq__(self, other): + ... + + def _repr_base(self) -> str: + ... + + def __repr__(self) -> str: + ... + + def __str__(self) -> str: + ... class JSONStreamingReader(RecordBatchReader): schema: Schema - def __init__(self): ... - -def read_json(input_file, read_options=None, parse_options=None, memory_pool: MemoryPool | None=None): ... -def open_json(input_file, read_options=None, parse_options=None, memory_pool: MemoryPool | None=None): ... + def __init__(self): + ... diff --git a/python/pyarrow-stubs/pyarrow/_orc.pyi b/python/pyarrow-stubs/pyarrow/_orc.pyi index dafb6f54f4f8..8f2979c10636 100644 --- a/python/pyarrow-stubs/pyarrow/_orc.pyi +++ b/python/pyarrow-stubs/pyarrow/_orc.pyi @@ -14,58 +14,105 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_orc.pyx -from _typeshed import Incomplete +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence from typing import TypedDict - from _typeshed import Incomplete -from pyarrow.includes.common import * -from pyarrow.includes.libarrow import * -from pyarrow.includes.libarrow import CCompressionType from pyarrow.lib import MemoryPool, Table, _Weakrefable from typing_extensions import TypeAlias - class ORCReader(_Weakrefable): - def __init__(self, memory_pool: MemoryPool | None=None): ... - def open(self, source: object, use_memory_map: c_bool=True): ... - def metadata(self): ... - def schema(self): ... - def nrows(self): ... - def nstripes(self): ... - def file_version(self): ... - def software_version(self): ... - def compression(self): ... - def compression_size(self): ... - def row_index_stride(self): ... - def writer(self): ... - def writer_version(self): ... - def nstripe_statistics(self): ... - def content_length(self): ... - def stripe_statistics_length(self): ... - def file_footer_length(self): ... - def file_postscript_length(self): ... - def file_length(self): ... - def serialized_file_tail(self): ... - def read_stripe(self, n, columns=None): ... - def read(self, columns=None): ... + + def __init__(self, memory_pool: MemoryPool | None=None): + ... + + def open(self, source: object, use_memory_map: bool=True) -> None: + ... + + def metadata(self) -> pyarrow.KeyValueMetadata: + ... + + def schema(self) -> pyarrow.Schema: + ... + + def nrows(self): + ... + + def nstripes(self): + ... + + def file_version(self): + ... + + def software_version(self) -> str: + ... + + def compression(self): + ... + + def compression_size(self): + ... + + def row_index_stride(self): + ... + + def writer(self): + ... + + def writer_version(self): + ... + + def nstripe_statistics(self): + ... + + def content_length(self): + ... + + def stripe_statistics_length(self): + ... + + def file_footer_length(self): + ... + + def file_postscript_length(self): + ... + + def file_length(self): + ... + + def serialized_file_tail(self): + ... + + def read_stripe(self, n, columns=None) -> pyarrow.RecordBatch: + ... + + def read(self, columns=None) -> pyarrow.Table: + ... class ORCWriter(_Weakrefable): - def open(self, where: object, *, file_version=None, batch_size=None, stripe_size=None, compression=None, compression_block_size=None, compression_strategy=None, row_index_stride=None, padding_tolerance=None, dictionary_key_size_threshold=None, bloom_filter_columns=None, bloom_filter_fpp=None): ... - def write(self, table: Table): ... - def close(self): ... + + def open(self, where: object, *, file_version=None, batch_size=None, stripe_size=None, compression=None, compression_block_size=None, compression_strategy=None, row_index_stride=None, padding_tolerance=None, dictionary_key_size_threshold=None, bloom_filter_columns=None, bloom_filter_fpp=None) -> None: + ... + + def write(self, table: Table) -> None: + ... + + def close(self) -> None: + ... class WriteOptions(TypedDict): batch_size: int - file_version: FileVersion + file_version: Incomplete stripe_size: int - compression: CCompressionType + compression: Incomplete compression_block_size: int - compression_strategy: CompressionStrategy + compression_strategy: Incomplete row_index_stride: int padding_tolerance: float dictionary_key_size_threshold: float - bloom_filter_columns: std_vector[int] + bloom_filter_columns: list[int] bloom_filter_fpp: float diff --git a/python/pyarrow-stubs/pyarrow/_parquet.pyi b/python/pyarrow-stubs/pyarrow/_parquet.pyi index ec09ead3d918..ce6bb4fc20d8 100644 --- a/python/pyarrow-stubs/pyarrow/_parquet.pyi +++ b/python/pyarrow-stubs/pyarrow/_parquet.pyi @@ -14,282 +14,591 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_parquet.pyx -from _typeshed import Incomplete +import os +import re +import pyarrow +from typing import Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete from typing import Any - -from pyarrow.includes.common import * -from pyarrow.includes.libarrow import * -from pyarrow.includes.libarrow_python import * -from pyarrow.includes.libparquet import * from pyarrow.lib import Buffer, MemoryPool, Schema, Table, _Weakrefable +_DEFAULT_ROW_GROUP_SIZE: int +_MAX_ROW_GROUP_SIZE: int -_DEFAULT_ROW_GROUP_SIZE = 1024 * 1024 -_MAX_ROW_GROUP_SIZE = 64 * 1024 * 1024 +def _reconstruct_filemetadata(serialized: Buffer) -> FileMetaData: + ... class Statistics(_Weakrefable): - def __init__(self): ... - def __repr__(self): ... - def to_dict(self): ... - def __eq__(self, other): ... - def equals(self, other: Statistics): ... + + def __init__(self): + ... + + def __repr__(self) -> str: + ... + + def to_dict(self) -> dict: + ... + + def __eq__(self, other): + ... + + def equals(self, other: Statistics) -> bool: + ... + @property - def has_min_max(self): ... + def has_min_max(self): + ... + @property - def has_null_count(self): ... + def has_null_count(self): + ... + @property - def has_distinct_count(self): ... + def has_distinct_count(self): + ... + @property - def min_raw(self): ... + def min_raw(self): + ... + @property - def max_raw(self): ... + def max_raw(self): + ... + @property - def min(self): ... + def min(self): + ... + @property - def max(self): ... + def max(self): + ... + @property - def null_count(self): ... + def null_count(self): + ... + @property - def distinct_count(self): ... + def distinct_count(self): + ... + @property - def num_values(self): ... + def num_values(self): + ... + @property - def physical_type(self): ... + def physical_type(self): + ... + @property - def logical_type(self): ... + def logical_type(self): + ... + @property - def converted_type(self): ... + def converted_type(self): + ... class ParquetLogicalType(_Weakrefable): - def __init__(self): ... - def __repr__(self): ... - def __str__(self): ... - def to_json(self): ... + + def __init__(self): + ... + + def __repr__(self) -> str: + ... + + def __str__(self) -> str: + ... + + def to_json(self) -> str: + ... + @property - def type(self): ... + def type(self): + ... class GeoStatistics(_Weakrefable): - def __init__(self): ... - def __repr__(self): ... - def to_dict(self): ... + + def __init__(self): + ... + + def __repr__(self) -> str: + ... + + def to_dict(self) -> dict: + ... + @property - def geospatial_types(self): ... + def geospatial_types(self) -> None | list: + ... + @property - def xmin(self): ... + def xmin(self): + ... + @property - def xmax(self): ... + def xmax(self): + ... + @property - def ymin(self): ... + def ymin(self): + ... + @property - def ymax(self): ... + def ymax(self): + ... + @property - def zmin(self): ... + def zmin(self): + ... + @property - def zmax(self): ... + def zmax(self): + ... + @property - def mmin(self): ... + def mmin(self): + ... + @property - def mmax(self): ... - def _x_valid(self): ... - def _y_valid(self): ... - def _z_valid(self): ... - def _m_valid(self): ... + def mmax(self): + ... + + def _x_valid(self): + ... + + def _y_valid(self): + ... + + def _z_valid(self): + ... + + def _m_valid(self): + ... class ColumnChunkMetaData(_Weakrefable): - def __init__(self): ... - def __repr__(self): ... - def to_dict(self): ... - def __eq__(self, other): ... - def equals(self, other: ColumnChunkMetaData): ... + + def __init__(self): + ... + + def __repr__(self) -> str: + ... + + def to_dict(self) -> dict: + ... + + def __eq__(self, other): + ... + + def equals(self, other: ColumnChunkMetaData) -> bool: + ... + @property - def file_offset(self): ... + def file_offset(self): + ... + @property - def file_path(self): ... + def file_path(self) -> str: + ... + @property - def physical_type(self): ... + def physical_type(self): + ... + @property - def num_values(self): ... + def num_values(self): + ... + @property - def path_in_schema(self): ... + def path_in_schema(self) -> str: + ... + @property - def is_stats_set(self): ... + def is_stats_set(self): + ... + @property - def statistics(self): ... + def statistics(self) -> None | Statistics: + ... + @property - def is_geo_stats_set(self): ... + def is_geo_stats_set(self): + ... + @property - def geo_statistics(self): ... + def geo_statistics(self) -> None | GeoStatistics: + ... + @property - def compression(self): ... + def compression(self): + ... + @property - def encodings(self): ... + def encodings(self) -> tuple: + ... + @property - def has_dictionary_page(self): ... + def has_dictionary_page(self) -> bool: + ... + @property - def dictionary_page_offset(self): ... + def dictionary_page_offset(self): + ... + @property - def data_page_offset(self): ... + def data_page_offset(self): + ... + @property - def has_index_page(self): ... + def has_index_page(self): + ... + @property - def index_page_offset(self): ... + def index_page_offset(self): + ... + @property - def total_compressed_size(self): ... + def total_compressed_size(self): + ... + @property - def total_uncompressed_size(self): ... + def total_uncompressed_size(self): + ... + @property - def bloom_filter_offset(self): ... + def bloom_filter_offset(self): + ... + @property - def bloom_filter_length(self): ... + def bloom_filter_length(self): + ... + @property - def has_offset_index(self): ... + def has_offset_index(self): + ... + @property - def has_column_index(self): ... + def has_column_index(self): + ... + @property - def metadata(self): ... + def metadata(self): + ... class SortingColumn: - def __init__(self, column_index: int, descending: c_bool=False, nulls_first: c_bool=False): ... + + def __init__(self, column_index: int, descending: bool=False, nulls_first: bool=False): + ... + @classmethod - def from_ordering(cls, schema: Schema, sort_keys, null_placement='at_end'): ... + def from_ordering(cls, schema: Schema, sort_keys, null_placement: str='at_end') -> tuple: + ... + @staticmethod - def to_ordering(schema: Schema, sorting_columns): ... - def __repr__(self): ... - def __eq__(self, other: SortingColumn): ... - def __hash__(self): ... + def to_ordering(schema: Schema, sorting_columns: tuple[SortingColumn, ...]) -> tuple: + ... + + def __repr__(self) -> str: + ... + + def __eq__(self, other: SortingColumn): + ... + + def __hash__(self): + ... + @property - def column_index(self): ... + def column_index(self): + ... + @property - def descending(self): ... + def descending(self): + ... + @property - def nulls_first(self): ... - def to_dict(self): ... + def nulls_first(self): + ... + + def to_dict(self) -> dict: + ... class RowGroupMetaData(_Weakrefable): - def __init__(self): ... - def __reduce__(self): ... - def __eq__(self, other): ... - def equals(self, other: RowGroupMetaData): ... - def column(self, i: int): ... - def __repr__(self): ... - def to_dict(self): ... + + def __init__(self): + ... + + def __reduce__(self) -> tuple: + ... + + def __eq__(self, other): + ... + + def equals(self, other: RowGroupMetaData) -> bool: + ... + + def column(self, i: int) -> ColumnChunkMetaData: + ... + + def __repr__(self) -> str: + ... + + def to_dict(self) -> dict: + ... + @property - def num_columns(self): ... + def num_columns(self): + ... + @property - def num_rows(self): ... + def num_rows(self): + ... + @property - def total_byte_size(self): ... + def total_byte_size(self): + ... + @property - def sorting_columns(self): ... + def sorting_columns(self) -> tuple: + ... class FileMetaData(_Weakrefable): - def __init__(self): ... - def __reduce__(self): ... - def __hash__(self): ... - def __repr__(self): ... - def to_dict(self): ... - def __eq__(self, other): ... - def equals(self, other: FileMetaData): ... + + def __init__(self): + ... + + def __reduce__(self) -> tuple: + ... + + def __hash__(self): + ... + + def __repr__(self) -> str: + ... + + def to_dict(self) -> dict: + ... + + def __eq__(self, other): + ... + + def equals(self, other: FileMetaData) -> bool: + ... + @property - def schema(self): ... + def schema(self): + ... + @property - def serialized_size(self): ... + def serialized_size(self): + ... + @property - def num_columns(self): ... + def num_columns(self): + ... + @property - def num_rows(self): ... + def num_rows(self): + ... + @property - def num_row_groups(self): ... + def num_row_groups(self): + ... + @property - def format_version(self): ... + def format_version(self) -> str: + ... + @property - def created_by(self): ... + def created_by(self) -> str: + ... + @property - def metadata(self): ... - def row_group(self, i: int): ... - def set_file_path(self, path): ... - def append_row_groups(self, other: FileMetaData): ... - def write_metadata_file(self, where): ... + def metadata(self): + ... + + def row_group(self, i: int) -> RowGroupMetaData: + ... + + def set_file_path(self, path: str) -> None: + ... + + def append_row_groups(self, other: FileMetaData) -> None: + ... + + def write_metadata_file(self, where) -> None: + ... class ParquetSchema(_Weakrefable): - def __init__(self, container: FileMetaData): ... - def __repr__(self): ... - def __reduce__(self): ... - def __len__(self): ... - def __getitem__(self, i): ... - def __hash__(self): ... - @property - def names(self): ... - def to_arrow_schema(self): ... - def __eq__(self, other): ... - def equals(self, other: ParquetSchema): ... - def column(self, i): ... + + def __init__(self, container: FileMetaData): + ... + + def __repr__(self) -> str: + ... + + def __reduce__(self) -> tuple: + ... + + def __len__(self): + ... + + def __getitem__(self, i): + ... + + def __hash__(self): + ... + + @property + def names(self) -> list: + ... + + def to_arrow_schema(self) -> pyarrow.Schema: + ... + + def __eq__(self, other): + ... + + def equals(self, other: ParquetSchema) -> bool: + ... + + def column(self, i: int) -> ColumnSchema: + ... class ColumnSchema(_Weakrefable): - def __init__(self, schema: ParquetSchema, index: int): ... - def __eq__(self, other): ... - def __reduce__(self): ... - def equals(self, other: ColumnSchema): ... - def __repr__(self): ... + + def __init__(self, schema: ParquetSchema, index: int): + ... + + def __eq__(self, other): + ... + + def __reduce__(self) -> tuple: + ... + + def equals(self, other: ColumnSchema) -> bool: + ... + + def __repr__(self) -> str: + ... + @property - def name(self): ... + def name(self) -> str: + ... + @property - def path(self): ... + def path(self) -> str: + ... + @property - def max_definition_level(self): ... + def max_definition_level(self): + ... + @property - def max_repetition_level(self): ... + def max_repetition_level(self): + ... + @property - def physical_type(self): ... + def physical_type(self): + ... + @property - def logical_type(self): ... + def logical_type(self): + ... + @property - def converted_type(self): ... + def converted_type(self): + ... + @property - def length(self): ... + def length(self): + ... + @property - def precision(self): ... + def precision(self): + ... + @property - def scale(self): ... + def scale(self): + ... class ParquetReader(_Weakrefable): _column_idx_map: Any - def __init__(self, memory_pool: MemoryPool | None=None): ... - def open(self, source: object, *, use_memory_map: bool=False, read_dictionary=None, binary_type=None, list_type=None, metadata: FileMetaData | None=None, buffer_size: int=0, pre_buffer: bool=True, coerce_int96_timestamp_unit=None, decryption_properties: FileDecryptionProperties | None=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, page_checksum_verification=False, arrow_extensions_enabled=False): ... + def __init__(self, memory_pool: MemoryPool | None=None): + ... + + def open(self, source: object, *, use_memory_map: bool=False, read_dictionary=None, binary_type: pyarrow.DataType | None=None, list_type=None, metadata: FileMetaData | None=None, buffer_size: int=0, pre_buffer: bool=True, coerce_int96_timestamp_unit: str | None=None, decryption_properties: FileDecryptionProperties | None=None, thrift_string_size_limit: int | None=None, thrift_container_size_limit: int | None=None, schema_depth_limit: int | None=None, page_checksum_verification: bool=False, arrow_extensions_enabled: bool=False) -> None: + ... + @property - def column_paths(self): ... + def column_paths(self) -> list: + ... + @property - def metadata(self): ... + def metadata(self): + ... + @property - def schema_arrow(self): ... + def schema_arrow(self) -> pyarrow.Schema: + ... + @property - def num_row_groups(self): ... - def set_use_threads(self, use_threads: bool): ... - def set_batch_size(self, batch_size: int): ... - def iter_batches(self, batch_size: int, row_groups, column_indices=None, use_threads: bool=True): ... - def read_row_group(self, i: int, column_indices=None, use_threads: bool=True): ... - def read_row_groups(self, row_groups, column_indices=None, use_threads: bool=True): ... - def read_all(self, column_indices=None, use_threads: bool=True): ... - def scan_contents(self, column_indices=None, batch_size=65536): ... - def column_name_idx(self, column_name): ... - def read_column(self, column_index: int): ... - def close(self): ... + def num_row_groups(self): + ... + + def set_use_threads(self, use_threads: bool) -> None: + ... + + def set_batch_size(self, batch_size: int) -> None: + ... + + def iter_batches(self, batch_size: int, row_groups: list[int], column_indices: list[int] | None=None, use_threads: bool=True) -> Iterator[pyarrow.RecordBatch]: + ... + + def read_row_group(self, i: int, column_indices: list[int] | None=None, use_threads: bool=True) -> pyarrow.Table: + ... + + def read_row_groups(self, row_groups: list[int], column_indices: list[int] | None=None, use_threads: bool=True) -> pyarrow.Table: + ... + + def read_all(self, column_indices: list[int] | None=None, use_threads: bool=True) -> pyarrow.Table: + ... + + def scan_contents(self, column_indices: list[int] | None=None, batch_size: int=65536) -> int: + ... + + def column_name_idx(self, column_name: str) -> int: + ... + + def read_column(self, column_index: int) -> pyarrow.ChunkedArray: + ... + + def close(self) -> None: + ... + @property - def closed(self): ... + def closed(self): + ... class ParquetWriter(_Weakrefable): - def __init__(self, where, schema: Schema, use_dictionary=None, compression=None, version=None, write_statistics=None, memory_pool: MemoryPool | None=None, use_deprecated_int96_timestamps=False, coerce_timestamps=None, data_page_size=None, max_rows_per_page=None, allow_truncated_timestamps=False, compression_level=None, use_byte_stream_split=False, column_encoding=None, writer_engine_version=None, data_page_version=None, use_compliant_nested_type=True, encryption_properties=None, write_batch_size=None, dictionary_pagesize_limit=None, store_schema=True, write_page_index=False, write_page_checksum=False, sorting_columns=None, store_decimal_as_integer=False, use_content_defined_chunking=False, write_time_adjusted_to_utc=False, bloom_filter_options=None): ... - def close(self): ... - def write_table(self, table: Table, row_group_size=None): ... - def add_key_value_metadata(self, key_value_metadata): ... - @property - def metadata(self): ... -class FileEncryptionProperties: ... + def __init__(self, where, schema: Schema, use_dictionary=None, compression=None, version=None, write_statistics=None, memory_pool: MemoryPool | None=None, use_deprecated_int96_timestamps: bool=False, coerce_timestamps=None, data_page_size=None, max_rows_per_page=None, allow_truncated_timestamps: bool=False, compression_level=None, use_byte_stream_split: bool=False, column_encoding=None, writer_engine_version=None, data_page_version=None, use_compliant_nested_type: bool=True, encryption_properties=None, write_batch_size=None, dictionary_pagesize_limit=None, store_schema: bool=True, write_page_index: bool=False, write_page_checksum: bool=False, sorting_columns=None, store_decimal_as_integer: bool=False, use_content_defined_chunking: bool=False, write_time_adjusted_to_utc: bool=False, bloom_filter_options=None): + ... + + def close(self) -> None: + ... + + def write_table(self, table: Table, row_group_size=None) -> None: + ... + + def add_key_value_metadata(self, key_value_metadata) -> None: + ... + + @property + def metadata(self) -> FileMetaData: + ... -class FileDecryptionProperties: ... +class FileEncryptionProperties: + ... -def _reconstruct_filemetadata(serialized: Buffer): ... +class FileDecryptionProperties: + ... diff --git a/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi b/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi index 0c835d889e62..3ef1683e171b 100644 --- a/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi +++ b/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi @@ -14,133 +14,239 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_parquet_encryption.pyx -from _typeshed import Incomplete +import os +import re +import pyarrow +from typing import Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +import pyarrow.parquet +from _typeshed import Incomplete from typing import Any - from pyarrow._fs import FileSystem -from pyarrow.includes.common import * -from pyarrow.includes.libarrow import * -from pyarrow.includes.libparquet_encryption import * from pyarrow.lib import _Weakrefable +def create_decryption_properties(footer_key: bytes, *, aad_prefix: bytes | None=None, check_footer_integrity: bool=True, allow_plaintext_files: bool=False) -> pyarrow.parquet.FileDecryptionProperties: + ... + +def create_encryption_properties(footer_key: bytes, *, aad_prefix: bytes | None=None, store_aad_prefix: bool=True, encryption_algorithm: str='AES_GCM_V1', plaintext_footer: bool=False) -> pyarrow.parquet.FileEncryptionProperties: + ... class EncryptionConfiguration(_Weakrefable): __slots__ = () - def __init__(self, footer_key, *, column_keys=None, uniform_encryption=None, encryption_algorithm=None, plaintext_footer=None, double_wrapping=None, cache_lifetime=None, internal_key_material=None, data_key_length_bits=None): ... + def __init__(self, footer_key, *, column_keys=None, uniform_encryption=None, encryption_algorithm=None, plaintext_footer=None, double_wrapping=None, cache_lifetime=None, internal_key_material=None, data_key_length_bits=None): + ... + @property - def footer_key(self): ... + def footer_key(self) -> str: + ... + @property - def column_keys(self): ... + def column_keys(self) -> None: + ... + @column_keys.setter - def column_keys(self, value: dict[Any, Any]): ... + def column_keys(self, value: dict[Any, Any]) -> None: + ... + @property - def uniform_encryption(self): ... + def uniform_encryption(self) -> None: + ... + @uniform_encryption.setter - def uniform_encryption(self, value): ... + def uniform_encryption(self, value) -> None: + ... + @property - def encryption_algorithm(self): ... + def encryption_algorithm(self) -> None: + ... + @encryption_algorithm.setter - def encryption_algorithm(self, value): ... + def encryption_algorithm(self, value) -> None: + ... + @property - def plaintext_footer(self): ... + def plaintext_footer(self) -> None: + ... + @plaintext_footer.setter - def plaintext_footer(self, value): ... + def plaintext_footer(self, value) -> None: + ... + @property - def double_wrapping(self): ... + def double_wrapping(self) -> None: + ... + @double_wrapping.setter - def double_wrapping(self, value): ... + def double_wrapping(self, value) -> None: + ... + @property - def cache_lifetime(self): ... + def cache_lifetime(self) -> None: + ... + @cache_lifetime.setter - def cache_lifetime(self, value): ... + def cache_lifetime(self, value) -> None: + ... + @property - def internal_key_material(self): ... + def internal_key_material(self) -> None: + ... + @internal_key_material.setter - def internal_key_material(self, value): ... + def internal_key_material(self, value) -> None: + ... + @property - def data_key_length_bits(self): ... + def data_key_length_bits(self) -> None: + ... + @data_key_length_bits.setter - def data_key_length_bits(self, value): ... + def data_key_length_bits(self, value) -> None: + ... class DecryptionConfiguration(_Weakrefable): __slots__ = () - def __init__(self, *, cache_lifetime=None, read_kms_url=None): ... + def __init__(self, *, cache_lifetime=None, read_kms_url=None): + ... + @property - def cache_lifetime(self): ... + def cache_lifetime(self) -> None: + ... + @cache_lifetime.setter - def cache_lifetime(self, value): ... + def cache_lifetime(self, value) -> None: + ... + @property - def read_kms_url(self): ... + def read_kms_url(self) -> None: + ... + @read_kms_url.setter - def read_kms_url(self, value): ... + def read_kms_url(self, value) -> None: + ... class KmsConnectionConfig(_Weakrefable): __slots__ = () - def __init__(self, *, kms_instance_id=None, kms_instance_url=None, key_access_token=None, custom_kms_conf=None): ... + def __init__(self, *, kms_instance_id=None, kms_instance_url=None, key_access_token=None, custom_kms_conf=None): + ... + @property - def kms_instance_id(self): ... + def kms_instance_id(self) -> None: + ... + @kms_instance_id.setter - def kms_instance_id(self, value): ... + def kms_instance_id(self, value) -> None: + ... + @property - def kms_instance_url(self): ... + def kms_instance_url(self) -> None: + ... + @kms_instance_url.setter - def kms_instance_url(self, value): ... + def kms_instance_url(self, value) -> None: + ... + @property - def key_access_token(self): ... + def key_access_token(self) -> None: + ... + @key_access_token.setter - def key_access_token(self, value): ... + def key_access_token(self, value) -> None: + ... + @property - def custom_kms_conf(self): ... + def custom_kms_conf(self) -> None: + ... + @custom_kms_conf.setter - def custom_kms_conf(self, value: dict[Any, Any]): ... - def refresh_key_access_token(self, value): ... + def custom_kms_conf(self, value: dict[Any, Any]) -> None: + ... + + def refresh_key_access_token(self, value) -> None: + ... class KmsClient(_Weakrefable): - def __init__(self): ... - def wrap_key(self, key_bytes, master_key_identifier): ... - def unwrap_key(self, wrapped_key, master_key_identifier): ... + + def __init__(self): + ... + + def wrap_key(self, key_bytes, master_key_identifier): + ... + + def unwrap_key(self, wrapped_key, master_key_identifier): + ... class CryptoFactory(_Weakrefable): __slots__ = () - def __init__(self, kms_client_factory): ... - def file_encryption_properties(self, kms_connection_config: KmsConnectionConfig, encryption_config: EncryptionConfiguration, parquet_file_path=None, filesystem: FileSystem | None=None): ... - def file_decryption_properties(self, kms_connection_config: KmsConnectionConfig, decryption_config: DecryptionConfiguration | None=None, parquet_file_path=None, filesystem: FileSystem | None=None): ... - def remove_cache_entries_for_token(self, access_token): ... - def remove_cache_entries_for_all_tokens(self): ... - def rotate_master_keys(self, kms_connection_config: KmsConnectionConfig, parquet_file_path, filesystem: FileSystem | None=None, double_wrapping=True, cache_lifetime_seconds=600, read_kms_url=False): ... + def __init__(self, kms_client_factory): + ... + + def file_encryption_properties(self, kms_connection_config: KmsConnectionConfig, encryption_config: EncryptionConfiguration, parquet_file_path=None, filesystem: FileSystem | None=None) -> pyarrow.parquet.FileEncryptionProperties: + ... + + def file_decryption_properties(self, kms_connection_config: KmsConnectionConfig, decryption_config: DecryptionConfiguration | None=None, parquet_file_path=None, filesystem: FileSystem | None=None) -> pyarrow.parquet.FileDecryptionProperties: + ... + + def remove_cache_entries_for_token(self, access_token) -> None: + ... + + def remove_cache_entries_for_all_tokens(self) -> None: + ... + + def rotate_master_keys(self, kms_connection_config: KmsConnectionConfig, parquet_file_path, filesystem: FileSystem | None=None, double_wrapping: bool=True, cache_lifetime_seconds: int | float=600, read_kms_url: bool=False) -> None: + ... class KeyMaterial(_Weakrefable): + @property - def is_footer_key(self): ... + def is_footer_key(self): + ... + @property - def is_double_wrapped(self): ... + def is_double_wrapped(self): + ... + @property - def master_key_id(self): ... + def master_key_id(self) -> str: + ... + @property - def wrapped_dek(self): ... + def wrapped_dek(self) -> str: + ... + @property - def kek_id(self): ... + def kek_id(self) -> str: + ... + @property - def wrapped_kek(self): ... + def wrapped_kek(self) -> str: + ... + @property - def kms_instance_id(self): ... + def kms_instance_id(self) -> str: + ... + @property - def kms_instance_url(self): ... + def kms_instance_url(self) -> str: + ... + @staticmethod - def parse(key_material_string: c_string): ... + def parse(key_material_string: bytes) -> KeyMaterial: + ... class FileSystemKeyMaterialStore(_Weakrefable): - def get_key_material(self, key_id): ... - def get_key_id_set(self): ... - @classmethod - def for_file(cls, parquet_file_path, filesystem: FileSystem | None=None): ... -def create_decryption_properties(footer_key, *, aad_prefix=None, check_footer_integrity: bool=True, allow_plaintext_files: bool=False): ... -def create_encryption_properties(footer_key, *, aad_prefix=None, store_aad_prefix: bool=True, encryption_algorithm='AES_GCM_V1', plaintext_footer: bool=False): ... + def get_key_material(self, key_id): + ... + + def get_key_id_set(self): + ... + + @classmethod + def for_file(cls, parquet_file_path, filesystem: FileSystem | None=None) -> FileSystemKeyMaterialStore: + ... diff --git a/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi b/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi index ca27b2108ae0..5593c5fb2d2b 100644 --- a/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi +++ b/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi @@ -14,23 +14,27 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_pyarrow_cpp_tests.pyx -from _typeshed import Incomplete +import os +import re +import pyarrow +from typing import Any, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence from typing import Callable - from _typeshed import Incomplete -from pyarrow.includes.common import * -from pyarrow.includes.libarrow import * -from pyarrow.includes.libarrow import CStatus from typing_extensions import TypeAlias +def get_cpp_tests() -> list: + ... class CppTestCase: + @property - def name(self): ... - def __repr__(self): ... - def __call__(self): ... + def name(self) -> str: + ... + + def __repr__(self) -> str: + ... -def get_cpp_tests(): ... + def __call__(self) -> None: + ... diff --git a/python/pyarrow-stubs/pyarrow/_s3fs.pyi b/python/pyarrow-stubs/pyarrow/_s3fs.pyi index f1219553325d..c0c611c821ed 100644 --- a/python/pyarrow-stubs/pyarrow/_s3fs.pyi +++ b/python/pyarrow-stubs/pyarrow/_s3fs.pyi @@ -14,17 +14,30 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_s3fs.pyx -from _typeshed import Incomplete +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete from enum import IntEnum - from pyarrow._fs import FileSystem -from pyarrow.includes.common import * -from pyarrow.includes.libarrow import * -from pyarrow.includes.libarrow_fs import * +def initialize_s3(log_level: S3LogLevel=S3LogLevel.Fatal, num_event_loop_threads: int=1) -> None: + ... + +def ensure_s3_initialized() -> None: + ... + +def finalize_s3() -> None: + ... + +def ensure_s3_finalized() -> None: + ... + +def resolve_s3_region(bucket: str) -> str: + ... class S3LogLevel(IntEnum): Off = ... @@ -36,22 +49,28 @@ class S3LogLevel(IntEnum): Trace = ... class S3RetryStrategy: - def __init__(self, max_attempts=3): ... -class AwsStandardS3RetryStrategy(S3RetryStrategy): ... + def __init__(self, max_attempts: int=3): + ... + +class AwsStandardS3RetryStrategy(S3RetryStrategy): + ... -class AwsDefaultS3RetryStrategy(S3RetryStrategy): ... +class AwsDefaultS3RetryStrategy(S3RetryStrategy): + ... class S3FileSystem(FileSystem): - def __init__(self, *, access_key=None, secret_key=None, session_token=None, anonymous: bool=False, region=None, request_timeout=None, connect_timeout=None, scheme=None, endpoint_override=None, background_writes: bool=True, default_metadata=None, role_arn=None, session_name=None, external_id=None, load_frequency=900, proxy_options=None, allow_delayed_open=False, allow_bucket_creation=False, allow_bucket_deletion=False, check_directory_existence_before_creation=False, retry_strategy: S3RetryStrategy=AwsStandardS3RetryStrategy(max_attempts=3), force_virtual_addressing=False, tls_ca_file_path=None): ... + + def __init__(self, *, access_key=None, secret_key=None, session_token=None, anonymous: bool=False, region=None, request_timeout=None, connect_timeout=None, scheme=None, endpoint_override=None, background_writes: bool=True, default_metadata=None, role_arn=None, session_name=None, external_id=None, load_frequency: int=900, proxy_options=None, allow_delayed_open: bool=False, allow_bucket_creation: bool=False, allow_bucket_deletion: bool=False, check_directory_existence_before_creation: bool=False, retry_strategy: S3RetryStrategy=AwsStandardS3RetryStrategy(max_attempts=3), force_virtual_addressing: bool=False, tls_ca_file_path=None): + ... + @staticmethod - def _reconstruct(kwargs): ... - def __reduce__(self): ... - @property - def region(self): ... + def _reconstruct(kwargs) -> S3FileSystem: + ... + + def __reduce__(self) -> tuple: + ... -def initialize_s3(log_level: S3LogLevel=S3LogLevel.Fatal, num_event_loop_threads: int=1): ... -def ensure_s3_initialized(): ... -def finalize_s3(): ... -def ensure_s3_finalized(): ... -def resolve_s3_region(bucket): ... + @property + def region(self) -> str: + ... diff --git a/python/pyarrow-stubs/pyarrow/_substrait.pyi b/python/pyarrow-stubs/pyarrow/_substrait.pyi index 8916775be1a8..11e067fb5514 100644 --- a/python/pyarrow-stubs/pyarrow/_substrait.pyi +++ b/python/pyarrow-stubs/pyarrow/_substrait.pyi @@ -14,36 +14,64 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/_substrait.pyx -from _typeshed import Incomplete -from pyarrow.includes.libarrow import * -from pyarrow.includes.libarrow_substrait import * +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +import pyarrow.compute +from pyarrow.lib import _Weakrefable +from _typeshed import Incomplete from pyarrow.lib import * +from types import ModuleType + +def run_query(plan, *, table_provider: object | None=None, use_threads: bool=True) -> pyarrow.RecordBatchReader: + ... + +def _parse_json_plan(plan: bytes) -> pyarrow.Buffer: + ... + +def serialize_schema(schema: pyarrow.Schema) -> SubstraitSchema: + ... + +def _serialize_namedstruct_schema(schema): + ... + +def deserialize_schema(buf: pyarrow.Buffer | bytes | SubstraitSchema) -> pyarrow.Schema: + ... + +def serialize_expressions(exprs: list[pyarrow.compute.Expression], names: list[str], schema: pyarrow.Schema, *, allow_arrow_extensions: bool=False) -> pyarrow.Buffer: + ... + +def deserialize_expressions(buf: pyarrow.Buffer | bytes) -> BoundExpressions: + ... +def get_supported_functions() -> list: + ... +py_substrait: ModuleType | None class SubstraitSchema: - def __init__(self, schema, expression): ... - def to_pysubstrait(self): ... + + def __init__(self, schema, expression): + ... + + def to_pysubstrait(self): + ... class BoundExpressions(_Weakrefable): - def __init__(self): ... - @property - def schema(self): ... + + def __init__(self): + ... + @property - def expressions(self): ... - @classmethod - def from_substrait(cls, message): ... + def schema(self) -> pyarrow.Schema: + ... -def run_query(plan, *, table_provider=None, use_threads=True): ... -def _parse_json_plan(plan): ... -def serialize_schema(schema): ... -def _serialize_namedstruct_schema(schema): ... -def deserialize_schema(buf): ... -def serialize_expressions(exprs, names, schema, *, allow_arrow_extensions=False): ... -def deserialize_expressions(buf): ... -def get_supported_functions(): ... + @property + def expressions(self) -> dict: + ... -from types import ModuleType -py_substrait: ModuleType | None + @classmethod + def from_substrait(cls, message) -> BoundExpressions: + ... diff --git a/python/pyarrow-stubs/pyarrow/acero.pyi b/python/pyarrow-stubs/pyarrow/acero.pyi index 3e7b3736f5b0..4e3b177f7980 100644 --- a/python/pyarrow-stubs/pyarrow/acero.pyi +++ b/python/pyarrow-stubs/pyarrow/acero.pyi @@ -15,19 +15,41 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete +import pyarrow.compute +import pyarrow.dataset from pyarrow._acero import AggregateNodeOptions as AggregateNodeOptions, AsofJoinNodeOptions as AsofJoinNodeOptions, Declaration as Declaration, ExecNodeOptions as ExecNodeOptions, FilterNodeOptions as FilterNodeOptions, HashJoinNodeOptions as HashJoinNodeOptions, OrderByNodeOptions as OrderByNodeOptions, ProjectNodeOptions as ProjectNodeOptions, RecordBatchReaderSourceNodeOptions as RecordBatchReaderSourceNodeOptions, TableSourceNodeOptions as TableSourceNodeOptions from pyarrow._dataset import ScanNodeOptions as ScanNodeOptions from pyarrow.compute import Expression as Expression, field as field from pyarrow.lib import RecordBatch as RecordBatch, Table as Table, array as array +ds: Incomplete + +def _dataset_to_decl(dataset, use_threads: bool=True, implicit_ordering: bool=False): + ... + +def _perform_join(join_type: str, left_operand: Table | pyarrow.dataset.Dataset, left_keys: str | list[str], right_operand: Table | pyarrow.dataset.Dataset, right_keys: str | list[str], left_suffix: str | None=None, right_suffix: str | None=None, use_threads: bool=True, coalesce_keys: bool=False, output_type=..., filter_expression: pyarrow.compute.Expression | None=None) -> Table | pyarrow.dataset.InMemoryDataset: + ... + +def _perform_join_asof(left_operand: Table | pyarrow.dataset.Dataset, left_on: str, left_by, right_operand: Table | pyarrow.dataset.Dataset, right_on: str | list[str], right_by, tolerance: int, use_threads: bool=True, output_type=...) -> Table | pyarrow.dataset.InMemoryDataset: + ... + +def _filter_table(table: Table | RecordBatch, expression: Expression) -> Table | RecordBatch: + ... + +def _sort_source(table_or_dataset, sort_keys, output_type=..., **kwargs): + ... + +def _group_by(table, aggregates, keys, use_threads: bool=True): + ... class DatasetModuleStub: - class Dataset: ... - class InMemoryDataset: ... -ds = DatasetModuleStub - -def _dataset_to_decl(dataset, use_threads: bool = True, implicit_ordering: bool = False): ... -def _perform_join(join_type, left_operand, left_keys, right_operand, right_keys, left_suffix=None, right_suffix=None, use_threads: bool = True, coalesce_keys: bool = False, output_type=..., filter_expression=None): ... -def _perform_join_asof(left_operand, left_on, left_by, right_operand, right_on, right_by, tolerance, use_threads: bool = True, output_type=...): ... -def _filter_table(table, expression): ... -def _sort_source(table_or_dataset, sort_keys, output_type=..., **kwargs): ... -def _group_by(table, aggregates, keys, use_threads: bool = True): ... + + class Dataset: + ... + + class InMemoryDataset: + ... diff --git a/python/pyarrow-stubs/pyarrow/benchmark.pyi b/python/pyarrow-stubs/pyarrow/benchmark.pyi index 68aa3ab3db99..8fa8a22c1956 100644 --- a/python/pyarrow-stubs/pyarrow/benchmark.pyi +++ b/python/pyarrow-stubs/pyarrow/benchmark.pyi @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete from pyarrow.lib import benchmark_PandasObjectIsNull as benchmark_PandasObjectIsNull diff --git a/python/pyarrow-stubs/pyarrow/cffi.pyi b/python/pyarrow-stubs/pyarrow/cffi.pyi index 69456bd3fc53..8dac34060d11 100644 --- a/python/pyarrow-stubs/pyarrow/cffi.pyi +++ b/python/pyarrow-stubs/pyarrow/cffi.pyi @@ -15,7 +15,10 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence from _typeshed import Incomplete - c_source: str ffi: Incomplete diff --git a/python/pyarrow-stubs/pyarrow/compute.pyi b/python/pyarrow-stubs/pyarrow/compute.pyi index f3232c6a4f49..246ed53633ad 100644 --- a/python/pyarrow-stubs/pyarrow/compute.pyi +++ b/python/pyarrow-stubs/pyarrow/compute.pyi @@ -15,308 +15,901 @@ # specific language governing permissions and limitations # under the License. +import os +import re +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence import pyarrow.lib from _typeshed import Incomplete from pyarrow._compute import ArraySortOptions as ArraySortOptions, AssumeTimezoneOptions as AssumeTimezoneOptions, CastOptions as CastOptions, CountOptions as CountOptions, CumulativeOptions as CumulativeOptions, CumulativeSumOptions as CumulativeSumOptions, DayOfWeekOptions as DayOfWeekOptions, DictionaryEncodeOptions as DictionaryEncodeOptions, ElementWiseAggregateOptions as ElementWiseAggregateOptions, Expression as Expression, ExtractRegexOptions as ExtractRegexOptions, ExtractRegexSpanOptions as ExtractRegexSpanOptions, FilterOptions as FilterOptions, Function as Function, FunctionOptions as FunctionOptions, FunctionRegistry as FunctionRegistry, HashAggregateFunction as HashAggregateFunction, HashAggregateKernel as HashAggregateKernel, IndexOptions as IndexOptions, InversePermutationOptions as InversePermutationOptions, JoinOptions as JoinOptions, Kernel as Kernel, ListFlattenOptions as ListFlattenOptions, ListSliceOptions as ListSliceOptions, MakeStructOptions as MakeStructOptions, MapLookupOptions as MapLookupOptions, MatchSubstringOptions as MatchSubstringOptions, ModeOptions as ModeOptions, NullOptions as NullOptions, PadOptions as PadOptions, PairwiseOptions as PairwiseOptions, PartitionNthOptions as PartitionNthOptions, PivotWiderOptions as PivotWiderOptions, QuantileOptions as QuantileOptions, RandomOptions as RandomOptions, RankOptions as RankOptions, RankQuantileOptions as RankQuantileOptions, ReplaceSliceOptions as ReplaceSliceOptions, ReplaceSubstringOptions as ReplaceSubstringOptions, RoundBinaryOptions as RoundBinaryOptions, RoundOptions as RoundOptions, RoundTemporalOptions as RoundTemporalOptions, RoundToMultipleOptions as RoundToMultipleOptions, RunEndEncodeOptions as RunEndEncodeOptions, ScalarAggregateFunction as ScalarAggregateFunction, ScalarAggregateKernel as ScalarAggregateKernel, ScalarAggregateOptions as ScalarAggregateOptions, ScalarFunction as ScalarFunction, ScalarKernel as ScalarKernel, ScatterOptions as ScatterOptions, SearchSortedOptions as SearchSortedOptions, SelectKOptions as SelectKOptions, SetLookupOptions as SetLookupOptions, SkewOptions as SkewOptions, SliceOptions as SliceOptions, SortOptions as SortOptions, SplitOptions as SplitOptions, SplitPatternOptions as SplitPatternOptions, StrftimeOptions as StrftimeOptions, StrptimeOptions as StrptimeOptions, StructFieldOptions as StructFieldOptions, TDigestOptions as TDigestOptions, TakeOptions as TakeOptions, TrimOptions as TrimOptions, UdfContext as UdfContext, Utf8NormalizeOptions as Utf8NormalizeOptions, VarianceOptions as VarianceOptions, VectorFunction as VectorFunction, VectorKernel as VectorKernel, WeekOptions as WeekOptions, WinsorizeOptions as WinsorizeOptions, ZeroFillOptions as ZeroFillOptions, call_function as call_function, call_tabular_function as call_tabular_function, function_registry as function_registry, get_function as get_function, list_functions as list_functions, register_aggregate_function as register_aggregate_function, register_scalar_function as register_scalar_function, register_tabular_function as register_tabular_function, register_vector_function as register_vector_function from typing import ClassVar +def abs(x, *, memory_pool: Incomplete | None=...): + ... + +def abs_checked(x, *, memory_pool: Incomplete | None=...): + ... + +def acos(x, *, memory_pool: Incomplete | None=...): + ... + +def acos_checked(x, *, memory_pool: Incomplete | None=...): + ... + +def acosh(x, *, memory_pool: Incomplete | None=...): + ... + +def acosh_checked(x, *, memory_pool: Incomplete | None=...): + ... + +def add(x, y, *, memory_pool: Incomplete | None=...): + ... + +def add_checked(x, y, *, memory_pool: Incomplete | None=...): + ... + +def all(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def and_(x, y, *, memory_pool: Incomplete | None=...): + ... + +def and_kleene(x, y, *, memory_pool: Incomplete | None=...): + ... + +def and_not(x, y, *, memory_pool: Incomplete | None=...): + ... + +def and_not_kleene(x, y, *, memory_pool: Incomplete | None=...): + ... + +def any(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def approximate_median(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def array_filter(array, selection_filter, null_selection_behavior: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def array_sort_indices(array, order: str=..., *, null_placement: str=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def array_take(array, indices, *, boundscheck: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def ascii_capitalize(strings, *, memory_pool: Incomplete | None=...): + ... + +def ascii_center(strings, width: int, padding: str=..., lean_left_on_odd_padding: bool=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def ascii_is_alnum(strings, *, memory_pool: Incomplete | None=...): + ... + +def ascii_is_alpha(strings, *, memory_pool: Incomplete | None=...): + ... + +def ascii_is_decimal(strings, *, memory_pool: Incomplete | None=...): + ... + +def ascii_is_lower(strings, *, memory_pool: Incomplete | None=...): + ... + +def ascii_is_printable(strings, *, memory_pool: Incomplete | None=...): + ... + +def ascii_is_space(strings, *, memory_pool: Incomplete | None=...): + ... + +def ascii_is_title(strings, *, memory_pool: Incomplete | None=...): + ... + +def ascii_is_upper(strings, *, memory_pool: Incomplete | None=...): + ... + +def ascii_lower(strings, *, memory_pool: Incomplete | None=...): + ... + +def ascii_lpad(strings, width: int, padding: str=..., lean_left_on_odd_padding: bool=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def ascii_ltrim(strings, characters: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def ascii_ltrim_whitespace(strings, *, memory_pool: Incomplete | None=...): + ... + +def ascii_reverse(strings, *, memory_pool: Incomplete | None=...): + ... + +def ascii_rpad(strings, width: int, padding: str=..., lean_left_on_odd_padding: bool=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def ascii_rtrim(strings, characters: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def ascii_rtrim_whitespace(strings, *, memory_pool: Incomplete | None=...): + ... + +def ascii_split_whitespace(strings, *, max_splits: Incomplete | None=..., reverse: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def ascii_swapcase(strings, *, memory_pool: Incomplete | None=...): + ... + +def ascii_title(strings, *, memory_pool: Incomplete | None=...): + ... + +def ascii_trim(strings, characters: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def ascii_trim_whitespace(strings, *, memory_pool: Incomplete | None=...): + ... + +def ascii_upper(strings, *, memory_pool: Incomplete | None=...): + ... + +def asin(x, *, memory_pool: Incomplete | None=...): + ... + +def asin_checked(x, *, memory_pool: Incomplete | None=...): + ... + +def asinh(x, *, memory_pool: Incomplete | None=...): + ... + +def assume_timezone(timestamps, timezone: str, *, ambiguous: str=..., nonexistent: str=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def atan(x, *, memory_pool: Incomplete | None=...): + ... + +def atan2(y, x, *, memory_pool: Incomplete | None=...): + ... + +def atanh(x, *, memory_pool: Incomplete | None=...): + ... + +def atanh_checked(x, *, memory_pool: Incomplete | None=...): + ... + +def binary_join(strings, separator, *, memory_pool: Incomplete | None=...): + ... + +def binary_join_element_wise(*strings, null_handling: str=..., null_replacement: str=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def binary_length(strings, *, memory_pool: Incomplete | None=...): + ... + +def binary_repeat(strings, num_repeats, *, memory_pool: Incomplete | None=...): + ... + +def binary_replace_slice(strings, start: int, stop: int, replacement: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def binary_reverse(strings, *, memory_pool: Incomplete | None=...): + ... + +def binary_slice(strings, start: int, stop: Incomplete | None=..., step: int=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def bit_wise_and(x, y, *, memory_pool: Incomplete | None=...): + ... + +def bit_wise_not(x, *, memory_pool: Incomplete | None=...): + ... + +def bit_wise_or(x, y, *, memory_pool: Incomplete | None=...): + ... + +def bit_wise_xor(x, y, *, memory_pool: Incomplete | None=...): + ... + +def case_when(cond, *cases, memory_pool: Incomplete | None=...): + ... + +def cast(arr, target_type: Incomplete | None=..., safe: Incomplete | None=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...) -> pyarrow.Array: + ... + +def ceil(x, *, memory_pool: Incomplete | None=...): + ... + +def ceil_temporal(timestamps, multiple: int=..., unit: str=..., *, week_starts_monday: bool=..., ceil_is_strictly_greater: bool=..., calendar_based_origin: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def choose(indices, *values, memory_pool: Incomplete | None=...): + ... + +def coalesce(*values, memory_pool: Incomplete | None=...): + ... + +def cos(x, *, memory_pool: Incomplete | None=...): + ... + +def cos_checked(x, *, memory_pool: Incomplete | None=...): + ... + +def cosh(x, *, memory_pool: Incomplete | None=...): + ... + +def count(array, mode: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def count_distinct(array, mode: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def count_substring(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def count_substring_regex(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def cumulative_max(values, start: Incomplete | None=..., *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def cumulative_mean(values, start: Incomplete | None=..., *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def cumulative_min(values, start: Incomplete | None=..., *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def cumulative_prod(values, start: Incomplete | None=..., *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def cumulative_prod_checked(values, start: Incomplete | None=..., *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def cumulative_sum(values, start: Incomplete | None=..., *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def cumulative_sum_checked(values, start: Incomplete | None=..., *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def day(values, *, memory_pool: Incomplete | None=...): + ... + +def day_of_week(values, *, count_from_zero: bool=..., week_start: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def day_of_year(values, *, memory_pool: Incomplete | None=...): + ... + +def day_time_interval_between(start, end, *, memory_pool: Incomplete | None=...): + ... + +def days_between(start, end, *, memory_pool: Incomplete | None=...): + ... + +def dictionary_decode(dictionary_array, *, memory_pool: Incomplete | None=...): + ... + +def dictionary_encode(array, null_encoding: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def divide(dividend, divisor, *, memory_pool: Incomplete | None=...): + ... + +def divide_checked(dividend, divisor, *, memory_pool: Incomplete | None=...): + ... + +def drop_null(input, *, memory_pool: Incomplete | None=...): + ... + +def ends_with(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def equal(x, y, *, memory_pool: Incomplete | None=...): + ... + +def exp(exponent, *, memory_pool: Incomplete | None=...): + ... + +def expm1(exponent, *, memory_pool: Incomplete | None=...): + ... + +def extract_regex(strings, pattern: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def extract_regex_span(strings, pattern: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def fill_null_backward(values, *, memory_pool: Incomplete | None=...): + ... + +def fill_null_forward(values, *, memory_pool: Incomplete | None=...): + ... + +def filter(input, selection_filter, null_selection_behavior: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def find_substring(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def find_substring_regex(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def first(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def first_last(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def floor(x, *, memory_pool: Incomplete | None=...): + ... + +def floor_temporal(timestamps, multiple: int=..., unit: str=..., *, week_starts_monday: bool=..., ceil_is_strictly_greater: bool=..., calendar_based_origin: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def greater(x, y, *, memory_pool: Incomplete | None=...): + ... + +def greater_equal(x, y, *, memory_pool: Incomplete | None=...): + ... + +def hour(values, *, memory_pool: Incomplete | None=...): + ... + +def hours_between(start, end, *, memory_pool: Incomplete | None=...): + ... + +def hypot(x, y, *, memory_pool: Incomplete | None=...): + ... + +def if_else(cond, left, right, *, memory_pool: Incomplete | None=...): + ... + +def index(data, value, start: Incomplete | None=..., end: Incomplete | None=..., *, memory_pool: Incomplete | None=...) -> int: + ... + +def index_in(values, value_set: pyarrow.Array, *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def index_in_meta_binary(values, value_set, *, memory_pool: Incomplete | None=...): + ... + +def indices_nonzero(values, *, memory_pool: Incomplete | None=...): + ... + +def inverse_permutation(indices, max_index: int=..., output_type: Incomplete | None=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def invert(values, *, memory_pool: Incomplete | None=...): + ... + +def is_dst(values, *, memory_pool: Incomplete | None=...): + ... + +def is_finite(values, *, memory_pool: Incomplete | None=...): + ... + +def is_in(values, value_set: pyarrow.Array, *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def is_in_meta_binary(values, value_set, *, memory_pool: Incomplete | None=...): + ... + +def is_inf(values, *, memory_pool: Incomplete | None=...): + ... + +def is_leap_year(values, *, memory_pool: Incomplete | None=...): + ... + +def is_nan(values, *, memory_pool: Incomplete | None=...): + ... + +def is_null(values, *, nan_is_null: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def is_valid(values, *, memory_pool: Incomplete | None=...): + ... + +def iso_calendar(values, *, memory_pool: Incomplete | None=...): + ... + +def iso_week(values, *, memory_pool: Incomplete | None=...): + ... + +def iso_year(values, *, memory_pool: Incomplete | None=...): + ... + +def kurtosis(array, *, skip_nulls: bool=..., biased: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def last(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def less(x, y, *, memory_pool: Incomplete | None=...): + ... + +def less_equal(x, y, *, memory_pool: Incomplete | None=...): + ... + +def list_element(lists, index, *, memory_pool: Incomplete | None=...): + ... + +def list_flatten(lists, recursive: bool=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def list_parent_indices(lists, *, memory_pool: Incomplete | None=...): + ... + +def list_slice(lists, start: int, stop: Incomplete | None=..., step: int=..., return_fixed_size_list: Incomplete | None=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def list_value_length(lists, *, memory_pool: Incomplete | None=...): + ... + +def ln(x, *, memory_pool: Incomplete | None=...): + ... + +def ln_checked(x, *, memory_pool: Incomplete | None=...): + ... + +def local_timestamp(values, *, memory_pool: Incomplete | None=...): + ... + +def log10(x, *, memory_pool: Incomplete | None=...): + ... + +def log10_checked(x, *, memory_pool: Incomplete | None=...): + ... + +def log1p(x, *, memory_pool: Incomplete | None=...): + ... + +def log1p_checked(x, *, memory_pool: Incomplete | None=...): + ... + +def log2(x, *, memory_pool: Incomplete | None=...): + ... + +def log2_checked(x, *, memory_pool: Incomplete | None=...): + ... + +def logb(x, b, *, memory_pool: Incomplete | None=...): + ... + +def logb_checked(x, b, *, memory_pool: Incomplete | None=...): + ... + +def make_struct(*args, field_names: tuple=..., field_nullability: Incomplete | None=..., field_metadata: Incomplete | None=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def map_lookup(container, query_key, occurrence: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def match_like(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def match_substring(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def match_substring_regex(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def max(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def max_element_wise(*args, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def mean(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def microsecond(values, *, memory_pool: Incomplete | None=...): + ... + +def microseconds_between(start, end, *, memory_pool: Incomplete | None=...): + ... + +def millisecond(values, *, memory_pool: Incomplete | None=...): + ... + +def milliseconds_between(start, end, *, memory_pool: Incomplete | None=...): + ... + +def min(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def min_element_wise(*args, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def min_max(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def minute(values, *, memory_pool: Incomplete | None=...): + ... + +def minutes_between(start, end, *, memory_pool: Incomplete | None=...): + ... + +def mode(array, n: int=..., *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def modulo(dividend, divisor, *, memory_pool: Incomplete | None=...): + ... + +def modulo_checked(dividend, divisor, *, memory_pool: Incomplete | None=...): + ... + +def month(values, *, memory_pool: Incomplete | None=...): + ... + +def month_day_nano_interval_between(start, end, *, memory_pool: Incomplete | None=...): + ... + +def month_interval_between(start, end, *, memory_pool: Incomplete | None=...): + ... + +def multiply(x, y, *, memory_pool: Incomplete | None=...): + ... + +def multiply_checked(x, y, *, memory_pool: Incomplete | None=...): + ... + +def nanosecond(values, *, memory_pool: Incomplete | None=...): + ... + +def nanoseconds_between(start, end, *, memory_pool: Incomplete | None=...): + ... + +def negate(x, *, memory_pool: Incomplete | None=...): + ... + +def negate_checked(x, *, memory_pool: Incomplete | None=...): + ... + +def not_equal(x, y, *, memory_pool: Incomplete | None=...): + ... + +def or_(x, y, *, memory_pool: Incomplete | None=...): + ... + +def or_kleene(x, y, *, memory_pool: Incomplete | None=...): + ... + +def pairwise_diff(input, period: int=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def pairwise_diff_checked(input, period: int=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def partition_nth_indices(array, pivot: int, *, null_placement: str=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def pivot_wider(pivot_keys, pivot_values, key_names: Sequence[str], *, unexpected_key_behavior: str=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def power(base, exponent, *, memory_pool: Incomplete | None=...): + ... + +def power_checked(base, exponent, *, memory_pool: Incomplete | None=...): + ... + +def product(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def quantile(array, q: float=..., *, interpolation: str=..., skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def quarter(values, *, memory_pool: Incomplete | None=...): + ... + +def quarters_between(start, end, *, memory_pool: Incomplete | None=...): + ... + +def random(n: int, *, initializer: str=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def rank(input, sort_keys: str=..., *, null_placement: Incomplete | None=..., tiebreaker: str=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def rank_normal(input, sort_keys: str=..., *, null_placement: Incomplete | None=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def rank_quantile(input, sort_keys: str=..., *, null_placement: Incomplete | None=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def remainder(dividend, divisor, *, memory_pool: Incomplete | None=...): + ... + +def remainder_checked(dividend, divisor, *, memory_pool: Incomplete | None=...): + ... + +def replace_substring(strings, pattern: str, replacement: str, *, max_replacements: Incomplete | None=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def replace_substring_regex(strings, pattern: str, replacement: str, *, max_replacements: Incomplete | None=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def replace_with_mask(values, mask, replacements, *, memory_pool: Incomplete | None=...): + ... + +def round(x, ndigits: int=..., round_mode: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def round_binary(x, s, round_mode: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def round_temporal(timestamps, multiple: int=..., unit: str=..., *, week_starts_monday: bool=..., ceil_is_strictly_greater: bool=..., calendar_based_origin: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def round_to_multiple(x, multiple: float=..., round_mode: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def run_end_decode(array, *, memory_pool: Incomplete | None=...): + ... + +def run_end_encode(array, run_end_type: pyarrow.lib.DataType=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def scatter(values, indices, max_index: int=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def search_sorted(values, needles, side: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def second(values, *, memory_pool: Incomplete | None=...): + ... + +def seconds_between(start, end, *, memory_pool: Incomplete | None=...): + ... + +def select_k_unstable(input, k: int, sort_keys, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def shift_left(x, y, *, memory_pool: Incomplete | None=...): + ... + +def shift_left_checked(x, y, *, memory_pool: Incomplete | None=...): + ... + +def shift_right(x, y, *, memory_pool: Incomplete | None=...): + ... + +def shift_right_checked(x, y, *, memory_pool: Incomplete | None=...): + ... + +def sign(x, *, memory_pool: Incomplete | None=...): + ... + +def sin(x, *, memory_pool: Incomplete | None=...): + ... + +def sin_checked(x, *, memory_pool: Incomplete | None=...): + ... + +def sinh(x, *, memory_pool: Incomplete | None=...): + ... + +def skew(array, *, skip_nulls: bool=..., biased: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def sort_indices(input, sort_keys: tuple=..., *, null_placement: Incomplete | None=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def split_pattern(strings, pattern: str, *, max_splits: Incomplete | None=..., reverse: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def split_pattern_regex(strings, pattern: str, *, max_splits: Incomplete | None=..., reverse: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def sqrt(x, *, memory_pool: Incomplete | None=...): + ... + +def sqrt_checked(x, *, memory_pool: Incomplete | None=...): + ... + +def starts_with(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def stddev(array, *, ddof: int=..., skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def strftime(timestamps, format: str=..., locale: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def string_is_ascii(strings, *, memory_pool: Incomplete | None=...): + ... + +def strptime(strings, format: str, unit: str, error_is_null: bool=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def struct_field(values, indices: list[str] | list[bytes] | list[int] | Expression | bytes | str | int, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def subsecond(values, *, memory_pool: Incomplete | None=...): + ... + +def subtract(x, y, *, memory_pool: Incomplete | None=...): + ... + +def subtract_checked(x, y, *, memory_pool: Incomplete | None=...): + ... + +def sum(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def take(data: pyarrow.Array | pyarrow.ChunkedArray | pyarrow.RecordBatch | pyarrow.Table, indices: pyarrow.Array | pyarrow.ChunkedArray, *, boundscheck: bool=..., memory_pool: Incomplete | None=...): + ... + +def tan(x, *, memory_pool: Incomplete | None=...): + ... + +def tan_checked(x, *, memory_pool: Incomplete | None=...): + ... + +def tanh(x, *, memory_pool: Incomplete | None=...): + ... + +def tdigest(array, q: float=..., *, delta: int=..., buffer_size: int=..., skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def true_unless_null(values, *, memory_pool: Incomplete | None=...): + ... + +def trunc(x, *, memory_pool: Incomplete | None=...): + ... + +def unique(array, *, memory_pool: Incomplete | None=...): + ... + +def us_week(values, *, memory_pool: Incomplete | None=...): + ... + +def us_year(values, *, memory_pool: Incomplete | None=...): + ... + +def utf8_capitalize(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_center(strings, width: int, padding: str=..., lean_left_on_odd_padding: bool=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def utf8_is_alnum(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_is_alpha(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_is_decimal(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_is_digit(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_is_lower(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_is_numeric(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_is_printable(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_is_space(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_is_title(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_is_upper(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_length(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_lower(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_lpad(strings, width: int, padding: str=..., lean_left_on_odd_padding: bool=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def utf8_ltrim(strings, characters: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def utf8_ltrim_whitespace(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_normalize(strings, form: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def utf8_replace_slice(strings, start: int, stop: int, replacement: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def utf8_reverse(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_rpad(strings, width: int, padding: str=..., lean_left_on_odd_padding: bool=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def utf8_rtrim(strings, characters: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def utf8_rtrim_whitespace(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_slice_codeunits(strings, start: int, stop: Incomplete | None=..., step: int=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def utf8_split_whitespace(strings, *, max_splits: Incomplete | None=..., reverse: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def utf8_swapcase(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_title(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_trim(strings, characters: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def utf8_trim_whitespace(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_upper(strings, *, memory_pool: Incomplete | None=...): + ... + +def utf8_zero_fill(strings, width: int, padding: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def value_counts(array, *, memory_pool: Incomplete | None=...): + ... + +def variance(array, *, ddof: int=..., skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def week(values, *, week_starts_monday: bool=..., count_from_zero: bool=..., first_week_is_fully_in_year: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def weeks_between(start, end, *, count_from_zero: bool=..., week_start: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def winsorize(array, lower_limit, upper_limit, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def xor(x, y, *, memory_pool: Incomplete | None=...): + ... + +def year(values, *, memory_pool: Incomplete | None=...): + ... + +def year_month_day(values, *, memory_pool: Incomplete | None=...): + ... + +def years_between(start, end, *, memory_pool: Incomplete | None=...): + ... + +def utf8_zfill(strings, width: int, padding: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): + ... + +def fill_null(values, fill_value): + ... + +def top_k_unstable(values: pyarrow.Array | pyarrow.ChunkedArray | pyarrow.RecordBatch | pyarrow.Table, k: int, sort_keys: Incomplete | None=..., null_placements: Incomplete | None=..., *, memory_pool: Incomplete | None=...) -> pyarrow.Array: + ... + +def bottom_k_unstable(values: pyarrow.Array | pyarrow.ChunkedArray | pyarrow.RecordBatch | pyarrow.Table, k: int, sort_keys: Incomplete | None=..., null_placements: Incomplete | None=..., *, memory_pool: Incomplete | None=...): + ... + +def field(*name_or_index) -> Expression: + ... + +def scalar(value: bool | int | float | str) -> Expression: + ... + class _OptionsClassDoc(tuple): _fields: ClassVar[tuple] = ... _field_defaults: ClassVar[dict] = ... __match_args__: ClassVar[tuple] = ... params: Incomplete - def __init__(self, _cls, params) -> None: ... - def __replace__(self, **kwds): ... - def __getnewargs__(self): ... -def abs(x, *, memory_pool: Incomplete | None = ...): ... -def abs_checked(x, *, memory_pool: Incomplete | None = ...): ... -def acos(x, *, memory_pool: Incomplete | None = ...): ... -def acos_checked(x, *, memory_pool: Incomplete | None = ...): ... -def acosh(x, *, memory_pool: Incomplete | None = ...): ... -def acosh_checked(x, *, memory_pool: Incomplete | None = ...): ... -def add(x, y, *, memory_pool: Incomplete | None = ...): ... -def add_checked(x, y, *, memory_pool: Incomplete | None = ...): ... -def all(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def and_(x, y, *, memory_pool: Incomplete | None = ...): ... -def and_kleene(x, y, *, memory_pool: Incomplete | None = ...): ... -def and_not(x, y, *, memory_pool: Incomplete | None = ...): ... -def and_not_kleene(x, y, *, memory_pool: Incomplete | None = ...): ... -def any(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def approximate_median(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def array_filter(array, selection_filter, null_selection_behavior: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def array_sort_indices(array, order: str = ..., *, null_placement: str = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def array_take(array, indices, *, boundscheck: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def ascii_capitalize(strings, *, memory_pool: Incomplete | None = ...): ... -def ascii_center(strings, width, padding: str = ..., lean_left_on_odd_padding: bool = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def ascii_is_alnum(strings, *, memory_pool: Incomplete | None = ...): ... -def ascii_is_alpha(strings, *, memory_pool: Incomplete | None = ...): ... -def ascii_is_decimal(strings, *, memory_pool: Incomplete | None = ...): ... -def ascii_is_lower(strings, *, memory_pool: Incomplete | None = ...): ... -def ascii_is_printable(strings, *, memory_pool: Incomplete | None = ...): ... -def ascii_is_space(strings, *, memory_pool: Incomplete | None = ...): ... -def ascii_is_title(strings, *, memory_pool: Incomplete | None = ...): ... -def ascii_is_upper(strings, *, memory_pool: Incomplete | None = ...): ... -def ascii_lower(strings, *, memory_pool: Incomplete | None = ...): ... -def ascii_lpad(strings, width, padding: str = ..., lean_left_on_odd_padding: bool = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def ascii_ltrim(strings, characters, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def ascii_ltrim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... -def ascii_reverse(strings, *, memory_pool: Incomplete | None = ...): ... -def ascii_rpad(strings, width, padding: str = ..., lean_left_on_odd_padding: bool = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def ascii_rtrim(strings, characters, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def ascii_rtrim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... -def ascii_split_whitespace(strings, *, max_splits: Incomplete | None = ..., reverse: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def ascii_swapcase(strings, *, memory_pool: Incomplete | None = ...): ... -def ascii_title(strings, *, memory_pool: Incomplete | None = ...): ... -def ascii_trim(strings, characters, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def ascii_trim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... -def ascii_upper(strings, *, memory_pool: Incomplete | None = ...): ... -def asin(x, *, memory_pool: Incomplete | None = ...): ... -def asin_checked(x, *, memory_pool: Incomplete | None = ...): ... -def asinh(x, *, memory_pool: Incomplete | None = ...): ... -def assume_timezone(timestamps, timezone, *, ambiguous: str = ..., nonexistent: str = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def atan(x, *, memory_pool: Incomplete | None = ...): ... -def atan2(y, x, *, memory_pool: Incomplete | None = ...): ... -def atanh(x, *, memory_pool: Incomplete | None = ...): ... -def atanh_checked(x, *, memory_pool: Incomplete | None = ...): ... -def binary_join(strings, separator, *, memory_pool: Incomplete | None = ...): ... -def binary_join_element_wise(*strings, null_handling: str = ..., null_replacement: str = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def binary_length(strings, *, memory_pool: Incomplete | None = ...): ... -def binary_repeat(strings, num_repeats, *, memory_pool: Incomplete | None = ...): ... -def binary_replace_slice(strings, start, stop, replacement, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def binary_reverse(strings, *, memory_pool: Incomplete | None = ...): ... -def binary_slice(strings, start, stop: Incomplete | None = ..., step: int = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def bit_wise_and(x, y, *, memory_pool: Incomplete | None = ...): ... -def bit_wise_not(x, *, memory_pool: Incomplete | None = ...): ... -def bit_wise_or(x, y, *, memory_pool: Incomplete | None = ...): ... -def bit_wise_xor(x, y, *, memory_pool: Incomplete | None = ...): ... -def case_when(cond, *cases, memory_pool: Incomplete | None = ...): ... -def cast(arr, target_type: Incomplete | None = ..., safe: Incomplete | None = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def ceil(x, *, memory_pool: Incomplete | None = ...): ... -def ceil_temporal(timestamps, multiple: int = ..., unit: str = ..., *, week_starts_monday: bool = ..., ceil_is_strictly_greater: bool = ..., calendar_based_origin: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def choose(indices, *values, memory_pool: Incomplete | None = ...): ... -def coalesce(*values, memory_pool: Incomplete | None = ...): ... -def cos(x, *, memory_pool: Incomplete | None = ...): ... -def cos_checked(x, *, memory_pool: Incomplete | None = ...): ... -def cosh(x, *, memory_pool: Incomplete | None = ...): ... -def count(array, mode: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def count_distinct(array, mode: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def count_substring(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def count_substring_regex(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def cumulative_max(values, start: Incomplete | None = ..., *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def cumulative_mean(values, start: Incomplete | None = ..., *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def cumulative_min(values, start: Incomplete | None = ..., *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def cumulative_prod(values, start: Incomplete | None = ..., *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def cumulative_prod_checked(values, start: Incomplete | None = ..., *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def cumulative_sum(values, start: Incomplete | None = ..., *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def cumulative_sum_checked(values, start: Incomplete | None = ..., *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def day(values, *, memory_pool: Incomplete | None = ...): ... -def day_of_week(values, *, count_from_zero: bool = ..., week_start: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def day_of_year(values, *, memory_pool: Incomplete | None = ...): ... -def day_time_interval_between(start, end, *, memory_pool: Incomplete | None = ...): ... -def days_between(start, end, *, memory_pool: Incomplete | None = ...): ... -def dictionary_decode(dictionary_array, *, memory_pool: Incomplete | None = ...): ... -def dictionary_encode(array, null_encoding: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def divide(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... -def divide_checked(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... -def drop_null(input, *, memory_pool: Incomplete | None = ...): ... -def ends_with(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def equal(x, y, *, memory_pool: Incomplete | None = ...): ... -def exp(exponent, *, memory_pool: Incomplete | None = ...): ... -def expm1(exponent, *, memory_pool: Incomplete | None = ...): ... -def extract_regex(strings, pattern, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def extract_regex_span(strings, pattern, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def fill_null_backward(values, *, memory_pool: Incomplete | None = ...): ... -def fill_null_forward(values, *, memory_pool: Incomplete | None = ...): ... -def filter(input, selection_filter, null_selection_behavior: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def find_substring(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def find_substring_regex(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def first(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def first_last(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def floor(x, *, memory_pool: Incomplete | None = ...): ... -def floor_temporal(timestamps, multiple: int = ..., unit: str = ..., *, week_starts_monday: bool = ..., ceil_is_strictly_greater: bool = ..., calendar_based_origin: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def greater(x, y, *, memory_pool: Incomplete | None = ...): ... -def greater_equal(x, y, *, memory_pool: Incomplete | None = ...): ... -def hour(values, *, memory_pool: Incomplete | None = ...): ... -def hours_between(start, end, *, memory_pool: Incomplete | None = ...): ... -def hypot(x, y, *, memory_pool: Incomplete | None = ...): ... -def if_else(cond, left, right, *, memory_pool: Incomplete | None = ...): ... -def index(data, value, start: Incomplete | None = ..., end: Incomplete | None = ..., *, memory_pool: Incomplete | None = ...): ... -def index_in(values, value_set, *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def index_in_meta_binary(values, value_set, *, memory_pool: Incomplete | None = ...): ... -def indices_nonzero(values, *, memory_pool: Incomplete | None = ...): ... -def inverse_permutation(indices, max_index: int = ..., output_type: Incomplete | None = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def invert(values, *, memory_pool: Incomplete | None = ...): ... -def is_dst(values, *, memory_pool: Incomplete | None = ...): ... -def is_finite(values, *, memory_pool: Incomplete | None = ...): ... -def is_in(values, value_set, *, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def is_in_meta_binary(values, value_set, *, memory_pool: Incomplete | None = ...): ... -def is_inf(values, *, memory_pool: Incomplete | None = ...): ... -def is_leap_year(values, *, memory_pool: Incomplete | None = ...): ... -def is_nan(values, *, memory_pool: Incomplete | None = ...): ... -def is_null(values, *, nan_is_null: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def is_valid(values, *, memory_pool: Incomplete | None = ...): ... -def iso_calendar(values, *, memory_pool: Incomplete | None = ...): ... -def iso_week(values, *, memory_pool: Incomplete | None = ...): ... -def iso_year(values, *, memory_pool: Incomplete | None = ...): ... -def kurtosis(array, *, skip_nulls: bool = ..., biased: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def last(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def less(x, y, *, memory_pool: Incomplete | None = ...): ... -def less_equal(x, y, *, memory_pool: Incomplete | None = ...): ... -def list_element(lists, index, *, memory_pool: Incomplete | None = ...): ... -def list_flatten(lists, recursive: bool = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def list_parent_indices(lists, *, memory_pool: Incomplete | None = ...): ... -def list_slice(lists, start, stop: Incomplete | None = ..., step: int = ..., return_fixed_size_list: Incomplete | None = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def list_value_length(lists, *, memory_pool: Incomplete | None = ...): ... -def ln(x, *, memory_pool: Incomplete | None = ...): ... -def ln_checked(x, *, memory_pool: Incomplete | None = ...): ... -def local_timestamp(values, *, memory_pool: Incomplete | None = ...): ... -def log10(x, *, memory_pool: Incomplete | None = ...): ... -def log10_checked(x, *, memory_pool: Incomplete | None = ...): ... -def log1p(x, *, memory_pool: Incomplete | None = ...): ... -def log1p_checked(x, *, memory_pool: Incomplete | None = ...): ... -def log2(x, *, memory_pool: Incomplete | None = ...): ... -def log2_checked(x, *, memory_pool: Incomplete | None = ...): ... -def logb(x, b, *, memory_pool: Incomplete | None = ...): ... -def logb_checked(x, b, *, memory_pool: Incomplete | None = ...): ... -def make_struct(*args, field_names: tuple = ..., field_nullability: Incomplete | None = ..., field_metadata: Incomplete | None = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def map_lookup(container, query_key, occurrence, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def match_like(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def match_substring(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def match_substring_regex(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def max(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def max_element_wise(*args, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def mean(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def microsecond(values, *, memory_pool: Incomplete | None = ...): ... -def microseconds_between(start, end, *, memory_pool: Incomplete | None = ...): ... -def millisecond(values, *, memory_pool: Incomplete | None = ...): ... -def milliseconds_between(start, end, *, memory_pool: Incomplete | None = ...): ... -def min(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def min_element_wise(*args, skip_nulls: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def min_max(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def minute(values, *, memory_pool: Incomplete | None = ...): ... -def minutes_between(start, end, *, memory_pool: Incomplete | None = ...): ... -def mode(array, n: int = ..., *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def modulo(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... -def modulo_checked(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... -def month(values, *, memory_pool: Incomplete | None = ...): ... -def month_day_nano_interval_between(start, end, *, memory_pool: Incomplete | None = ...): ... -def month_interval_between(start, end, *, memory_pool: Incomplete | None = ...): ... -def multiply(x, y, *, memory_pool: Incomplete | None = ...): ... -def multiply_checked(x, y, *, memory_pool: Incomplete | None = ...): ... -def nanosecond(values, *, memory_pool: Incomplete | None = ...): ... -def nanoseconds_between(start, end, *, memory_pool: Incomplete | None = ...): ... -def negate(x, *, memory_pool: Incomplete | None = ...): ... -def negate_checked(x, *, memory_pool: Incomplete | None = ...): ... -def not_equal(x, y, *, memory_pool: Incomplete | None = ...): ... -def or_(x, y, *, memory_pool: Incomplete | None = ...): ... -def or_kleene(x, y, *, memory_pool: Incomplete | None = ...): ... -def pairwise_diff(input, period: int = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def pairwise_diff_checked(input, period: int = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def partition_nth_indices(array, pivot, *, null_placement: str = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def pivot_wider(pivot_keys, pivot_values, key_names, *, unexpected_key_behavior: str = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def power(base, exponent, *, memory_pool: Incomplete | None = ...): ... -def power_checked(base, exponent, *, memory_pool: Incomplete | None = ...): ... -def product(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def quantile(array, q: float = ..., *, interpolation: str = ..., skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def quarter(values, *, memory_pool: Incomplete | None = ...): ... -def quarters_between(start, end, *, memory_pool: Incomplete | None = ...): ... -def random(n, *, initializer: str = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def rank(input, sort_keys: str = ..., *, null_placement: Incomplete | None = ..., tiebreaker: str = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def rank_normal(input, sort_keys: str = ..., *, null_placement: Incomplete | None = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def rank_quantile(input, sort_keys: str = ..., *, null_placement: Incomplete | None = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def remainder(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... -def remainder_checked(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... -def replace_substring(strings, pattern, replacement, *, max_replacements: Incomplete | None = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def replace_substring_regex(strings, pattern, replacement, *, max_replacements: Incomplete | None = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def replace_with_mask(values, mask, replacements, *, memory_pool: Incomplete | None = ...): ... -def round(x, ndigits: int = ..., round_mode: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def round_binary(x, s, round_mode: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def round_temporal(timestamps, multiple: int = ..., unit: str = ..., *, week_starts_monday: bool = ..., ceil_is_strictly_greater: bool = ..., calendar_based_origin: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def round_to_multiple(x, multiple: float = ..., round_mode: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def run_end_decode(array, *, memory_pool: Incomplete | None = ...): ... -def run_end_encode(array, run_end_type: pyarrow.lib.DataType = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def scatter(values, indices, max_index: int = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def search_sorted(values, needles, side: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def second(values, *, memory_pool: Incomplete | None = ...): ... -def seconds_between(start, end, *, memory_pool: Incomplete | None = ...): ... -def select_k_unstable(input, k, sort_keys, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def shift_left(x, y, *, memory_pool: Incomplete | None = ...): ... -def shift_left_checked(x, y, *, memory_pool: Incomplete | None = ...): ... -def shift_right(x, y, *, memory_pool: Incomplete | None = ...): ... -def shift_right_checked(x, y, *, memory_pool: Incomplete | None = ...): ... -def sign(x, *, memory_pool: Incomplete | None = ...): ... -def sin(x, *, memory_pool: Incomplete | None = ...): ... -def sin_checked(x, *, memory_pool: Incomplete | None = ...): ... -def sinh(x, *, memory_pool: Incomplete | None = ...): ... -def skew(array, *, skip_nulls: bool = ..., biased: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def sort_indices(input, sort_keys: tuple = ..., *, null_placement: Incomplete | None = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def split_pattern(strings, pattern, *, max_splits: Incomplete | None = ..., reverse: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def split_pattern_regex(strings, pattern, *, max_splits: Incomplete | None = ..., reverse: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def sqrt(x, *, memory_pool: Incomplete | None = ...): ... -def sqrt_checked(x, *, memory_pool: Incomplete | None = ...): ... -def starts_with(strings, pattern, *, ignore_case: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def stddev(array, *, ddof: int = ..., skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def strftime(timestamps, format: str = ..., locale: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def string_is_ascii(strings, *, memory_pool: Incomplete | None = ...): ... -def strptime(strings, format, unit, error_is_null: bool = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def struct_field(values, indices, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def subsecond(values, *, memory_pool: Incomplete | None = ...): ... -def subtract(x, y, *, memory_pool: Incomplete | None = ...): ... -def subtract_checked(x, y, *, memory_pool: Incomplete | None = ...): ... -def sum(array, *, skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def take(data, indices, *, boundscheck: bool = ..., memory_pool: Incomplete | None = ...): ... -def tan(x, *, memory_pool: Incomplete | None = ...): ... -def tan_checked(x, *, memory_pool: Incomplete | None = ...): ... -def tanh(x, *, memory_pool: Incomplete | None = ...): ... -def tdigest(array, q: float = ..., *, delta: int = ..., buffer_size: int = ..., skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def true_unless_null(values, *, memory_pool: Incomplete | None = ...): ... -def trunc(x, *, memory_pool: Incomplete | None = ...): ... -def unique(array, *, memory_pool: Incomplete | None = ...): ... -def us_week(values, *, memory_pool: Incomplete | None = ...): ... -def us_year(values, *, memory_pool: Incomplete | None = ...): ... -def utf8_capitalize(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_center(strings, width, padding: str = ..., lean_left_on_odd_padding: bool = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def utf8_is_alnum(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_is_alpha(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_is_decimal(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_is_digit(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_is_lower(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_is_numeric(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_is_printable(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_is_space(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_is_title(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_is_upper(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_length(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_lower(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_lpad(strings, width, padding: str = ..., lean_left_on_odd_padding: bool = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def utf8_ltrim(strings, characters, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def utf8_ltrim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_normalize(strings, form, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def utf8_replace_slice(strings, start, stop, replacement, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def utf8_reverse(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_rpad(strings, width, padding: str = ..., lean_left_on_odd_padding: bool = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def utf8_rtrim(strings, characters, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def utf8_rtrim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_slice_codeunits(strings, start, stop: Incomplete | None = ..., step: int = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def utf8_split_whitespace(strings, *, max_splits: Incomplete | None = ..., reverse: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def utf8_swapcase(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_title(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_trim(strings, characters, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def utf8_trim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_upper(strings, *, memory_pool: Incomplete | None = ...): ... -def utf8_zero_fill(strings, width, padding: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def value_counts(array, *, memory_pool: Incomplete | None = ...): ... -def variance(array, *, ddof: int = ..., skip_nulls: bool = ..., min_count: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def week(values, *, week_starts_monday: bool = ..., count_from_zero: bool = ..., first_week_is_fully_in_year: bool = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def weeks_between(start, end, *, count_from_zero: bool = ..., week_start: int = ..., options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def winsorize(array, lower_limit, upper_limit, *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def xor(x, y, *, memory_pool: Incomplete | None = ...): ... -def year(values, *, memory_pool: Incomplete | None = ...): ... -def year_month_day(values, *, memory_pool: Incomplete | None = ...): ... -def years_between(start, end, *, memory_pool: Incomplete | None = ...): ... -def utf8_zfill(strings, width, padding: str = ..., *, options: Incomplete | None = ..., memory_pool: Incomplete | None = ...): ... -def fill_null(values, fill_value): ... -def top_k_unstable(values, k, sort_keys: Incomplete | None = ..., null_placements: Incomplete | None = ..., *, memory_pool: Incomplete | None = ...): ... -def bottom_k_unstable(values, k, sort_keys: Incomplete | None = ..., null_placements: Incomplete | None = ..., *, memory_pool: Incomplete | None = ...): ... -def field(*name_or_index): ... -def scalar(value): ... + + def __init__(self, _cls, params) -> None: + ... + + def __replace__(self, **kwds): + ... + + def __getnewargs__(self): + ... diff --git a/python/pyarrow-stubs/pyarrow/csv.pyi b/python/pyarrow-stubs/pyarrow/csv.pyi index ead7186863df..8020c3bbcd5e 100644 --- a/python/pyarrow-stubs/pyarrow/csv.pyi +++ b/python/pyarrow-stubs/pyarrow/csv.pyi @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete from pyarrow._csv import CSVStreamingReader as CSVStreamingReader, CSVWriter as CSVWriter, ConvertOptions as ConvertOptions, ISO8601 as ISO8601, InvalidRow as InvalidRow, ParseOptions as ParseOptions, ReadOptions as ReadOptions, WriteOptions as WriteOptions, open_csv as open_csv, read_csv as read_csv, write_csv as write_csv diff --git a/python/pyarrow-stubs/pyarrow/dataset.pyi b/python/pyarrow-stubs/pyarrow/dataset.pyi index 8b3272168662..80d65fd8ec9a 100644 --- a/python/pyarrow-stubs/pyarrow/dataset.pyi +++ b/python/pyarrow-stubs/pyarrow/dataset.pyi @@ -15,28 +15,59 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete +import pyarrow.dataset +import pyarrow.fs from pyarrow._dataset import CsvFileFormat as CsvFileFormat, CsvFragmentScanOptions as CsvFragmentScanOptions, Dataset as Dataset, DatasetFactory as DatasetFactory, DirectoryPartitioning as DirectoryPartitioning, FeatherFileFormat as FeatherFileFormat, FileFormat as FileFormat, FileFragment as FileFragment, FileSystemDataset as FileSystemDataset, FileSystemDatasetFactory as FileSystemDatasetFactory, FileSystemFactoryOptions as FileSystemFactoryOptions, FileWriteOptions as FileWriteOptions, FilenamePartitioning as FilenamePartitioning, Fragment as Fragment, FragmentScanOptions as FragmentScanOptions, HivePartitioning as HivePartitioning, InMemoryDataset as InMemoryDataset, IpcFileFormat as IpcFileFormat, IpcFileWriteOptions as IpcFileWriteOptions, JsonFileFormat as JsonFileFormat, JsonFragmentScanOptions as JsonFragmentScanOptions, Partitioning as Partitioning, PartitioningFactory as PartitioningFactory, Scanner as Scanner, TaggedRecordBatch as TaggedRecordBatch, UnionDataset as UnionDataset, UnionDatasetFactory as UnionDatasetFactory, WrittenFile as WrittenFile, _filesystemdataset_write as _filesystemdataset_write, get_partition_keys as get_partition_keys from pyarrow._dataset_orc import OrcFileFormat as OrcFileFormat from pyarrow._dataset_parquet import ParquetDatasetFactory as ParquetDatasetFactory, ParquetFactoryOptions as ParquetFactoryOptions, ParquetFileFormat as ParquetFileFormat, ParquetFileFragment as ParquetFileFragment, ParquetFileWriteOptions as ParquetFileWriteOptions, ParquetFragmentScanOptions as ParquetFragmentScanOptions, ParquetReadOptions as ParquetReadOptions, RowGroupInfo as RowGroupInfo from pyarrow._dataset_parquet_encryption import ParquetDecryptionConfig as ParquetDecryptionConfig, ParquetEncryptionConfig as ParquetEncryptionConfig from pyarrow.compute import Expression as Expression, field as field, scalar as scalar from pyarrow.util import _is_iterable as _is_iterable, _is_path_like as _is_path_like, _stringify_path as _stringify_path - _orc_available: bool _orc_msg: str _parquet_available: bool _parquet_msg: str -def __getattr__(name) -> None: ... -def partitioning(schema=None, field_names=None, flavor=None, dictionaries=None): ... -def _ensure_partitioning(scheme): ... -def _ensure_format(obj): ... -def _ensure_multiple_sources(paths, filesystem=None): ... -def _ensure_single_source(path, filesystem=None): ... -def _filesystem_dataset(source, schema=None, filesystem=None, partitioning=None, format=None, partition_base_dir=None, exclude_invalid_files=None, selector_ignore_prefixes=None): ... -def _in_memory_dataset(source, schema=None, **kwargs): ... -def _union_dataset(children, schema=None, **kwargs): ... -def parquet_dataset(metadata_path, schema=None, filesystem=None, format=None, partitioning=None, partition_base_dir=None): ... -def dataset(source, schema=None, format=None, filesystem=None, partitioning=None, partition_base_dir=None, exclude_invalid_files=None, ignore_prefixes=None): ... -def _ensure_write_partitioning(part, schema, flavor): ... -def write_dataset(data, base_dir, *, basename_template=None, format=None, partitioning=None, partitioning_flavor=None, schema=None, filesystem=None, file_options=None, use_threads: bool = True, preserve_order: bool = False, max_partitions=None, max_open_files=None, max_rows_per_file=None, min_rows_per_group=None, max_rows_per_group=None, file_visitor=None, existing_data_behavior: str = 'error', create_dir: bool = True) -> None: ... +def __getattr__(name) -> None: + ... + +def partitioning(schema: pyarrow.Schema | None=None, field_names: list[str] | None=None, flavor: str | None=None, dictionaries=None) -> Partitioning | PartitioningFactory: + ... + +def _ensure_partitioning(scheme): + ... + +def _ensure_format(obj): + ... + +def _ensure_multiple_sources(paths: list[os.PathLike[str]], filesystem: pyarrow.fs.FileSystem | str | None=None): + ... + +def _ensure_single_source(path: os.PathLike[str], filesystem: pyarrow.fs.FileSystem | str | None=None): + ... + +def _filesystem_dataset(source, schema=None, filesystem=None, partitioning=None, format=None, partition_base_dir=None, exclude_invalid_files=None, selector_ignore_prefixes=None) -> FileSystemDataset: + ... + +def _in_memory_dataset(source, schema=None, **kwargs): + ... + +def _union_dataset(children, schema=None, **kwargs): + ... + +def parquet_dataset(metadata_path, schema: pyarrow.Schema | None=None, filesystem=None, format: ParquetFileFormat | None=None, partitioning: Partitioning | PartitioningFactory | str | list[str] | None=None, partition_base_dir: str | None=None) -> FileSystemDataset: + ... + +def dataset(source, schema: pyarrow.Schema | None=None, format: FileFormat | str | None=None, filesystem=None, partitioning: Partitioning | PartitioningFactory | str | list[str] | None=None, partition_base_dir: str | None=None, exclude_invalid_files: bool | None=None, ignore_prefixes: list | None=None) -> Dataset: + ... + +def _ensure_write_partitioning(part, schema, flavor): + ... + +def write_dataset(data, base_dir: str, *, basename_template: str | None=None, format: FileFormat | str | None=None, partitioning: Partitioning | list[str] | None=None, partitioning_flavor: str | None=None, schema: pyarrow.Schema | None=None, filesystem: pyarrow.fs.FileSystem | None=None, file_options: pyarrow.dataset.FileWriteOptions | None=None, use_threads: bool=True, preserve_order: bool=False, max_partitions: int | None=None, max_open_files: int | None=None, max_rows_per_file: int | None=None, min_rows_per_group: int | None=None, max_rows_per_group: int | None=None, file_visitor: Callable[..., Any] | None=None, existing_data_behavior: str='error', create_dir: bool=True) -> None: + ... diff --git a/python/pyarrow-stubs/pyarrow/feather.pyi b/python/pyarrow-stubs/pyarrow/feather.pyi index ea0053a5765b..cf741d5adc74 100644 --- a/python/pyarrow-stubs/pyarrow/feather.pyi +++ b/python/pyarrow-stubs/pyarrow/feather.pyi @@ -15,27 +15,46 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence from _typeshed import Incomplete from pyarrow import _feather as _feather from pyarrow._feather import FeatherError as FeatherError from pyarrow.lib import Codec as Codec, Table as Table, concat_tables as concat_tables, schema as schema from pyarrow.pandas_compat import _pandas_api as _pandas_api +def check_chunked_overflow(name, col) -> None: + ... +_FEATHER_SUPPORTED_CODECS: Incomplete + +def write_feather(df, dest: str, compression: str | None=None, compression_level: int | None=None, chunksize: int | None=None, version: int=2) -> None: + ... + +def read_feather(source, columns: Sequence[Any] | None=None, use_threads: bool=True, memory_map: bool=False, **kwargs): + ... + +def _read_table_internal(source, columns=None, memory_map: bool=False, use_threads: bool=True): + ... + +def read_table(source, columns: Sequence[Any] | None=None, memory_map: bool=False, use_threads: bool=True) -> Table: + ... + class FeatherDataset: paths: Incomplete validate_schema: Incomplete - def __init__(self, path_or_paths, validate_schema: bool = True) -> None: ... + + def __init__(self, path_or_paths, validate_schema: bool=True) -> None: + ... _tables: Incomplete schema: Incomplete - def read_table(self, columns=None): ... - def validate_schemas(self, piece, table) -> None: ... - def read_pandas(self, columns=None, use_threads: bool = True): ... -def check_chunked_overflow(name, col) -> None: ... + def read_table(self, columns: list[str] | None=None) -> Table: + ... -_FEATHER_SUPPORTED_CODECS: Incomplete + def validate_schemas(self, piece, table) -> None: + ... -def write_feather(df, dest, compression=None, compression_level=None, chunksize=None, version: int = 2) -> None: ... -def read_feather(source, columns=None, use_threads: bool = True, memory_map: bool = False, **kwargs): ... -def _read_table_internal(source, columns=None, memory_map: bool = False, use_threads: bool = True): ... -def read_table(source, columns=None, memory_map: bool = False, use_threads: bool = True): ... + def read_pandas(self, columns: list[str] | None=None, use_threads: bool=True): + ... diff --git a/python/pyarrow-stubs/pyarrow/flight.pyi b/python/pyarrow-stubs/pyarrow/flight.pyi index 42ff65f0df41..dcebdef09c4a 100644 --- a/python/pyarrow-stubs/pyarrow/flight.pyi +++ b/python/pyarrow-stubs/pyarrow/flight.pyi @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete from pyarrow._flight import Action as Action, ActionType as ActionType, BasicAuth as BasicAuth, CallInfo as CallInfo, CertKeyPair as CertKeyPair, ClientAuthHandler as ClientAuthHandler, ClientMiddleware as ClientMiddleware, ClientMiddlewareFactory as ClientMiddlewareFactory, DescriptorType as DescriptorType, FlightCallOptions as FlightCallOptions, FlightCancelledError as FlightCancelledError, FlightClient as FlightClient, FlightDataStream as FlightDataStream, FlightDescriptor as FlightDescriptor, FlightEndpoint as FlightEndpoint, FlightError as FlightError, FlightInfo as FlightInfo, FlightInternalError as FlightInternalError, FlightMetadataReader as FlightMetadataReader, FlightMetadataWriter as FlightMetadataWriter, FlightMethod as FlightMethod, FlightServerBase as FlightServerBase, FlightServerError as FlightServerError, FlightStreamChunk as FlightStreamChunk, FlightStreamReader as FlightStreamReader, FlightStreamWriter as FlightStreamWriter, FlightTimedOutError as FlightTimedOutError, FlightUnauthenticatedError as FlightUnauthenticatedError, FlightUnauthorizedError as FlightUnauthorizedError, FlightUnavailableError as FlightUnavailableError, FlightWriteSizeExceededError as FlightWriteSizeExceededError, GeneratorStream as GeneratorStream, Location as Location, MetadataRecordBatchReader as MetadataRecordBatchReader, MetadataRecordBatchWriter as MetadataRecordBatchWriter, RecordBatchStream as RecordBatchStream, Result as Result, SchemaResult as SchemaResult, ServerAuthHandler as ServerAuthHandler, ServerCallContext as ServerCallContext, ServerMiddleware as ServerMiddleware, ServerMiddlewareFactory as ServerMiddlewareFactory, Ticket as Ticket, TracingServerMiddlewareFactory as TracingServerMiddlewareFactory, connect as connect diff --git a/python/pyarrow-stubs/pyarrow/fs.pyi b/python/pyarrow-stubs/pyarrow/fs.pyi index 78e8e3f3afda..58ddf41f685f 100644 --- a/python/pyarrow-stubs/pyarrow/fs.pyi +++ b/python/pyarrow-stubs/pyarrow/fs.pyi @@ -15,6 +15,10 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence from _typeshed import Incomplete from pyarrow._azurefs import AzureFileSystem as AzureFileSystem from pyarrow._fs import FileInfo as FileInfo, FileSelector as FileSelector, FileSystem as FileSystem, FileSystemHandler as FileSystemHandler, FileType as FileType, LocalFileSystem as LocalFileSystem, PyFileSystem as PyFileSystem, SubTreeFileSystem as SubTreeFileSystem, _MockFileSystem as _MockFileSystem, _copy_files as _copy_files, _copy_files_selector as _copy_files_selector @@ -22,35 +26,81 @@ from pyarrow._gcsfs import GcsFileSystem as GcsFileSystem from pyarrow._hdfs import HadoopFileSystem as HadoopFileSystem from pyarrow._s3fs import AwsDefaultS3RetryStrategy as AwsDefaultS3RetryStrategy, AwsStandardS3RetryStrategy as AwsStandardS3RetryStrategy, S3FileSystem as S3FileSystem, S3LogLevel as S3LogLevel, S3RetryStrategy as S3RetryStrategy, ensure_s3_finalized as ensure_s3_finalized, ensure_s3_initialized as ensure_s3_initialized, finalize_s3 as finalize_s3, initialize_s3 as initialize_s3, resolve_s3_region as resolve_s3_region from pyarrow.util import _is_path_like as _is_path_like, _stringify_path as _stringify_path - FileStats = FileInfo _not_imported: Incomplete -def __getattr__(name) -> None: ... -def _ensure_filesystem(filesystem, *, use_mmap: bool = False): ... -def _resolve_filesystem_and_path(path, filesystem=None, *, memory_map: bool = False): ... -def copy_files(source, destination, source_filesystem=None, destination_filesystem=None, *, chunk_size=..., use_threads: bool = True) -> None: ... +def __getattr__(name) -> None: + ... + +def _ensure_filesystem(filesystem, *, use_mmap: bool=False): + ... + +def _resolve_filesystem_and_path(path, filesystem=None, *, memory_map: bool=False): + ... + +def copy_files(source: str, destination: str, source_filesystem: FileSystem | None=None, destination_filesystem: FileSystem | None=None, *, chunk_size: int=..., use_threads: bool=True) -> None: + ... class FSSpecHandler(FileSystemHandler): fs: Incomplete - def __init__(self, fs) -> None: ... - def __eq__(self, other): ... - def __ne__(self, other): ... - def get_type_name(self): ... - def normalize_path(self, path): ... + + def __init__(self, fs) -> None: + ... + + def __eq__(self, other): + ... + + def __ne__(self, other): + ... + + def get_type_name(self): + ... + + def normalize_path(self, path: str): + ... + @staticmethod - def _create_file_info(path, info): ... - def get_file_info(self, paths): ... - def get_file_info_selector(self, selector): ... - def create_dir(self, path, recursive) -> None: ... - def delete_dir(self, path) -> None: ... - def _delete_dir_contents(self, path, missing_dir_ok) -> None: ... - def delete_dir_contents(self, path, missing_dir_ok) -> None: ... - def delete_root_dir_contents(self) -> None: ... - def delete_file(self, path) -> None: ... - def move(self, src, dest) -> None: ... - def copy_file(self, src, dest) -> None: ... - def open_input_stream(self, path): ... - def open_input_file(self, path): ... - def open_output_stream(self, path, metadata): ... - def open_append_stream(self, path, metadata): ... + def _create_file_info(path, info): + ... + + def get_file_info(self, paths: list[str]): + ... + + def get_file_info_selector(self, selector: FileSelector): + ... + + def create_dir(self, path: str, recursive: bool) -> None: + ... + + def delete_dir(self, path: str) -> None: + ... + + def _delete_dir_contents(self, path, missing_dir_ok) -> None: + ... + + def delete_dir_contents(self, path: str, missing_dir_ok: bool) -> None: + ... + + def delete_root_dir_contents(self) -> None: + ... + + def delete_file(self, path: str) -> None: + ... + + def move(self, src: str, dest: str) -> None: + ... + + def copy_file(self, src: str, dest: str) -> None: + ... + + def open_input_stream(self, path: str): + ... + + def open_input_file(self, path: str): + ... + + def open_output_stream(self, path: str, metadata: Mapping[Any, Any]): + ... + + def open_append_stream(self, path: str, metadata: Mapping[Any, Any]): + ... diff --git a/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi b/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi index 3d04612d9651..c5cb5ed212af 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete from .from_dataframe import from_dataframe as from_dataframe diff --git a/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi b/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi index 9abec869d2db..84b19a6e967b 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi @@ -15,6 +15,10 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence import enum import pyarrow as pa from _typeshed import Incomplete @@ -31,11 +35,23 @@ class DlpackDeviceType(enum.IntEnum): class _PyArrowBuffer: _x: Incomplete - def __init__(self, x: pa.Buffer, allow_copy: bool = True) -> None: ... + + def __init__(self, x: pa.Buffer, allow_copy: bool=True) -> None: + ... + @property - def bufsize(self) -> int: ... + def bufsize(self) -> int: + ... + @property - def ptr(self) -> int: ... - def __dlpack__(self) -> None: ... - def __dlpack_device__(self) -> tuple[DlpackDeviceType, int | None]: ... - def __repr__(self) -> str: ... + def ptr(self) -> int: + ... + + def __dlpack__(self) -> None: + ... + + def __dlpack_device__(self) -> tuple[DlpackDeviceType, int | None]: + ... + + def __repr__(self) -> str: + ... diff --git a/python/pyarrow-stubs/pyarrow/interchange/column.pyi b/python/pyarrow-stubs/pyarrow/interchange/column.pyi index d0d32f193b8b..780ae251cc65 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/column.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/column.pyi @@ -15,11 +15,16 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Callable, IO, Iterator, Literal, Mapping, NamedTuple, Self, Sequence import enum import pyarrow as pa from _typeshed import Incomplete from pyarrow.interchange.buffer import _PyArrowBuffer as _PyArrowBuffer from typing import Any, Iterable, TypedDict +_PYARROW_KINDS: Incomplete class DtypeKind(enum.IntEnum): INT = 0 @@ -30,7 +35,6 @@ class DtypeKind(enum.IntEnum): DATETIME = 22 CATEGORICAL = 23 Dtype = tuple[DtypeKind, int, str, str] -_PYARROW_KINDS: Incomplete class ColumnNullType(enum.IntEnum): NON_NULLABLE = 0 @@ -55,30 +59,61 @@ class Endianness: NATIVE: str NA: str -class NoBufferPresent(Exception): ... +class NoBufferPresent(Exception): + ... class _PyArrowColumn: _allow_copy: Incomplete _dtype: Incomplete _col: Incomplete - def __init__(self, column: pa.Array | pa.ChunkedArray, allow_copy: bool = True) -> None: ... - def size(self) -> int: ... + + def __init__(self, column: pa.Array | pa.ChunkedArray, allow_copy: bool=True) -> None: + ... + + def size(self) -> int: + ... + @property - def offset(self) -> int: ... + def offset(self) -> int: + ... + @property - def dtype(self) -> tuple[DtypeKind, int, str, str]: ... - def _dtype_from_arrowdtype(self, dtype: pa.DataType, bit_width: int) -> tuple[DtypeKind, int, str, str]: ... + def dtype(self) -> tuple[DtypeKind, int, str, str]: + ... + + def _dtype_from_arrowdtype(self, dtype: pa.DataType, bit_width: int) -> tuple[DtypeKind, int, str, str]: + ... + @property - def describe_categorical(self) -> CategoricalDescription: ... + def describe_categorical(self) -> CategoricalDescription: + ... + @property - def describe_null(self) -> tuple[ColumnNullType, Any]: ... + def describe_null(self) -> tuple[ColumnNullType, Any]: + ... + @property - def null_count(self) -> int: ... + def null_count(self) -> int: + ... + @property - def metadata(self) -> dict[str, Any]: ... - def num_chunks(self) -> int: ... - def get_chunks(self, n_chunks: int | None = None) -> Iterable[_PyArrowColumn]: ... - def get_buffers(self) -> ColumnBuffers: ... - def _get_data_buffer(self) -> tuple[_PyArrowBuffer, Any]: ... - def _get_validity_buffer(self) -> tuple[_PyArrowBuffer, Any]: ... - def _get_offsets_buffer(self) -> tuple[_PyArrowBuffer, Any]: ... + def metadata(self) -> dict[str, Any]: + ... + + def num_chunks(self) -> int: + ... + + def get_chunks(self, n_chunks: int | None=None) -> Iterable[_PyArrowColumn]: + ... + + def get_buffers(self) -> ColumnBuffers: + ... + + def _get_data_buffer(self) -> tuple[_PyArrowBuffer, Any]: + ... + + def _get_validity_buffer(self) -> tuple[_PyArrowBuffer, Any]: + ... + + def _get_offsets_buffer(self) -> tuple[_PyArrowBuffer, Any]: + ... diff --git a/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi b/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi index 6f4740b494d2..693e7fc9c5ec 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi @@ -15,6 +15,10 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Callable, IO, Iterator, Literal, Mapping, NamedTuple, Self import pyarrow as pa from _typeshed import Incomplete from pyarrow.interchange.column import _PyArrowColumn as _PyArrowColumn @@ -24,17 +28,43 @@ class _PyArrowDataFrame: _df: Incomplete _nan_as_null: Incomplete _allow_copy: Incomplete - def __init__(self, df: pa.Table | pa.RecordBatch, nan_as_null: bool = False, allow_copy: bool = True) -> None: ... - def __dataframe__(self, nan_as_null: bool = False, allow_copy: bool = True) -> _PyArrowDataFrame: ... + + def __init__(self, df: pa.Table | pa.RecordBatch, nan_as_null: bool=False, allow_copy: bool=True) -> None: + ... + + def __dataframe__(self, nan_as_null: bool=False, allow_copy: bool=True) -> _PyArrowDataFrame: + ... + @property - def metadata(self) -> dict[str, Any]: ... - def num_columns(self) -> int: ... - def num_rows(self) -> int: ... - def num_chunks(self) -> int: ... - def column_names(self) -> Iterable[str]: ... - def get_column(self, i: int) -> _PyArrowColumn: ... - def get_column_by_name(self, name: str) -> _PyArrowColumn: ... - def get_columns(self) -> Iterable[_PyArrowColumn]: ... - def select_columns(self, indices: Sequence[int]) -> _PyArrowDataFrame: ... - def select_columns_by_name(self, names: Sequence[str]) -> _PyArrowDataFrame: ... - def get_chunks(self, n_chunks: int | None = None) -> Iterable[_PyArrowDataFrame]: ... + def metadata(self) -> dict[str, Any]: + ... + + def num_columns(self) -> int: + ... + + def num_rows(self) -> int: + ... + + def num_chunks(self) -> int: + ... + + def column_names(self) -> Iterable[str]: + ... + + def get_column(self, i: int) -> _PyArrowColumn: + ... + + def get_column_by_name(self, name: str) -> _PyArrowColumn: + ... + + def get_columns(self) -> Iterable[_PyArrowColumn]: + ... + + def select_columns(self, indices: Sequence[int]) -> _PyArrowDataFrame: + ... + + def select_columns_by_name(self, names: Sequence[str]) -> _PyArrowDataFrame: + ... + + def get_chunks(self, n_chunks: int | None=None) -> Iterable[_PyArrowDataFrame]: + ... diff --git a/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi b/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi index 0062bd275b00..f299d3c4578f 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi @@ -15,23 +15,48 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete import pyarrow as pa from pyarrow.interchange.column import ColumnBuffers as ColumnBuffers, ColumnNullType as ColumnNullType, Dtype as Dtype, DtypeKind as DtypeKind from typing import Any - DataFrameObject = Any ColumnObject = Any BufferObject = Any _PYARROW_DTYPES: dict[DtypeKind, dict[int, Any]] -def from_dataframe(df: DataFrameObject, allow_copy: bool = True) -> pa.Table: ... -def _from_dataframe(df: DataFrameObject, allow_copy: bool = True): ... -def protocol_df_chunk_to_pyarrow(df: DataFrameObject, allow_copy: bool = True) -> pa.RecordBatch: ... -def column_to_array(col: ColumnObject, allow_copy: bool = True) -> pa.Array: ... -def bool_column_to_array(col: ColumnObject, allow_copy: bool = True) -> pa.Array: ... -def categorical_column_to_dictionary(col: ColumnObject, allow_copy: bool = True) -> pa.DictionaryArray: ... -def parse_datetime_format_str(format_str): ... -def map_date_type(data_type): ... -def buffers_to_array(buffers: ColumnBuffers, data_type: tuple[DtypeKind, int, str, str], length: int, describe_null: ColumnNullType, offset: int = 0, allow_copy: bool = True) -> pa.Array: ... -def validity_buffer_from_mask(validity_buff: BufferObject, validity_dtype: Dtype, describe_null: ColumnNullType, length: int, offset: int = 0, allow_copy: bool = True) -> pa.Buffer: ... -def validity_buffer_nan_sentinel(data_pa_buffer: BufferObject, data_type: Dtype, describe_null: ColumnNullType, length: int, offset: int = 0, allow_copy: bool = True) -> pa.Buffer: ... +def from_dataframe(df: DataFrameObject, allow_copy: bool=True) -> pa.Table: + ... + +def _from_dataframe(df: DataFrameObject, allow_copy: bool=True) -> pyarrow.Table: + ... + +def protocol_df_chunk_to_pyarrow(df: DataFrameObject, allow_copy: bool=True) -> pa.RecordBatch: + ... + +def column_to_array(col: ColumnObject, allow_copy: bool=True) -> pa.Array: + ... + +def bool_column_to_array(col: ColumnObject, allow_copy: bool=True) -> pa.Array: + ... + +def categorical_column_to_dictionary(col: ColumnObject, allow_copy: bool=True) -> pa.DictionaryArray: + ... + +def parse_datetime_format_str(format_str): + ... + +def map_date_type(data_type): + ... + +def buffers_to_array(buffers: ColumnBuffers, data_type: tuple[DtypeKind, int, str, str], length: int, describe_null: ColumnNullType, offset: int=0, allow_copy: bool=True) -> pa.Array: + ... + +def validity_buffer_from_mask(validity_buff: BufferObject, validity_dtype: Dtype, describe_null: ColumnNullType, length: int, offset: int=0, allow_copy: bool=True) -> pa.Buffer: + ... + +def validity_buffer_nan_sentinel(data_pa_buffer: BufferObject, data_type: Dtype, describe_null: ColumnNullType, length: int, offset: int=0, allow_copy: bool=True) -> pa.Buffer: + ... diff --git a/python/pyarrow-stubs/pyarrow/ipc.pyi b/python/pyarrow-stubs/pyarrow/ipc.pyi index 07ddda7bb164..73a20d7f1088 100644 --- a/python/pyarrow-stubs/pyarrow/ipc.pyi +++ b/python/pyarrow-stubs/pyarrow/ipc.pyi @@ -15,32 +15,60 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +import pyarrow.ipc +from pyarrow.lib import __doc__ as __doc__ import pyarrow.lib as lib from _typeshed import Incomplete from pyarrow.lib import Alignment as Alignment, IpcReadOptions as IpcReadOptions, IpcWriteOptions as IpcWriteOptions, Message as Message, MessageReader as MessageReader, MetadataVersion as MetadataVersion, ReadStats as ReadStats, RecordBatchReader as RecordBatchReader, WriteStats as WriteStats, _ReadPandasMixin as _ReadPandasMixin, get_record_batch_size as get_record_batch_size, get_tensor_size as get_tensor_size, read_message as read_message, read_record_batch as read_record_batch, read_schema as read_schema, read_tensor as read_tensor, write_tensor as write_tensor +_ipc_writer_class_doc: str +_ipc_file_writer_class_doc: Incomplete + +def _get_legacy_format_default(options): + ... + +def _ensure_default_ipc_read_options(options): + ... + +def new_stream(sink: str | pyarrow.NativeFile | IO[Any], schema: pyarrow.Schema, *, options: pyarrow.ipc.IpcWriteOptions | None=None) -> RecordBatchStreamWriter: + ... + +def open_stream(source, *, options: pyarrow.ipc.IpcReadOptions | None=None, memory_pool: pyarrow.MemoryPool | None=None) -> RecordBatchStreamReader: + ... + +def new_file(sink: str | pyarrow.NativeFile | IO[Any], schema: pyarrow.Schema, *, options: pyarrow.ipc.IpcWriteOptions | None=None, metadata: dict | pyarrow.KeyValueMetadata | None=None) -> RecordBatchFileWriter: + ... + +def open_file(source, footer_offset: int | None=None, *, options: pyarrow.ipc.IpcReadOptions | None=None, memory_pool: pyarrow.MemoryPool | None=None) -> RecordBatchFileReader: + ... + +def serialize_pandas(df, *, nthreads: int | None=None, preserve_index: bool | None=None): + ... + +def deserialize_pandas(buf, *, use_threads: bool=True): + ... class RecordBatchStreamReader(lib._RecordBatchStreamReader): - def __init__(self, source, *, options=None, memory_pool=None) -> None: ... -_ipc_writer_class_doc: str -_ipc_file_writer_class_doc: Incomplete + def __init__(self, source, *, options=None, memory_pool=None) -> None: + ... class RecordBatchStreamWriter(lib._RecordBatchStreamWriter): __doc__: Incomplete - def __init__(self, sink, schema, *, options=None) -> None: ... + + def __init__(self, sink, schema, *, options=None) -> None: + ... class RecordBatchFileReader(lib._RecordBatchFileReader): - def __init__(self, source, footer_offset=None, *, options=None, memory_pool=None) -> None: ... + + def __init__(self, source, footer_offset=None, *, options=None, memory_pool=None) -> None: + ... class RecordBatchFileWriter(lib._RecordBatchFileWriter): __doc__: Incomplete - def __init__(self, sink, schema, *, options=None, metadata=None) -> None: ... - -def _get_legacy_format_default(options): ... -def _ensure_default_ipc_read_options(options): ... -def new_stream(sink, schema, *, options=None): ... -def open_stream(source, *, options=None, memory_pool=None): ... -def new_file(sink, schema, *, options=None, metadata=None): ... -def open_file(source, footer_offset=None, *, options=None, memory_pool=None): ... -def serialize_pandas(df, *, nthreads=None, preserve_index=None): ... -def deserialize_pandas(buf, *, use_threads: bool = True): ... + + def __init__(self, sink, schema, *, options=None, metadata=None) -> None: + ... diff --git a/python/pyarrow-stubs/pyarrow/json.pyi b/python/pyarrow-stubs/pyarrow/json.pyi index 4a5e98125a51..559ea5be22b9 100644 --- a/python/pyarrow-stubs/pyarrow/json.pyi +++ b/python/pyarrow-stubs/pyarrow/json.pyi @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete from pyarrow._json import ParseOptions as ParseOptions, ReadOptions as ReadOptions, open_json as open_json, read_json as read_json diff --git a/python/pyarrow-stubs/pyarrow/jvm.pyi b/python/pyarrow-stubs/pyarrow/jvm.pyi index a7c3cb64e4fb..423f956be3db 100644 --- a/python/pyarrow-stubs/pyarrow/jvm.pyi +++ b/python/pyarrow-stubs/pyarrow/jvm.pyi @@ -15,20 +15,47 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence from _typeshed import Incomplete +def jvm_buffer(jvm_buf) -> pyarrow.Buffer: + ... + +def _from_jvm_int_type(jvm_type) -> pyarrow.DataType: + ... + +def _from_jvm_float_type(jvm_type): + ... + +def _from_jvm_time_type(jvm_type): + ... + +def _from_jvm_timestamp_type(jvm_type): + ... + +def _from_jvm_date_type(jvm_type): + ... + +def field(jvm_field) -> pyarrow.Field: + ... + +def schema(jvm_schema) -> pyarrow.Schema: + ... + +def array(jvm_array) -> pyarrow.Array: + ... + +def record_batch(jvm_vector_schema_root): + ... + class _JvmBufferNanny: ref_manager: Incomplete - def __init__(self, jvm_buf) -> None: ... - def __del__(self) -> None: ... - -def jvm_buffer(jvm_buf): ... -def _from_jvm_int_type(jvm_type): ... -def _from_jvm_float_type(jvm_type): ... -def _from_jvm_time_type(jvm_type): ... -def _from_jvm_timestamp_type(jvm_type): ... -def _from_jvm_date_type(jvm_type): ... -def field(jvm_field): ... -def schema(jvm_schema): ... -def array(jvm_array): ... -def record_batch(jvm_vector_schema_root): ... + + def __init__(self, jvm_buf) -> None: + ... + + def __del__(self) -> None: + ... diff --git a/python/pyarrow-stubs/pyarrow/lib.pyi b/python/pyarrow-stubs/pyarrow/lib.pyi index 8f73a5ad7067..91037d4a5e34 100644 --- a/python/pyarrow-stubs/pyarrow/lib.pyi +++ b/python/pyarrow-stubs/pyarrow/lib.pyi @@ -14,50 +14,20 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # Generated by stubgen-pyx from python/pyarrow/lib.pyx -from _typeshed import Incomplete +import os +import re +import pyarrow +from typing import Callable, IO, Iterable, Iterator, Literal, NamedTuple, Self +import pyarrow.compute +from _typeshed import Incomplete import sys from collections import namedtuple from collections.abc import Mapping, Sequence from enum import IntEnum from typing import Any - -from pyarrow.includes.common import * -from pyarrow.includes.libarrow import * -from pyarrow.includes.libarrow_python import * from typing_extensions import TypeAlias - - -class DeviceAllocationType(IntEnum): - CPU = ... - CUDA = ... - CUDA_HOST = ... - OPENCL = ... - VULKAN = ... - METAL = ... - VPI = ... - ROCM = ... - ROCM_HOST = ... - EXT_DEV = ... - CUDA_MANAGED = ... - ONEAPI = ... - WEBGPU = ... - HEXAGON = ... - -class MetadataVersion(IntEnum): - V1 = ... - V2 = ... - V3 = ... - V4 = ... - V5 = ... - -class Alignment(IntEnum): - Any = ... - DataTypeSpecific = ... - At64Byte = ... - MonthDayNano: Incomplete Type_NA: Incomplete Type_BOOL: Incomplete @@ -105,1631 +75,3981 @@ Type_INTERVAL_MONTHS: Incomplete Type_INTERVAL_DAY_TIME: Incomplete UnionMode_SPARSE: Incomplete UnionMode_DENSE: Incomplete -__pc = None -__pac = None -__cuda_loaded = None +__pc: Incomplete +__pac: Incomplete +__cuda_loaded: Incomplete ordered_dict = dict ArrowIOError = IOError -have_signal_refcycle = sys.version_info < (3, 8, 10) or (3, 9) <= sys.version_info or sys.version_info[:2] == (3, 10) -VersionInfo = namedtuple('VersionInfo', ('major', 'minor', 'patch')) -RuntimeInfo = namedtuple('RuntimeInfo', ('simd_level', 'detected_simd_level')) -BuildInfo = namedtuple('BuildInfo', ('build_type', 'cpp_build_info')) -CppBuildInfo = namedtuple('CppBuildInfo', ('version', 'version_info', 'so_version', 'full_so_version', 'compiler_id', 'compiler_version', 'compiler_flags', 'git_id', 'git_description', 'package_kind', 'build_type')) -build_info = _build_info() -cpp_build_info = build_info.cpp_build_info -cpp_version = build_info.cpp_build_info.version -cpp_version_info = build_info.cpp_build_info.version_info +have_signal_refcycle: bool + +class VersionInfo(NamedTuple): + major: Incomplete + minor: Incomplete + patch: Incomplete + +class RuntimeInfo(NamedTuple): + simd_level: Incomplete + detected_simd_level: Incomplete + +class BuildInfo(NamedTuple): + build_type: Incomplete + cpp_build_info: Incomplete + +class CppBuildInfo(NamedTuple): + version: Incomplete + version_info: Incomplete + so_version: Incomplete + full_so_version: Incomplete + compiler_id: Incomplete + compiler_version: Incomplete + compiler_flags: Incomplete + git_id: Incomplete + git_description: Incomplete + package_kind: Incomplete + build_type: Incomplete +build_info: BuildInfo +cpp_build_info: CppBuildInfo +cpp_version: str +cpp_version_info: VersionInfo _pandas_api: Incomplete -_python_extension_types_registry = [] -_registry_nanny = _ExtensionRegistryNanny() -DEFAULT_BUFFER_SIZE = 2 ** 16 -_WriteStats = namedtuple('WriteStats', ('num_messages', 'num_record_batches', 'num_dictionary_batches', 'num_dictionary_deltas', 'num_replaced_dictionaries')) -_ReadStats = namedtuple('ReadStats', ('num_messages', 'num_record_batches', 'num_dictionary_batches', 'num_dictionary_deltas', 'num_replaced_dictionaries')) -_RecordBatchWithMetadata = namedtuple('RecordBatchWithMetadata', ('batch', 'custom_metadata')) +_python_extension_types_registry: list[Incomplete] +_registry_nanny: _ExtensionRegistryNanny +DEFAULT_BUFFER_SIZE: int -class ArrowException(Exception): ... +class _WriteStats(NamedTuple): + num_messages: Incomplete + num_record_batches: Incomplete + num_dictionary_batches: Incomplete + num_dictionary_deltas: Incomplete + num_replaced_dictionaries: Incomplete -class ArrowInvalid(ValueError, ArrowException): ... +class _ReadStats(NamedTuple): + num_messages: Incomplete + num_record_batches: Incomplete + num_dictionary_batches: Incomplete + num_dictionary_deltas: Incomplete + num_replaced_dictionaries: Incomplete -class ArrowMemoryError(MemoryError, ArrowException): ... +class _RecordBatchWithMetadata(NamedTuple): + batch: Incomplete + custom_metadata: Incomplete -class ArrowKeyError(KeyError, ArrowException): - def __str__(self): ... +def cpu_count(): + ... -class ArrowTypeError(TypeError, ArrowException): ... +def set_cpu_count(count: int) -> None: + ... -class ArrowNotImplementedError(NotImplementedError, ArrowException): ... +def is_opentelemetry_enabled() -> bool: + ... -class ArrowCapacityError(ArrowException): ... +def is_threading_enabled() -> bool: + ... -class ArrowIndexError(IndexError, ArrowException): ... +def _pc(): + ... -class ArrowSerializationError(ArrowException): ... +def _pac(): + ... -class ArrowCancelled(ArrowException): - def __init__(self, message, signum=None): ... +def _ensure_cuda_loaded() -> None: + ... -class StopToken: ... +def _gdb_test_session() -> None: + ... -class SignalStopHandler: - def __init__(self): ... - def _init_signals(self): ... - def __enter__(self): ... - def __exit__(self, exc_type, exc_value, exc_tb): ... - @property - def stop_token(self): ... +def encode_file_path(path): + ... -class _PandasAPIShim(object): - _loose_version: object - _version: object - _pd: object - _types_api: object - _compat_module: object - _data_frame: object - _index: object - _series: object - _categorical_type: object - _datetimetz_type: object - _extension_array: object - _extension_dtype: object - _array_like_types: object - _lock: object - _is_ge_v23: bool - _is_ge_v3: bool - _is_ge_v3_strict: bool +def tobytes(o: str | bytes): + ... - def __init__(self): ... - def series(self, *args, **kwargs): ... - def data_frame(self, *args, **kwargs): ... - @property - def have_pandas(self): ... - @property - def compat(self): ... - @property - def pd(self): ... - def infer_dtype(self, obj): ... - def pandas_dtype(self, dtype): ... - @property - def loose_version(self): ... - @property - def version(self): ... - def is_ge_v23(self): ... - def is_ge_v3(self): ... - def is_ge_v3_strict(self): ... - def uses_string_dtype(self): ... - @property - def categorical_type(self): ... - @property - def datetimetz_type(self): ... - @property - def extension_dtype(self): ... - def is_array_like(self, obj): ... - def is_categorical(self, obj): ... - def is_datetimetz(self, obj): ... - def is_extension_array_dtype(self, obj): ... - def is_sparse(self, obj): ... - def is_data_frame(self, obj): ... - def is_series(self, obj): ... - def is_index(self, obj): ... - def get_values(self, obj): ... +def frombytes(o, *, safe: bool=False): + ... -class MemoryPool(_Weakrefable): - def __init__(self): ... - def release_unused(self): ... - def bytes_allocated(self): ... - def total_bytes_allocated(self): ... - def max_memory(self): ... - def num_allocations(self): ... - def print_stats(self): ... - @property - def backend_name(self): ... - def __repr__(self): ... +def enable_signal_handlers(enable: bool) -> None: + ... -class LoggingMemoryPool(MemoryPool): - def __init__(self): ... +def runtime_info() -> RuntimeInfo: + ... -class ProxyMemoryPool(MemoryPool): - def __init__(self): ... +def _build_info() -> BuildInfo: + ... -class Device(_Weakrefable): - def __init__(self): ... - def __eq__(self, other): ... - def __repr__(self): ... - @property - def type_name(self): ... - @property - def device_id(self): ... - @property - def is_cpu(self): ... - @property - def device_type(self): ... +def set_timezone_db_path(path: str) -> None: + ... -class MemoryManager(_Weakrefable): - def __init__(self): ... - def __repr__(self): ... - @property - def device(self): ... - @property - def is_cpu(self): ... +def default_memory_pool() -> MemoryPool: + ... -class DataType(_Weakrefable): - def __init__(self): ... - def field(self, i) -> Field: ... - @property - def id(self): ... - @property - def bit_width(self): ... - @property - def byte_width(self): ... - @property - def num_fields(self): ... - @property - def num_buffers(self): ... - @property - def has_variadic_buffers(self): ... - def __str__(self): ... - def __hash__(self): ... - def __reduce__(self): ... - def __repr__(self): ... - def __eq__(self, other): ... - def equals(self, other, *, check_metadata=False): ... - def to_pandas_dtype(self): ... - def _export_to_c(self, out_ptr): ... - @staticmethod - def _import_from_c(in_ptr): ... - def __arrow_c_schema__(self): ... - @staticmethod - def _import_from_c_capsule(schema): ... +def proxy_memory_pool(parent: MemoryPool) -> ProxyMemoryPool: + ... -class DictionaryMemo(_Weakrefable): - def __init__(self): ... +def logging_memory_pool(parent: MemoryPool) -> LoggingMemoryPool: + ... -class DictionaryType(DataType): - def __reduce__(self): ... - @property - def ordered(self): ... - @property - def index_type(self): ... - @property - def value_type(self): ... +def system_memory_pool() -> MemoryPool: + ... -class ListType(DataType): - def __reduce__(self): ... - @property - def value_field(self): ... - @property - def value_type(self): ... +def jemalloc_memory_pool() -> MemoryPool: + ... -class LargeListType(DataType): - def __reduce__(self): ... - @property - def value_field(self): ... - @property - def value_type(self): ... +def mimalloc_memory_pool() -> MemoryPool: + ... -class ListViewType(DataType): - def __reduce__(self): ... - @property - def value_field(self): ... - @property - def value_type(self): ... +def set_memory_pool(pool: MemoryPool) -> None: + ... -class LargeListViewType(DataType): - def __reduce__(self): ... - @property - def value_field(self): ... - @property - def value_type(self): ... +def log_memory_allocations(enable: bool=True) -> None: + ... -class MapType(DataType): - def __reduce__(self): ... - @property - def key_field(self): ... - @property - def key_type(self): ... - @property - def item_field(self): ... - @property - def item_type(self): ... - @property - def keys_sorted(self): ... +def total_allocated_bytes(): + ... -class FixedSizeListType(DataType): - def __reduce__(self): ... - @property - def value_field(self): ... - @property - def value_type(self): ... - @property - def list_size(self): ... +def jemalloc_set_decay_ms(decay_ms: int) -> None: + ... -class StructType(DataType): - def get_field_index(self, name): ... - def field(self, i) -> Field: ... - def get_all_field_indices(self, name): ... - def __len__(self): ... - def __iter__(self): ... - def __getitem__(self, i): ... - def __reduce__(self): ... - @property - def names(self): ... - @property - def fields(self): ... +def supported_memory_backends() -> list: + ... -class UnionType(DataType): - @property - def mode(self): ... - @property - def type_codes(self): ... - def __len__(self): ... - def __iter__(self): ... - def field(self, i) -> Field: ... - def __getitem__(self, i): ... - def __reduce__(self): ... +def default_cpu_memory_manager() -> MemoryManager: + ... -class SparseUnionType(UnionType): ... +def _get_pandas_type_map(): + ... -class DenseUnionType(UnionType): ... +def _is_primitive(type: int): + ... -class TimestampType(DataType): - @property - def unit(self): ... - @property - def tz(self): ... - def __reduce__(self): ... +def _is_nested(type: int): + ... -class Time32Type(DataType): - @property - def unit(self): ... +def _get_pandas_type(arrow_type, coerce_to_ns: bool=False): + ... -class Time64Type(DataType): - @property - def unit(self): ... +def _get_pandas_tz_type(arrow_type, coerce_to_ns: bool=False): + ... -class DurationType(DataType): - @property - def unit(self): ... +def _to_pandas_dtype(arrow_type, options=None): + ... -class FixedSizeBinaryType(DataType): - def __reduce__(self): ... +def register_extension_type(ext_type) -> None: + ... -class Decimal32Type(FixedSizeBinaryType): - def __reduce__(self): ... - @property - def precision(self): ... - @property - def scale(self): ... +def unregister_extension_type(type_name: str) -> None: + ... -class Decimal64Type(FixedSizeBinaryType): - def __reduce__(self): ... - @property - def precision(self): ... - @property - def scale(self): ... +def ensure_metadata(meta: object, allow_none: bool=False) -> KeyValueMetadata: + ... -class Decimal128Type(FixedSizeBinaryType): - def __reduce__(self): ... - @property - def precision(self): ... - @property - def scale(self): ... +def unify_schemas(schemas: list[Schema], *, promote_options: str='default') -> Schema: + ... -class Decimal256Type(FixedSizeBinaryType): - def __reduce__(self): ... - @property - def precision(self): ... - @property - def scale(self): ... +def field(name: str | bytes, type: DataType | str | None=None, nullable: bool | None=None, metadata: dict | None=None) -> Field: + ... -class RunEndEncodedType(DataType): - def __reduce__(self): ... - @property - def run_end_type(self): ... - @property - def value_type(self): ... +def null() -> DataType: + ... -class BaseExtensionType(DataType): - def __arrow_ext_class__(self): ... - def __arrow_ext_scalar_class__(self): ... - @property - def extension_name(self): ... - @property - def storage_type(self): ... - @property - def byte_width(self): ... - @property - def bit_width(self): ... - def wrap_array(self, storage): ... +def bool_() -> DataType: + ... -class ExtensionType(BaseExtensionType): - def __init__(self, storage_type: DataType, extension_name): ... - def __eq__(self, other): ... - def __repr__(self): ... - def __arrow_ext_serialize__(self): ... - @classmethod - def __arrow_ext_deserialize__(cls, storage_type, serialized): ... - def __reduce__(self): ... - def __arrow_ext_class__(self): ... - def __arrow_ext_scalar_class__(self): ... +def uint8() -> DataType: + ... -class JsonType(BaseExtensionType): - def __arrow_ext_class__(self): ... - def __reduce__(self): ... - def __arrow_ext_scalar_class__(self): ... +def int8() -> DataType: + ... -class UuidType(BaseExtensionType): - def __arrow_ext_class__(self): ... - def __reduce__(self): ... - def __arrow_ext_scalar_class__(self): ... +def uint16() -> DataType: + ... -class FixedShapeTensorType(BaseExtensionType): - @property - def value_type(self): ... - @property - def shape(self): ... - @property - def dim_names(self): ... - @property - def permutation(self): ... - def __arrow_ext_class__(self): ... - def __reduce__(self): ... - def __arrow_ext_scalar_class__(self): ... +def int16() -> DataType: + ... -class Bool8Type(BaseExtensionType): - def __arrow_ext_class__(self): ... - def __reduce__(self): ... - def __arrow_ext_scalar_class__(self): ... +def uint32() -> DataType: + ... -class OpaqueType(BaseExtensionType): - @property - def type_name(self): ... - @property - def vendor_name(self): ... - def __arrow_ext_class__(self): ... - def __reduce__(self): ... - def __arrow_ext_scalar_class__(self): ... +def int32() -> DataType: + ... -class UnknownExtensionType(ExtensionType): - def __init__(self, storage_type: DataType, serialized): ... - def __arrow_ext_serialize__(self): ... - @classmethod - def __arrow_ext_deserialize__(cls, storage_type, serialized): ... +def uint64() -> DataType: + ... -class KeyValueMetadata(_Metadata, Mapping): - def __init__(self, __arg0__=None, **kwargs): ... - def equals(self, other: KeyValueMetadata): ... - def __repr__(self): ... - def __str__(self): ... - def __eq__(self, other): ... - def __len__(self): ... - def __contains__(self, key): ... - def __getitem__(self, key): ... - def __iter__(self): ... - def __reduce__(self): ... - def key(self, i): ... - def value(self, i): ... - def keys(self): ... - def values(self): ... - def items(self): ... - def get_all(self, key): ... - def to_dict(self): ... +def int64() -> DataType: + ... -class Field(_Weakrefable): - type: DataType +def tzinfo_to_string(tz) -> str: + ... - def __init__(self): ... - def equals(self, other: Field, check_metadata: bool=False): ... - def __eq__(self, other): ... - def __reduce__(self): ... - def __str__(self): ... - def __repr__(self): ... - def __hash__(self): ... - @property - def nullable(self): ... - @property - def name(self): ... - @property - def metadata(self): ... - def with_metadata(self, metadata): ... - def remove_metadata(self): ... - def with_type(self, new_type: DataType): ... - def with_name(self, name): ... - def with_nullable(self, nullable): ... - def flatten(self): ... - def _export_to_c(self, out_ptr): ... - @staticmethod - def _import_from_c(in_ptr): ... - def __arrow_c_schema__(self): ... - @staticmethod - def _import_from_c_capsule(schema): ... +def string_to_tzinfo(name, *, prefer_zoneinfo: bool=True): + ... -class Schema(_Weakrefable): - def __init__(self): ... - def __len__(self): ... - def __getitem__(self, key): ... - def __iter__(self): ... - def __reduce__(self): ... - def __hash__(self): ... - def __sizeof__(self): ... - @property - def pandas_metadata(self): ... - @property - def names(self): ... - @property - def types(self): ... - @property - def metadata(self): ... - def __eq__(self, other): ... - def empty_table(self): ... - def equals(self, other: Schema, check_metadata: bool=False): ... - @classmethod - def from_pandas(cls, df, preserve_index=None): ... - def field(self, i): ... - def _field(self, i: int): ... - def field_by_name(self, name): ... - def get_field_index(self, name): ... - def get_all_field_indices(self, name): ... - def append(self, field: Field): ... - def insert(self, i: int, field: Field): ... - def remove(self, i: int): ... - def set(self, i: int, field: Field): ... - def add_metadata(self, metadata): ... - def with_metadata(self, metadata): ... - def serialize(self, memory_pool=None): ... - def remove_metadata(self): ... - def to_string(self, truncate_metadata=True, show_field_metadata=True, show_schema_metadata=True, element_size_limit=100): ... - def _export_to_c(self, out_ptr): ... - @staticmethod - def _import_from_c(in_ptr): ... - def __str__(self): ... - def __repr__(self): ... - def __arrow_c_schema__(self): ... - @staticmethod - def _import_from_c_capsule(schema): ... +def timestamp(unit: str, tz: str | None=None) -> TimestampType: + ... -class _ExtensionRegistryNanny(_Weakrefable): - def __init__(self): ... - def release_registry(self): ... +def time32(unit: str) -> Time32Type: + ... -class Scalar(_Weakrefable): - def __init__(self): ... - @property - def type(self): ... - @property - def is_valid(self): ... - def cast(self, target_type: object | None=None, safe=None, options=None, memory_pool=None): ... - def validate(self, *, full=False): ... - def __repr__(self): ... - def __str__(self): ... - def equals(self, other: Scalar): ... - def __eq__(self, other): ... - def __hash__(self): ... - def __reduce__(self): ... - def as_py(self, *, maps_as_pydicts=None): ... - def __abs__(self): ... - def __add__(self, other: object): ... - def __radd__(self, other: object): ... - def __truediv__(self, other: object): ... - def __rtruediv__(self, other: object): ... - def __mul__(self, other: object): ... - def __rmul__(self, other: object): ... - def __neg__(self): ... - def __pow__(self, other: object, mod: None = None): ... - def __rpow__(self, other: object, mod: None = None): ... - def __sub__(self, other: object): ... - def __rsub__(self, other: object): ... - def __and__(self, other: object): ... - def __rand__(self, other: object): ... - def __or__(self, other: object): ... - def __ror__(self, other: object): ... - def __xor__(self, other: object): ... - def __rxor__(self, other: object): ... - def __lshift__(self, other: object): ... - def __rlshift__(self, other: object): ... - def __rshift__(self, other: object): ... - def __rrshift__(self, other: object): ... +def time64(unit: str) -> Time64Type: + ... -class NullScalar(Scalar): - def __init__(self): ... - def as_py(self, *, maps_as_pydicts=None): ... +def duration(unit: str) -> DurationType: + ... -class BooleanScalar(Scalar): - def as_py(self, *, maps_as_pydicts=None): ... - def __bool__(self): ... +def month_day_nano_interval() -> DataType: + ... -class UInt8Scalar(Scalar): - def as_py(self, *, maps_as_pydicts=None): ... - def __index__(self): ... +def date32() -> DataType: + ... -class Int8Scalar(Scalar): - def as_py(self, *, maps_as_pydicts=None): ... - def __index__(self): ... +def date64() -> DataType: + ... -class UInt16Scalar(Scalar): - def as_py(self, *, maps_as_pydicts=None): ... - def __index__(self): ... +def float16() -> DataType: + ... -class Int16Scalar(Scalar): - def as_py(self, *, maps_as_pydicts=None): ... - def __index__(self): ... +def float32() -> DataType: + ... -class UInt32Scalar(Scalar): - def as_py(self, *, maps_as_pydicts=None): ... - def __index__(self): ... +def float64() -> DataType: + ... -class Int32Scalar(Scalar): - def as_py(self, *, maps_as_pydicts=None): ... - def __index__(self): ... +def decimal32(precision: int, scale: int=0) -> DataType: + ... -class UInt64Scalar(Scalar): - def as_py(self, *, maps_as_pydicts=None): ... - def __index__(self): ... +def decimal64(precision: int, scale: int=0) -> DataType: + ... -class Int64Scalar(Scalar): - def as_py(self, *, maps_as_pydicts=None): ... - def __index__(self): ... +def decimal128(precision: int, scale: int=0) -> DataType: + ... -class HalfFloatScalar(Scalar): - def as_py(self, *, maps_as_pydicts=None): ... - def __float__(self): ... - def __int__(self): ... +def decimal256(precision: int, scale: int=0) -> DataType: + ... -class FloatScalar(Scalar): - def as_py(self, *, maps_as_pydicts=None): ... - def __float__(self): ... - def __int__(self): ... +def string() -> DataType: + ... -class DoubleScalar(Scalar): - def as_py(self, *, maps_as_pydicts=None): ... - def __float__(self): ... - def __int__(self): ... +def utf8(): + ... -class Decimal32Scalar(Scalar): - def as_py(self, *, maps_as_pydicts=None): ... +def binary(length: int=-1) -> DataType: + ... -class Decimal64Scalar(Scalar): - def as_py(self, *, maps_as_pydicts=None): ... +def large_binary() -> DataType: + ... -class Decimal128Scalar(Scalar): - def as_py(self, *, maps_as_pydicts=None): ... +def large_string() -> DataType: + ... -class Decimal256Scalar(Scalar): - def as_py(self, *, maps_as_pydicts=None): ... +def large_utf8(): + ... -class Date32Scalar(Scalar): - @property - def value(self): ... - def as_py(self, *, maps_as_pydicts=None): ... +def binary_view() -> DataType: + ... -class Date64Scalar(Scalar): - @property - def value(self): ... - def as_py(self, *, maps_as_pydicts=None): ... +def string_view() -> DataType: + ... -class Time32Scalar(Scalar): - @property - def value(self): ... - def as_py(self, *, maps_as_pydicts=None): ... +def list_(value_type: DataType | Field, list_size: int=-1) -> DataType: + ... -class Time64Scalar(Scalar): - @property - def value(self): ... - def as_py(self, *, maps_as_pydicts=None): ... +def large_list(value_type: DataType | Field) -> LargeListType: + ... -class TimestampScalar(Scalar): - @property - def value(self): ... - def as_py(self, *, maps_as_pydicts=None): ... - def __repr__(self): ... +def list_view(value_type: DataType | Field) -> ListViewType: + ... -class DurationScalar(Scalar): - @property - def value(self): ... - def as_py(self, *, maps_as_pydicts=None): ... +def large_list_view(value_type: DataType | Field) -> LargeListViewType: + ... -class MonthDayNanoIntervalScalar(Scalar): - @property - def value(self): ... - def as_py(self, *, maps_as_pydicts=None): ... +def map_(key_type: DataType | Field, item_type: DataType | Field, keys_sorted: bool=False) -> MapType: + ... -class BinaryScalar(Scalar): - def as_buffer(self): ... - def as_py(self, *, maps_as_pydicts=None): ... - def __bytes__(self): ... +def dictionary(index_type: DataType, value_type: DataType, ordered: bool=False) -> DictionaryType: + ... -class LargeBinaryScalar(BinaryScalar): ... +def struct(fields) -> DataType: + ... -class FixedSizeBinaryScalar(BinaryScalar): ... +def sparse_union(child_fields, type_codes: list[int] | None=None) -> DataType: + ... -class StringScalar(BinaryScalar): - def as_py(self, *, maps_as_pydicts=None): ... +def dense_union(child_fields, type_codes: list[int] | None=None) -> DataType: + ... -class LargeStringScalar(StringScalar): ... +def union(child_fields, mode: str, type_codes: list[int] | None=None) -> UnionType: + ... -class BinaryViewScalar(BinaryScalar): ... +def run_end_encoded(run_end_type: DataType, value_type: DataType) -> DataType: + ... -class StringViewScalar(StringScalar): ... +def json_(storage_type: DataType=utf8()) -> JsonType: + ... -class ListScalar(Scalar, Sequence): - @property - def values(self): ... - def __len__(self): ... - def __getitem__(self, i): ... - def __iter__(self): ... - def as_py(self, *, maps_as_pydicts=None): ... +def uuid() -> UuidType: + ... -class FixedSizeListScalar(ListScalar): ... +def fixed_shape_tensor(value_type: DataType, shape: tuple | list[int], dim_names: tuple | list[str] | None=None, permutation=None) -> FixedShapeTensorType: + ... -class LargeListScalar(ListScalar): ... +def bool8() -> Bool8Type: + ... -class ListViewScalar(ListScalar): ... +def opaque(storage_type: DataType, type_name: str, vendor_name: str) -> OpaqueType: + ... -class LargeListViewScalar(ListScalar): ... +def type_for_alias(name: str) -> DataType: + ... -class StructScalar(Scalar, Mapping): - def __len__(self): ... - def __iter__(self): ... - def items(self): ... - def __contains__(self, key): ... - def __getitem__(self, key): ... - def as_py(self, *, maps_as_pydicts=None): ... - def _as_py_tuple(self): ... - def __repr__(self): ... - def __str__(self): ... +def ensure_type(ty: object, allow_none: bool=False) -> DataType: + ... -class MapScalar(ListScalar, Mapping): - def __getitem__(self, i): ... - def __iter__(self): ... - def as_py(self, *, maps_as_pydicts=None): ... - def keys(self): ... +def schema(fields, metadata: dict | None=None) -> Schema: + ... -class DictionaryScalar(Scalar): - @staticmethod - def _reconstruct(type, is_valid, index, dictionary): ... - def __reduce__(self): ... - @property - def index(self): ... - @property - def value(self): ... - @property - def dictionary(self): ... - def as_py(self, *, maps_as_pydicts=None): ... +def from_numpy_dtype(dtype: object) -> DataType: + ... -class RunEndEncodedScalar(Scalar): - @property - def value(self): ... - def as_py(self, *, maps_as_pydicts=None): ... +def is_boolean_value(obj: object): + ... -class UnionScalar(Scalar): - @property - def value(self): ... - def as_py(self, *, maps_as_pydicts=None): ... - @property - def type_code(self): ... +def is_integer_value(obj: object): + ... -class ExtensionScalar(Scalar): - @property - def value(self): ... - def as_py(self, *, maps_as_pydicts=None): ... - @staticmethod - def from_storage(typ: BaseExtensionType, value): ... +def is_float_value(obj: object): + ... -class JsonScalar(ExtensionScalar): ... +def _datetime_from_int(value: int, unit: int, tzinfo=None): + ... -class UuidScalar(ExtensionScalar): - def as_py(self, *, maps_as_pydicts=None): ... +def scalar(value: Any, type: DataType | None=None, *, from_pandas: bool | None=None, memory_pool: MemoryPool | None=None) -> Scalar: + ... -class FixedShapeTensorScalar(ExtensionScalar): - def to_numpy(self): ... - def to_tensor(self): ... - def __dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None): ... - def __dlpack_device__(self): ... +def _ndarray_to_arrow_type(values: object, type: DataType) -> DataType: + ... -class OpaqueScalar(ExtensionScalar): ... +def _handle_arrow_array_protocol(obj, type, mask, size): + ... -class Bool8Scalar(ExtensionScalar): - def as_py(self, *, maps_as_pydicts=None): ... +def array(obj: object, type: DataType | None=None, mask=None, size=None, from_pandas: bool | None=None, safe: bool=True, memory_pool: MemoryPool | None=None) -> Array | ChunkedArray: + ... -class ArrayStatistics(_Weakrefable): - def __init__(self): ... - def __repr__(self): ... - @property - def null_count(self): ... - @property - def is_null_count_exact(self): ... - @property - def distinct_count(self): ... - @property - def is_distinct_count_exact(self): ... - @property - def min(self): ... - @property - def is_min_exact(self): ... - @property - def max(self): ... - @property - def is_max_exact(self): ... +def asarray(values, type: str | DataType | None=None) -> Array | ChunkedArray: + ... -class _PandasConvertible(_Weakrefable): - def to_pandas(self, memory_pool=None, categories=None, strings_to_categorical: bool=False, zero_copy_only: bool=False, integer_object_nulls: bool=False, date_as_object: bool=True, timestamp_as_object: bool=False, use_threads: bool=True, deduplicate_objects: bool=True, ignore_metadata: bool=False, safe: bool=True, split_blocks: bool=False, self_destruct: bool=False, maps_as_pydicts: str | None=None, types_mapper=None, coerce_temporal_nanoseconds: bool=False): ... +def nulls(size: int, type: DataType | None=None, memory_pool: MemoryPool | None=None) -> Array: + ... -class Array(_PandasConvertible): - type: DataType - _name: object +def repeat(value, size: int, memory_pool: MemoryPool | None=None) -> Array: + ... - def __init__(self): ... - def _debug_print(self): ... - def diff(self, other: Array): ... - def cast(self, target_type: object | None=None, safe=None, options=None, memory_pool=None): ... - def view(self, target_type: object): ... - def sum(self, **kwargs): ... - def unique(self): ... - def dictionary_encode(self, null_encoding='mask'): ... - def value_counts(self): ... - @staticmethod - def from_pandas(obj, mask=None, type=None, safe: bool=True, memory_pool: MemoryPool | None=None): ... - def __reduce__(self): ... - @staticmethod - def from_buffers(type: DataType, length, buffers, null_count=-1, offset=0, children=None): ... - @property - def null_count(self): ... - @property - def nbytes(self): ... - def get_total_buffer_size(self): ... - def __sizeof__(self): ... - def __iter__(self): ... - def __repr__(self): ... - def to_string(self, *, indent: int=2, top_level_indent: int=0, window: int=10, container_window: int=2, skip_new_lines: c_bool=False, element_size_limit: int=100): ... - def __str__(self): ... - def __eq__(self, other): ... - def equals(self: Array, other: Array): ... - def __len__(self): ... - def is_null(self, *, nan_is_null=False): ... - def is_nan(self): ... - def is_valid(self): ... - def fill_null(self, fill_value): ... - def __getitem__(self, key): ... - def slice(self, offset=0, length=None): ... - def take(self, indices: object): ... - def drop_null(self): ... - def filter(self, mask: object, *, null_selection_behavior='drop'): ... - def index(self, value, start=None, end=None, *, memory_pool=None): ... - def sort(self, order='ascending', null_placement='at_end', **kwargs): ... - def _to_pandas(self, options, types_mapper=None, **kwargs): ... - def __array__(self, dtype=None, copy=None): ... - def to_numpy(self, zero_copy_only=True, writable=False): ... - def to_tensor(self, *, allow_nulls=False): ... - def to_pylist(self, *, maps_as_pydicts=None): ... - def tolist(self): ... - def validate(self, *, full=False): ... - @property - def offset(self): ... - def buffers(self): ... - def copy_to(self, destination): ... - def _export_to_c(self, out_ptr, out_schema_ptr=0): ... - @staticmethod - def _import_from_c(in_ptr, type): ... - def __arrow_c_array__(self, requested_schema=None): ... - @staticmethod - def _import_from_c_capsule(schema_capsule, array_capsule): ... - def _export_to_c_device(self, out_ptr, out_schema_ptr=0): ... - @staticmethod - def _import_from_c_device(in_ptr, type): ... - def __arrow_c_device_array__(self, requested_schema=None, **kwargs): ... - @staticmethod - def _import_from_c_device_capsule(schema_capsule, array_capsule): ... - @staticmethod - def from_dlpack(x, /, *, device=None, copy=None): ... - def __dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None): ... - def __dlpack_device__(self): ... - @property - def device_type(self): ... - @property - def is_cpu(self): ... - @property - def statistics(self): ... - def __abs__(self): ... - def __add__(self, other: object): ... - def __radd__(self, other: object): ... - def __truediv__(self, other: object): ... - def __rtruediv__(self, other: object): ... - def __mul__(self, other: object): ... - def __rmul__(self, other: object): ... - def __neg__(self): ... - def __pow__(self, other: object, mod: None = None): ... - def __rpow__(self, other: object, mod: None = None): ... - def __sub__(self, other: object): ... - def __rsub__(self, other: object): ... - def __and__(self, other: object): ... - def __rand__(self, other: object): ... - def __or__(self, other: object): ... - def __ror__(self, other: object): ... - def __xor__(self, other: object): ... - def __rxor__(self, other: object): ... - def __lshift__(self, other: object): ... - def __rlshift__(self, other: object): ... - def __rshift__(self, other: object): ... - def __rrshift__(self, other: object): ... - -class NullArray(Array): ... +def infer_type(values, mask=None, from_pandas: bool=False) -> DataType: + ... -class BooleanArray(Array): - @property - def false_count(self): ... - @property - def true_count(self): ... +def arange(start: int, stop: int, step: int=1, *, memory_pool: MemoryPool | None=None) -> Array: + ... -class NumericArray(Array): ... +def _normalize_slice(arrow_obj: object, key: slice): + ... -class IntegerArray(NumericArray): ... +def _restore_array(data) -> Array: + ... -class FloatingPointArray(NumericArray): ... +def _compute_binary_op(func_name, left, right): + ... -class Int8Array(IntegerArray): ... +def concat_arrays(arrays: Iterable[Array], memory_pool: MemoryPool | None=None) -> Array: + ... -class UInt8Array(IntegerArray): ... +def _empty_array(type: DataType) -> DictionaryArray: + ... -class Int16Array(IntegerArray): ... +def chunked_array(arrays, type=None) -> ChunkedArray: + ... -class UInt16Array(IntegerArray): ... +def _reconstruct_record_batch(columns, schema) -> RecordBatch: + ... -class Int32Array(IntegerArray): ... +def table_to_blocks(options, table: Table, categories, extension_columns): + ... -class UInt32Array(IntegerArray): ... +def _reconstruct_table(arrays, schema) -> Table: + ... -class Int64Array(IntegerArray): ... +def record_batch(data, names: list | None=None, schema: Schema | None=None, metadata: dict | Mapping[Any, Any] | None=None) -> RecordBatch: + ... -class UInt64Array(IntegerArray): ... +def table(data, names: list | None=None, schema: Schema | None=None, metadata: dict | Mapping[Any, Any] | None=None, nthreads: int | None=None) -> Table: + ... -class Date32Array(NumericArray): ... +def concat_tables(tables, memory_pool: MemoryPool | None=None, promote_options: str='none', **kwargs) -> Table: + ... -class Date64Array(NumericArray): ... +def concat_batches(recordbatches, memory_pool: MemoryPool | None=None) -> RecordBatch: + ... -class TimestampArray(NumericArray): ... +def _from_pydict(cls, mapping: dict | Mapping[Any, Any], schema: Schema | None, metadata: dict | Mapping[Any, Any] | None) -> Self: + ... -class Time32Array(NumericArray): ... +def _from_pylist(cls, mapping, schema: Schema | None, metadata: dict | Mapping[Any, Any] | None) -> Self: + ... -class Time64Array(NumericArray): ... +def io_thread_count(): + ... -class DurationArray(NumericArray): ... +def set_io_thread_count(count: int) -> None: + ... -class MonthDayNanoIntervalArray(Array): - def to_pylist(self, *, maps_as_pydicts=None): ... +def memory_map(path: str, mode: str='r') -> MemoryMappedFile: + ... -class HalfFloatArray(FloatingPointArray): ... +def create_memory_map(path: str, size: int) -> MemoryMappedFile: + ... -class FloatArray(FloatingPointArray): ... +def allocate_buffer(size: int, memory_pool: MemoryPool | None=None, resizable: bool=False) -> ResizableBuffer | Buffer: + ... -class DoubleArray(FloatingPointArray): ... +def transcoding_input_stream(stream: NativeFile, src_encoding: str, dest_encoding: str): + ... -class FixedSizeBinaryArray(Array): ... +def py_buffer(obj: object) -> Buffer: + ... -class Decima32Array(FixedSizeBinaryArray): ... +def foreign_buffer(address: int, size: int, base=None) -> Buffer: + ... -class Decimal64Array(FixedSizeBinaryArray): ... +def as_buffer(o: object): + ... -class Decimal128Array(FixedSizeBinaryArray): ... +def _detect_compression(path) -> str: + ... -class Decimal256Array(FixedSizeBinaryArray): ... +def compress(buf: object, codec: str='lz4', asbytes: bool=False, memory_pool: MemoryPool | None=None) -> Buffer | bytes: + ... -class BaseListArray(Array): - def flatten(self, recursive=False): ... - def value_parent_indices(self): ... - def value_lengths(self): ... +def decompress(buf: object, decompressed_size: int | None=None, codec: str='lz4', asbytes: bool=False, memory_pool: MemoryPool | None=None) -> Buffer | bytes: + ... -class ListArray(BaseListArray): - @staticmethod - def from_arrays(offsets, values, type: DataType | None=None, pool: MemoryPool | None=None, mask=None): ... - @property - def values(self): ... - @property - def offsets(self): ... +def input_stream(source, compression: str | None='detect', buffer_size: int | None=None) -> NativeFile: + ... + +def output_stream(source, compression: str | None='detect', buffer_size: int | None=None) -> NativeFile: + ... + +def get_tensor_size(tensor: Tensor) -> int: + ... + +def get_record_batch_size(batch: RecordBatch) -> int: + ... + +def write_tensor(tensor: Tensor, dest: NativeFile) -> int: + ... + +def read_tensor(source: NativeFile) -> Tensor: + ... + +def read_message(source) -> Message: + ... + +def read_schema(obj, dictionary_memo: DictionaryMemo | None=None) -> Schema: + ... + +def read_record_batch(obj, schema: Schema, dictionary_memo: DictionaryMemo | None=None) -> RecordBatch: + ... + +def benchmark_PandasObjectIsNull(obj: list[Any]) -> None: + ... +_NULL: NullScalar +NA: NullScalar + +class DeviceAllocationType(IntEnum): + CPU = ... + CUDA = ... + CUDA_HOST = ... + OPENCL = ... + VULKAN = ... + METAL = ... + VPI = ... + ROCM = ... + ROCM_HOST = ... + EXT_DEV = ... + CUDA_MANAGED = ... + ONEAPI = ... + WEBGPU = ... + HEXAGON = ... + +class MetadataVersion(IntEnum): + V1 = ... + V2 = ... + V3 = ... + V4 = ... + V5 = ... + +class Alignment(IntEnum): + Any = ... + DataTypeSpecific = ... + At64Byte = ... + +class ArrowException(Exception): + ... + +class ArrowInvalid(ValueError, ArrowException): + ... + +class ArrowMemoryError(MemoryError, ArrowException): + ... + +class ArrowKeyError(KeyError, ArrowException): + + def __str__(self): + ... + +class ArrowTypeError(TypeError, ArrowException): + ... + +class ArrowNotImplementedError(NotImplementedError, ArrowException): + ... + +class ArrowCapacityError(ArrowException): + ... + +class ArrowIndexError(IndexError, ArrowException): + ... + +class ArrowSerializationError(ArrowException): + ... + +class ArrowCancelled(ArrowException): + + def __init__(self, message, signum=None): + ... + +class StopToken: + ... + +class SignalStopHandler: + + def __init__(self): + ... + + def _init_signals(self) -> None: + ... + + def __enter__(self) -> Self: + ... + + def __exit__(self, exc_type, exc_value, exc_tb) -> None: + ... + + @property + def stop_token(self): + ... + +class _PandasAPIShim(object): + _loose_version: object + _version: object + _pd: object + _types_api: object + _compat_module: object + _data_frame: object + _index: object + _series: object + _categorical_type: object + _datetimetz_type: object + _extension_array: object + _extension_dtype: object + _array_like_types: object + _lock: object + _is_ge_v23: bool + _is_ge_v3: bool + _is_ge_v3_strict: bool + + def __init__(self): + ... + + def series(self, *args, **kwargs): + ... + + def data_frame(self, *args, **kwargs): + ... + + @property + def have_pandas(self): + ... + + @property + def compat(self): + ... + + @property + def pd(self): + ... + + def infer_dtype(self, obj): + ... + + def pandas_dtype(self, dtype): + ... + + @property + def loose_version(self): + ... + + @property + def version(self): + ... + + def is_ge_v23(self): + ... + + def is_ge_v3(self): + ... + + def is_ge_v3_strict(self): + ... + + def uses_string_dtype(self): + ... + + @property + def categorical_type(self): + ... + + @property + def datetimetz_type(self): + ... + + @property + def extension_dtype(self): + ... + + def is_array_like(self, obj): + ... + + def is_categorical(self, obj): + ... + + def is_datetimetz(self, obj): + ... + + def is_extension_array_dtype(self, obj): + ... + + def is_sparse(self, obj): + ... + + def is_data_frame(self, obj): + ... + + def is_series(self, obj): + ... + + def is_index(self, obj): + ... + + def get_values(self, obj): + ... + +class _Weakrefable: + ... + +class MemoryPool(_Weakrefable): + + def __init__(self): + ... + + def release_unused(self) -> None: + ... + + def bytes_allocated(self): + ... + + def total_bytes_allocated(self): + ... + + def max_memory(self): + ... + + def num_allocations(self): + ... + + def print_stats(self) -> None: + ... + + @property + def backend_name(self) -> str: + ... + + def __repr__(self) -> str: + ... + +class LoggingMemoryPool(MemoryPool): + + def __init__(self): + ... + +class ProxyMemoryPool(MemoryPool): + + def __init__(self): + ... + +class Device(_Weakrefable): + + def __init__(self): + ... + + def __eq__(self, other): + ... + + def __repr__(self) -> str: + ... + + @property + def type_name(self) -> str: + ... + + @property + def device_id(self): + ... + + @property + def is_cpu(self): + ... + + @property + def device_type(self): + ... + +class MemoryManager(_Weakrefable): + + def __init__(self): + ... + + def __repr__(self) -> str: + ... + + @property + def device(self) -> Device: + ... + + @property + def is_cpu(self): + ... + +class DataType(_Weakrefable): + + def __init__(self): + ... + + def field(self, i: int) -> Field: + ... + + @property + def id(self): + ... + + @property + def bit_width(self): + ... + + @property + def byte_width(self): + ... + + @property + def num_fields(self): + ... + + @property + def num_buffers(self): + ... + + @property + def has_variadic_buffers(self): + ... + + def __str__(self) -> str: + ... + + def __hash__(self): + ... + + def __reduce__(self) -> tuple: + ... + + def __repr__(self) -> str: + ... + + def __eq__(self, other): + ... + + def equals(self, other, *, check_metadata: bool=False) -> bool: + ... + + def to_pandas_dtype(self): + ... + + def _export_to_c(self, out_ptr) -> None: + ... + + @staticmethod + def _import_from_c(in_ptr) -> DataType: + ... + + def __arrow_c_schema__(self): + ... + + @staticmethod + def _import_from_c_capsule(schema) -> DataType: + ... + +class DictionaryMemo(_Weakrefable): + + def __init__(self): + ... + +class DictionaryType(DataType): + + def __reduce__(self) -> tuple: + ... + + @property + def ordered(self): + ... + + @property + def index_type(self) -> DataType: + ... + + @property + def value_type(self) -> DataType: + ... + +class ListType(DataType): + + def __reduce__(self) -> tuple: + ... + + @property + def value_field(self) -> Field: + ... + + @property + def value_type(self) -> DataType: + ... + +class LargeListType(DataType): + + def __reduce__(self) -> tuple: + ... + + @property + def value_field(self) -> Field: + ... + + @property + def value_type(self) -> DataType: + ... + +class ListViewType(DataType): + + def __reduce__(self) -> tuple: + ... + + @property + def value_field(self) -> Field: + ... + + @property + def value_type(self) -> DataType: + ... + +class LargeListViewType(DataType): + + def __reduce__(self) -> tuple: + ... + + @property + def value_field(self) -> Field: + ... + + @property + def value_type(self) -> DataType: + ... + +class MapType(DataType): + + def __reduce__(self) -> tuple: + ... + + @property + def key_field(self) -> Field: + ... + + @property + def key_type(self) -> DataType: + ... + + @property + def item_field(self) -> Field: + ... + + @property + def item_type(self) -> DataType: + ... + + @property + def keys_sorted(self): + ... + +class FixedSizeListType(DataType): + + def __reduce__(self) -> tuple: + ... + + @property + def value_field(self) -> Field: + ... + + @property + def value_type(self) -> DataType: + ... + + @property + def list_size(self): + ... + +class StructType(DataType): + + def get_field_index(self, name: str) -> int: + ... + + def field(self, i: int | str) -> Field: + ... + + def get_all_field_indices(self, name: str) -> list[int]: + ... + + def __len__(self): + ... + + def __iter__(self) -> Iterator[Any]: + ... + + def __getitem__(self, i): + ... + + def __reduce__(self) -> tuple: + ... + + @property + def names(self) -> list: + ... + + @property + def fields(self) -> list: + ... + +class UnionType(DataType): + + @property + def mode(self) -> str: + ... + + @property + def type_codes(self): + ... + + def __len__(self): + ... + + def __iter__(self) -> Iterator[Any]: + ... + + def field(self, i: int) -> Field: + ... + + def __getitem__(self, i): + ... + + def __reduce__(self) -> tuple: + ... + +class SparseUnionType(UnionType): + ... + +class DenseUnionType(UnionType): + ... + +class TimestampType(DataType): + + @property + def unit(self): + ... + + @property + def tz(self) -> str | None: + ... + + def __reduce__(self) -> tuple: + ... + +class Time32Type(DataType): + + @property + def unit(self): + ... + +class Time64Type(DataType): + + @property + def unit(self): + ... + +class DurationType(DataType): + + @property + def unit(self): + ... + +class FixedSizeBinaryType(DataType): + + def __reduce__(self) -> tuple: + ... + +class Decimal32Type(FixedSizeBinaryType): + + def __reduce__(self) -> tuple: + ... + + @property + def precision(self): + ... + + @property + def scale(self): + ... + +class Decimal64Type(FixedSizeBinaryType): + + def __reduce__(self) -> tuple: + ... + + @property + def precision(self): + ... + + @property + def scale(self): + ... + +class Decimal128Type(FixedSizeBinaryType): + + def __reduce__(self) -> tuple: + ... + + @property + def precision(self): + ... + + @property + def scale(self): + ... + +class Decimal256Type(FixedSizeBinaryType): + + def __reduce__(self) -> tuple: + ... + + @property + def precision(self): + ... + + @property + def scale(self): + ... + +class RunEndEncodedType(DataType): + + def __reduce__(self) -> tuple: + ... + + @property + def run_end_type(self) -> DataType: + ... + + @property + def value_type(self) -> DataType: + ... + +class BaseExtensionType(DataType): + + def __arrow_ext_class__(self): + ... + + def __arrow_ext_scalar_class__(self): + ... + + @property + def extension_name(self) -> str: + ... + + @property + def storage_type(self) -> DataType: + ... + + @property + def byte_width(self): + ... + + @property + def bit_width(self): + ... + + def wrap_array(self, storage: Array | ChunkedArray) -> Array | ChunkedArray: + ... + +class ExtensionType(BaseExtensionType): + + def __init__(self, storage_type: DataType, extension_name): + ... + + def __eq__(self, other): + ... + + def __repr__(self) -> str: + ... + + def __arrow_ext_serialize__(self): + ... + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + ... + + def __reduce__(self) -> tuple: + ... + + def __arrow_ext_class__(self): + ... + + def __arrow_ext_scalar_class__(self): + ... + +class JsonType(BaseExtensionType): + + def __arrow_ext_class__(self): + ... + + def __reduce__(self) -> tuple: + ... + + def __arrow_ext_scalar_class__(self): + ... + +class UuidType(BaseExtensionType): + + def __arrow_ext_class__(self): + ... + + def __reduce__(self) -> tuple: + ... + + def __arrow_ext_scalar_class__(self): + ... + +class FixedShapeTensorType(BaseExtensionType): + + @property + def value_type(self) -> DataType: + ... + + @property + def shape(self): + ... + + @property + def dim_names(self) -> list | None: + ... + + @property + def permutation(self): + ... + + def __arrow_ext_class__(self): + ... + + def __reduce__(self) -> tuple: + ... + + def __arrow_ext_scalar_class__(self): + ... + +class Bool8Type(BaseExtensionType): + + def __arrow_ext_class__(self): + ... + + def __reduce__(self) -> tuple: + ... + + def __arrow_ext_scalar_class__(self): + ... + +class OpaqueType(BaseExtensionType): + + @property + def type_name(self) -> str: + ... + + @property + def vendor_name(self) -> str: + ... + + def __arrow_ext_class__(self): + ... + + def __reduce__(self) -> tuple: + ... + + def __arrow_ext_scalar_class__(self): + ... + +class UnknownExtensionType(ExtensionType): + + def __init__(self, storage_type: DataType, serialized): + ... + + def __arrow_ext_serialize__(self): + ... + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized) -> UnknownExtensionType: + ... + +class _Metadata(_Weakrefable): + ... + +class KeyValueMetadata(_Metadata, Mapping): + + def __init__(self, __arg0__=None, **kwargs): + ... + + def equals(self, other: KeyValueMetadata) -> bool: + ... + + def __repr__(self) -> str: + ... + + def __str__(self) -> str: + ... + + def __eq__(self, other): + ... + + def __len__(self): + ... + + def __contains__(self, key): + ... + + def __getitem__(self, key): + ... + + def __iter__(self): + ... + + def __reduce__(self) -> tuple: + ... + + def key(self, i: int): + ... + + def value(self, i: int): + ... + + def keys(self) -> Iterator[Any]: + ... + + def values(self) -> Iterator[Any]: + ... + + def items(self) -> Iterator[tuple]: + ... + + def get_all(self, key: str) -> list: + ... + + def to_dict(self): + ... + +class Field(_Weakrefable): + type: DataType + + def __init__(self): + ... + + def equals(self, other: Field, check_metadata: bool=False) -> bool: + ... + + def __eq__(self, other): + ... + + def __reduce__(self) -> tuple: + ... + + def __str__(self) -> str: + ... + + def __repr__(self): + ... + + def __hash__(self): + ... + + @property + def nullable(self): + ... + + @property + def name(self) -> str: + ... + + @property + def metadata(self) -> dict | None: + ... + + def with_metadata(self, metadata: dict) -> Field: + ... + + def remove_metadata(self) -> Field: + ... + + def with_type(self, new_type: DataType) -> Field: + ... + + def with_name(self, name: str) -> Field: + ... + + def with_nullable(self, nullable: bool) -> Field: + ... + + def flatten(self) -> list: + ... + + def _export_to_c(self, out_ptr) -> None: + ... + + @staticmethod + def _import_from_c(in_ptr) -> Field: + ... + + def __arrow_c_schema__(self): + ... + + @staticmethod + def _import_from_c_capsule(schema) -> Field: + ... + +class Schema(_Weakrefable): + + def __init__(self): + ... + + def __len__(self): + ... + + def __getitem__(self, key): + ... + + def __iter__(self) -> Iterator[Any]: + ... + + def __reduce__(self) -> tuple: + ... + + def __hash__(self): + ... + + def __sizeof__(self): + ... + + @property + def pandas_metadata(self): + ... + + @property + def names(self) -> list: + ... + + @property + def types(self) -> list: + ... + + @property + def metadata(self): + ... + + def __eq__(self, other): + ... + + def empty_table(self) -> Table: + ... + + def equals(self, other: Schema, check_metadata: bool=False) -> bool: + ... + + @classmethod + def from_pandas(cls, df, preserve_index: bool | None=None) -> Schema: + ... + + def field(self, i: int | str) -> Field: + ... + + def _field(self, i: int) -> Field: + ... + + def field_by_name(self, name: str) -> None | Field: + ... + + def get_field_index(self, name: str) -> int: + ... + + def get_all_field_indices(self, name: str) -> list[int]: + ... + + def append(self, field: Field): + ... + + def insert(self, i: int, field: Field) -> Schema: + ... + + def remove(self, i: int) -> Schema: + ... + + def set(self, i: int, field: Field) -> Schema: + ... + + def add_metadata(self, metadata: dict): + ... + + def with_metadata(self, metadata: dict) -> Schema: + ... + + def serialize(self, memory_pool: MemoryPool | None=None) -> Buffer: + ... + + def remove_metadata(self) -> Schema: + ... + + def to_string(self, truncate_metadata: bool=True, show_field_metadata: bool=True, show_schema_metadata: bool=True, element_size_limit: int=100) -> str: + ... + + def _export_to_c(self, out_ptr) -> None: + ... + + @staticmethod + def _import_from_c(in_ptr) -> Schema: + ... + + def __str__(self): + ... + + def __repr__(self): + ... + + def __arrow_c_schema__(self): + ... + + @staticmethod + def _import_from_c_capsule(schema) -> Schema: + ... + +class _ExtensionRegistryNanny(_Weakrefable): + + def __init__(self): + ... + + def release_registry(self) -> None: + ... + +class Scalar(_Weakrefable): + + def __init__(self): + ... + + @property + def type(self) -> DataType: + ... + + @property + def is_valid(self): + ... + + def cast(self, target_type: object | None=None, safe: bool | None=None, options: pyarrow.compute.CastOptions | None=None, memory_pool: MemoryPool | None=None): + ... + + def validate(self, *, full: bool=False) -> None: + ... + + def __repr__(self) -> str: + ... + + def __str__(self) -> str: + ... + + def equals(self, other: Scalar) -> bool: + ... + + def __eq__(self, other): + ... + + def __hash__(self): + ... + + def __reduce__(self) -> tuple: + ... + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + + def __abs__(self): + ... + + def __add__(self, other: object): + ... + + def __radd__(self, other: object): + ... + + def __truediv__(self, other: object): + ... + + def __rtruediv__(self, other: object): + ... + + def __mul__(self, other: object): + ... + + def __rmul__(self, other: object): + ... + + def __neg__(self): + ... + + def __pow__(self, other: object, mod: None=None): + ... + + def __rpow__(self, other: object, mod: None=None): + ... + + def __sub__(self, other: object): + ... + + def __rsub__(self, other: object): + ... + + def __and__(self, other: object): + ... + + def __rand__(self, other: object): + ... + + def __or__(self, other: object): + ... + + def __ror__(self, other: object): + ... + + def __xor__(self, other: object): + ... + + def __rxor__(self, other: object): + ... + + def __lshift__(self, other: object): + ... + + def __rlshift__(self, other: object): + ... + + def __rshift__(self, other: object): + ... + + def __rrshift__(self, other: object): + ... + +class NullScalar(Scalar): + + def __init__(self): + ... + + def as_py(self, *, maps_as_pydicts: str | None=None) -> None: + ... + +class BooleanScalar(Scalar): + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + + def __bool__(self): + ... + +class UInt8Scalar(Scalar): + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + + def __index__(self): + ... + +class Int8Scalar(Scalar): + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + + def __index__(self): + ... + +class UInt16Scalar(Scalar): + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + + def __index__(self): + ... + +class Int16Scalar(Scalar): + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + + def __index__(self): + ... + +class UInt32Scalar(Scalar): + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + + def __index__(self): + ... + +class Int32Scalar(Scalar): + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + + def __index__(self): + ... + +class UInt64Scalar(Scalar): + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + + def __index__(self): + ... + +class Int64Scalar(Scalar): + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + + def __index__(self): + ... + +class HalfFloatScalar(Scalar): + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + + def __float__(self): + ... + + def __int__(self) -> int: + ... + +class FloatScalar(Scalar): + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + + def __float__(self): + ... + + def __int__(self) -> int: + ... + +class DoubleScalar(Scalar): + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + + def __float__(self): + ... + + def __int__(self) -> int: + ... + +class Decimal32Scalar(Scalar): + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + +class Decimal64Scalar(Scalar): + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + +class Decimal128Scalar(Scalar): + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + +class Decimal256Scalar(Scalar): + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + +class Date32Scalar(Scalar): + + @property + def value(self): + ... + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + +class Date64Scalar(Scalar): + + @property + def value(self): + ... + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + +class Time32Scalar(Scalar): + + @property + def value(self): + ... + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + +class Time64Scalar(Scalar): + + @property + def value(self): + ... + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + +class TimestampScalar(Scalar): + + @property + def value(self): + ... + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + + def __repr__(self) -> str: + ... + +class DurationScalar(Scalar): + + @property + def value(self): + ... + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + +class MonthDayNanoIntervalScalar(Scalar): + + @property + def value(self): + ... + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + +class BinaryScalar(Scalar): + + def as_buffer(self) -> Buffer | None: + ... + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + + def __bytes__(self): + ... + +class LargeBinaryScalar(BinaryScalar): + ... + +class FixedSizeBinaryScalar(BinaryScalar): + ... + +class StringScalar(BinaryScalar): + + def as_py(self, *, maps_as_pydicts: str | None=None) -> None | str: + ... + +class LargeStringScalar(StringScalar): + ... + +class BinaryViewScalar(BinaryScalar): + ... + +class StringViewScalar(StringScalar): + ... + +class ListScalar(Scalar, Sequence): -class LargeListArray(BaseListArray): - @staticmethod - def from_arrays(offsets, values, type: DataType | None=None, pool: MemoryPool | None=None, mask=None): ... @property - def values(self): ... + def values(self) -> Array | None: + ... + + def __len__(self) -> int: + ... + + def __getitem__(self, i): + ... + + def __iter__(self): + ... + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + +class FixedSizeListScalar(ListScalar): + ... + +class LargeListScalar(ListScalar): + ... + +class ListViewScalar(ListScalar): + ... + +class LargeListViewScalar(ListScalar): + ... + +class StructScalar(Scalar, Mapping): + + def __len__(self): + ... + + def __iter__(self) -> Iterator[str]: + ... + + def items(self): + ... + + def __contains__(self, key) -> bool: + ... + + def __getitem__(self, key) -> Scalar: + ... + + def as_py(self, *, maps_as_pydicts: str | None=None) -> dict | None: + ... + + def _as_py_tuple(self) -> list | None: + ... + + def __repr__(self) -> str: + ... + + def __str__(self) -> str: + ... + +class MapScalar(ListScalar, Mapping): + + def __getitem__(self, i): + ... + + def __iter__(self) -> Iterator[tuple]: + ... + + def as_py(self, *, maps_as_pydicts: str | None=None) -> None | list | dict: + ... + + def keys(self) -> list: + ... + +class DictionaryScalar(Scalar): + + @staticmethod + def _reconstruct(type, is_valid, index, dictionary) -> Scalar: + ... + + def __reduce__(self) -> tuple: + ... + + @property + def index(self) -> Scalar: + ... + + @property + def value(self) -> Scalar: + ... + + @property + def dictionary(self) -> Array: + ... + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + +class RunEndEncodedScalar(Scalar): + + @property + def value(self) -> Scalar: + ... + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + +class UnionScalar(Scalar): + + @property + def value(self) -> Scalar | None: + ... + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + + @property + def type_code(self): + ... + +class ExtensionScalar(Scalar): + + @property + def value(self) -> Scalar | None: + ... + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + + @staticmethod + def from_storage(typ: BaseExtensionType, value: object) -> Scalar: + ... + +class JsonScalar(ExtensionScalar): + ... + +class UuidScalar(ExtensionScalar): + + def as_py(self, *, maps_as_pydicts: str | None=None): + ... + +class FixedShapeTensorScalar(ExtensionScalar): + + def to_numpy(self): + ... + + def to_tensor(self) -> Tensor: + ... + + def __dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None): + ... + + def __dlpack_device__(self) -> tuple: + ... + +class OpaqueScalar(ExtensionScalar): + ... + +class Bool8Scalar(ExtensionScalar): + + def as_py(self, *, maps_as_pydicts: str | None=None) -> None | bool: + ... + +class ArrayStatistics(_Weakrefable): + + def __init__(self): + ... + + def __repr__(self) -> str: + ... + + @property + def null_count(self): + ... + + @property + def is_null_count_exact(self): + ... + + @property + def distinct_count(self): + ... + + @property + def is_distinct_count_exact(self): + ... + + @property + def min(self): + ... + + @property + def is_min_exact(self): + ... + + @property + def max(self): + ... + + @property + def is_max_exact(self): + ... + +class _PandasConvertible(_Weakrefable): + + def to_pandas(self, memory_pool: MemoryPool | None=None, categories: list | None=None, strings_to_categorical: bool=False, zero_copy_only: bool=False, integer_object_nulls: bool=False, date_as_object: bool=True, timestamp_as_object: bool=False, use_threads: bool=True, deduplicate_objects: bool=True, ignore_metadata: bool=False, safe: bool=True, split_blocks: bool=False, self_destruct: bool=False, maps_as_pydicts: str | None=None, types_mapper: Callable[..., Any] | None=None, coerce_temporal_nanoseconds: bool=False): + ... + +class Array(_PandasConvertible): + type: DataType + _name: object + + def __init__(self): + ... + + def _debug_print(self) -> None: + ... + + def diff(self, other: Array) -> str: + ... + + def cast(self, target_type: object | None=None, safe: bool | None=None, options: pyarrow.compute.CastOptions | None=None, memory_pool: MemoryPool | None=None) -> Array: + ... + + def view(self, target_type: object) -> Array: + ... + + def sum(self, **kwargs) -> Scalar: + ... + + def unique(self) -> Array: + ... + + def dictionary_encode(self, null_encoding: str='mask') -> DictionaryArray: + ... + + def value_counts(self) -> StructArray: + ... + + @staticmethod + def from_pandas(obj, mask=None, type: DataType | None=None, safe: bool=True, memory_pool: MemoryPool | None=None) -> Array | ChunkedArray: + ... + + def __reduce__(self) -> tuple: + ... + + @staticmethod + def from_buffers(type: DataType, length: int, buffers, null_count: int=-1, offset: int=0, children: list[Array] | None=None) -> Array: + ... + + @property + def null_count(self): + ... + + @property + def nbytes(self): + ... + + def get_total_buffer_size(self) -> int: + ... + + def __sizeof__(self): + ... + + def __iter__(self) -> Iterator[Any]: + ... + + def __repr__(self) -> str: + ... + + def to_string(self, *, indent: int=2, top_level_indent: int=0, window: int=10, container_window: int=2, skip_new_lines: bool=False, element_size_limit: int=100) -> str: + ... + + def __str__(self): + ... + + def __eq__(self, other): + ... + + def equals(self: Array, other: Array) -> bool: + ... + + def __len__(self): + ... + + def is_null(self, *, nan_is_null: bool=False): + ... + + def is_nan(self): + ... + + def is_valid(self): + ... + + def fill_null(self, fill_value: Any) -> Array: + ... + + def __getitem__(self, key) -> Scalar | Array: + ... + + def slice(self, offset: int | Scalar=0, length: int | Scalar | None=None) -> Array: + ... + + def take(self, indices: object) -> Array: + ... + + def drop_null(self): + ... + + def filter(self, mask: object, *, null_selection_behavior: str='drop') -> Array: + ... + + def index(self, value: Scalar | object, start: int | None=None, end: int | None=None, *, memory_pool: MemoryPool | None=None) -> Int64Scalar: + ... + + def sort(self, order: str='ascending', null_placement: str='at_end', **kwargs) -> Array: + ... + + def _to_pandas(self, options, types_mapper=None, **kwargs): + ... + + def __array__(self, dtype=None, copy=None): + ... + + def to_numpy(self, zero_copy_only: bool=True, writable: bool=False): + ... + + def to_tensor(self, *, allow_nulls: bool=False) -> Tensor: + ... + + def to_pylist(self, *, maps_as_pydicts: str | None=None) -> list: + ... + + def tolist(self): + ... + + def validate(self, *, full: bool=False) -> None: + ... + + @property + def offset(self): + ... + + def buffers(self) -> list: + ... + + def copy_to(self, destination: MemoryManager | Device) -> Array: + ... + + def _export_to_c(self, out_ptr, out_schema_ptr: int=0) -> None: + ... + + @staticmethod + def _import_from_c(in_ptr, type) -> Array: + ... + + def __arrow_c_array__(self, requested_schema=None) -> tuple: + ... + + @staticmethod + def _import_from_c_capsule(schema_capsule, array_capsule) -> Array: + ... + + def _export_to_c_device(self, out_ptr, out_schema_ptr: int=0) -> None: + ... + + @staticmethod + def _import_from_c_device(in_ptr, type) -> Array: + ... + + def __arrow_c_device_array__(self, requested_schema=None, **kwargs) -> tuple: + ... + + @staticmethod + def _import_from_c_device_capsule(schema_capsule, array_capsule) -> Array: + ... + + @staticmethod + def from_dlpack(x: object, /, *, device=None, copy: bool | None=None) -> Array: + ... + + def __dlpack__(self, *, stream: int | None=None, max_version=None, dl_device=None, copy: bool | None=None): + ... + + def __dlpack_device__(self) -> tuple: + ... + + @property + def device_type(self) -> DeviceAllocationType: + ... + + @property + def is_cpu(self) -> bool: + ... + + @property + def statistics(self) -> None | ArrayStatistics: + ... + + def __abs__(self): + ... + + def __add__(self, other: object): + ... + + def __radd__(self, other: object): + ... + + def __truediv__(self, other: object): + ... + + def __rtruediv__(self, other: object): + ... + + def __mul__(self, other: object): + ... + + def __rmul__(self, other: object): + ... + + def __neg__(self): + ... + + def __pow__(self, other: object, mod: None=None): + ... + + def __rpow__(self, other: object, mod: None=None): + ... + + def __sub__(self, other: object): + ... + + def __rsub__(self, other: object): + ... + + def __and__(self, other: object): + ... + + def __rand__(self, other: object): + ... + + def __or__(self, other: object): + ... + + def __ror__(self, other: object): + ... + + def __xor__(self, other: object): + ... + + def __rxor__(self, other: object): + ... + + def __lshift__(self, other: object): + ... + + def __rlshift__(self, other: object): + ... + + def __rshift__(self, other: object): + ... + + def __rrshift__(self, other: object): + ... + +class NullArray(Array): + ... + +class BooleanArray(Array): + + @property + def false_count(self): + ... + + @property + def true_count(self): + ... + +class NumericArray(Array): + ... + +class IntegerArray(NumericArray): + ... + +class FloatingPointArray(NumericArray): + ... + +class Int8Array(IntegerArray): + ... + +class UInt8Array(IntegerArray): + ... + +class Int16Array(IntegerArray): + ... + +class UInt16Array(IntegerArray): + ... + +class Int32Array(IntegerArray): + ... + +class UInt32Array(IntegerArray): + ... + +class Int64Array(IntegerArray): + ... + +class UInt64Array(IntegerArray): + ... + +class Date32Array(NumericArray): + ... + +class Date64Array(NumericArray): + ... + +class TimestampArray(NumericArray): + ... + +class Time32Array(NumericArray): + ... + +class Time64Array(NumericArray): + ... + +class DurationArray(NumericArray): + ... + +class MonthDayNanoIntervalArray(Array): + + def to_pylist(self, *, maps_as_pydicts: str | None=None) -> list: + ... + +class HalfFloatArray(FloatingPointArray): + ... + +class FloatArray(FloatingPointArray): + ... + +class DoubleArray(FloatingPointArray): + ... + +class FixedSizeBinaryArray(Array): + ... + +class Decima32Array(FixedSizeBinaryArray): + ... + +class Decimal64Array(FixedSizeBinaryArray): + ... + +class Decimal128Array(FixedSizeBinaryArray): + ... + +class Decimal256Array(FixedSizeBinaryArray): + ... + +class BaseListArray(Array): + + def flatten(self, recursive: bool=False) -> Array: + ... + + def value_parent_indices(self): + ... + + def value_lengths(self): + ... + +class ListArray(BaseListArray): + + @staticmethod + def from_arrays(offsets: Array, values: Array, type: DataType | None=None, pool: MemoryPool | None=None, mask: Array | None=None) -> Array: + ... + + @property + def values(self) -> Array: + ... + + @property + def offsets(self) -> Array: + ... + +class LargeListArray(BaseListArray): + + @staticmethod + def from_arrays(offsets: Array, values: Array, type: DataType | None=None, pool: MemoryPool | None=None, mask: Array | None=None) -> Array: + ... + + @property + def values(self) -> Array: + ... + @property - def offsets(self): ... + def offsets(self) -> Array: + ... class ListViewArray(BaseListArray): + @staticmethod - def from_arrays(offsets, sizes, values, type: DataType | None=None, pool: MemoryPool | None=None, mask=None): ... + def from_arrays(offsets: Array, sizes: Array, values: Array, type: DataType | None=None, pool: MemoryPool | None=None, mask: Array | None=None) -> Array: + ... + @property - def values(self): ... + def values(self) -> Array: + ... + @property - def offsets(self): ... + def offsets(self) -> Array: + ... + @property - def sizes(self): ... + def sizes(self) -> Array: + ... class LargeListViewArray(BaseListArray): + @staticmethod - def from_arrays(offsets, sizes, values, type: DataType | None=None, pool: MemoryPool | None=None, mask=None): ... + def from_arrays(offsets: Array, sizes: Array, values: Array, type: DataType | None=None, pool: MemoryPool | None=None, mask: Array | None=None) -> Array: + ... + @property - def values(self): ... + def values(self) -> Array: + ... + @property - def offsets(self): ... + def offsets(self) -> Array: + ... + @property - def sizes(self): ... + def sizes(self) -> Array: + ... class MapArray(ListArray): + @staticmethod - def from_arrays(offsets, keys, items, type: DataType | None=None, pool: MemoryPool | None=None, mask=None): ... + def from_arrays(offsets, keys, items, type: DataType | None=None, pool: MemoryPool | None=None, mask: Array | None=None) -> Array: + ... + @property - def keys(self): ... + def keys(self) -> Array: + ... + @property - def items(self): ... + def items(self) -> Array: + ... class FixedSizeListArray(BaseListArray): + @staticmethod - def from_arrays(values, list_size=None, type: DataType | None=None, mask=None): ... + def from_arrays(values: Array, list_size: int | None=None, type: DataType | None=None, mask: Array | None=None) -> Array: + ... + @property - def values(self): ... + def values(self) -> Array: + ... class UnionArray(Array): - def child(self, pos: int): ... - def field(self, pos: int): ... + + def child(self, pos: int) -> Field: + ... + + def field(self, pos: int) -> Array: + ... + @property - def type_codes(self): ... + def type_codes(self) -> Array: + ... + @property - def offsets(self): ... + def offsets(self) -> Array: + ... + @staticmethod - def from_dense(types: Array, value_offsets: Array, children: list[Any], field_names: list[Any] | None=None, type_codes: list[Any] | None=None): ... + def from_dense(types: Array, value_offsets: Array, children: list[Any], field_names: list[Any] | None=None, type_codes: list[Any] | None=None) -> Array: + ... + @staticmethod - def from_sparse(types: Array, children: list[Any], field_names: list[Any] | None=None, type_codes: list[Any] | None=None): ... + def from_sparse(types: Array, children: list[Any], field_names: list[Any] | None=None, type_codes: list[Any] | None=None) -> Array: + ... class StringArray(Array): + @staticmethod - def from_buffers(length: int, value_offsets: Buffer, data: Buffer, null_bitmap: Buffer | None=None, null_count: int=-1, offset: int=0): ... + def from_buffers(length: int, value_offsets: Buffer, data: Buffer, null_bitmap: Buffer | None=None, null_count: int=-1, offset: int=0) -> Array: + ... class LargeStringArray(Array): + @staticmethod - def from_buffers(length: int, value_offsets: Buffer, data: Buffer, null_bitmap: Buffer | None=None, null_count: int=-1, offset: int=0): ... + def from_buffers(length: int, value_offsets: Buffer, data: Buffer, null_bitmap: Buffer | None=None, null_count: int=-1, offset: int=0) -> Array: + ... -class StringViewArray(Array): ... +class StringViewArray(Array): + ... class BinaryArray(Array): + @property - def total_values_length(self): ... + def total_values_length(self): + ... class LargeBinaryArray(Array): + @property - def total_values_length(self): ... + def total_values_length(self): + ... -class BinaryViewArray(Array): ... +class BinaryViewArray(Array): + ... class DictionaryArray(Array): - def dictionary_encode(self): ... - def dictionary_decode(self): ... + + def dictionary_encode(self) -> DictionaryArray: + ... + + def dictionary_decode(self): + ... + @property - def dictionary(self): ... + def dictionary(self): + ... + @property - def indices(self): ... + def indices(self): + ... + @staticmethod - def from_buffers(type: DataType, length: int, buffers, dictionary: Array, null_count: int=-1, offset: int=0): ... + def from_buffers(type: DataType, length: int, buffers, dictionary: Array, null_count: int=-1, offset: int=0) -> Array: + ... + @staticmethod - def from_arrays(indices, dictionary, mask=None, ordered: bool=False, from_pandas: bool=False, safe: bool=True, memory_pool: MemoryPool | None=None): ... + def from_arrays(indices, dictionary, mask=None, ordered: bool=False, from_pandas: bool=False, safe: bool=True, memory_pool: MemoryPool | None=None) -> Array: + ... class StructArray(Array): - def field(self, index): ... - def _flattened_field(self, index, memory_pool: MemoryPool | None=None): ... - def flatten(self, memory_pool: MemoryPool | None=None): ... + + def field(self, index) -> Array: + ... + + def _flattened_field(self, index, memory_pool: MemoryPool | None=None) -> Array: + ... + + def flatten(self, memory_pool: MemoryPool | None=None) -> list: + ... + @staticmethod - def from_arrays(arrays, names=None, fields=None, mask=None, memory_pool=None, type=None): ... - def sort(self, order='ascending', null_placement='at_end', by=None, **kwargs): ... + def from_arrays(arrays: Sequence[Array], names: list[str] | None=None, fields: list[Field] | None=None, mask=None, memory_pool: MemoryPool | None=None, type: StructType | None=None) -> StructArray: + ... + + def sort(self, order: str='ascending', null_placement: str='at_end', by: str | None=None, **kwargs) -> StructArray: + ... class RunEndEncodedArray(Array): + @staticmethod - def _from_arrays(type, allow_none_for_type, logical_length, run_ends, values, logical_offset): ... + def _from_arrays(type, allow_none_for_type, logical_length, run_ends, values, logical_offset) -> Array: + ... + @staticmethod - def from_arrays(run_ends, values, type=None): ... + def from_arrays(run_ends, values: Array, type: DataType | None=None) -> RunEndEncodedArray: + ... + @staticmethod - def from_buffers(type: DataType, length, buffers, null_count=-1, offset=0, children=None): ... + def from_buffers(type: DataType, length: int, buffers: list[Buffer], null_count: int=-1, offset: int=0, children: list[Array] | None=None) -> RunEndEncodedArray: + ... + @property - def run_ends(self): ... + def run_ends(self) -> Array: + ... + @property - def values(self): ... - def find_physical_offset(self): ... - def find_physical_length(self): ... + def values(self) -> Array: + ... + + def find_physical_offset(self): + ... + + def find_physical_length(self): + ... class ExtensionArray(Array): + @property - def storage(self): ... + def storage(self) -> Array: + ... + @staticmethod - def from_storage(typ: BaseExtensionType, storage: Array): ... + def from_storage(typ: BaseExtensionType, storage: Array) -> Array: + ... -class JsonArray(ExtensionArray): ... +class JsonArray(ExtensionArray): + ... -class UuidArray(ExtensionArray): ... +class UuidArray(ExtensionArray): + ... class FixedShapeTensorArray(ExtensionArray): - def to_numpy_ndarray(self): ... + + def to_numpy_ndarray(self): + ... + @staticmethod - def from_tensor(tensor: Tensor): ... + def from_tensor(tensor: Tensor) -> Array: + ... + @staticmethod - def from_numpy_ndarray(obj, dim_names=None): ... + def from_numpy_ndarray(obj, dim_names: tuple | list[str] | None=None) -> ExtensionArray: + ... + @staticmethod - def from_dlpack(x, /, *, device=None, copy=None): ... - def __dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None): ... + def from_dlpack(x: object, /, *, device=None, copy: bool | None=None) -> FixedShapeTensorArray: + ... -class OpaqueArray(ExtensionArray): ... + def __dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None): + ... + +class OpaqueArray(ExtensionArray): + ... class Bool8Array(ExtensionArray): - def to_numpy(self, zero_copy_only=True, writable=False): ... + + def to_numpy(self, zero_copy_only: bool=True, writable: bool=False): + ... + @staticmethod - def from_storage(storage: Int8Array): ... + def from_storage(storage: Int8Array) -> ExtensionArray: + ... + @staticmethod - def from_numpy(obj): ... + def from_numpy(obj) -> Bool8Array: + ... class StringBuilder(_Weakrefable): - def __init__(self, memory_pool: MemoryPool | None=None): ... - def append(self, value): ... - def append_values(self, values): ... - def finish(self): ... + + def __init__(self, memory_pool: MemoryPool | None=None): + ... + + def append(self, value) -> None: + ... + + def append_values(self, values) -> None: + ... + + def finish(self) -> Array: + ... + @property - def null_count(self): ... - def __len__(self): ... + def null_count(self): + ... + + def __len__(self): + ... class StringViewBuilder(_Weakrefable): - def __init__(self, memory_pool: MemoryPool | None=None): ... - def append(self, value): ... - def append_values(self, values): ... - def finish(self): ... + + def __init__(self, memory_pool: MemoryPool | None=None): + ... + + def append(self, value) -> None: + ... + + def append_values(self, values) -> None: + ... + + def finish(self) -> Array: + ... + @property - def null_count(self): ... - def __len__(self): ... + def null_count(self): + ... + + def __len__(self): + ... class ChunkedArray(_PandasConvertible): _name: object - def __init__(self): ... - def __reduce__(self): ... - @property - def data(self): ... - @property - def type(self): ... - def length(self): ... - def __len__(self): ... - def __repr__(self): ... - def to_string(self, *, indent: int=0, window: int=5, container_window: int=2, skip_new_lines: c_bool=False, element_size_limit: int=100): ... - def format(self, **kwargs): ... - def __str__(self): ... - def validate(self, *, full=False): ... - @property - def null_count(self): ... - @property - def nbytes(self): ... - def get_total_buffer_size(self): ... - def __sizeof__(self): ... - def __iter__(self): ... - def __getitem__(self, key): ... - def is_null(self, *, nan_is_null=False): ... - def is_nan(self): ... - def is_valid(self): ... - def __eq__(self, other): ... - def fill_null(self, fill_value): ... - def equals(self, other: ChunkedArray): ... - def _to_pandas(self, options, types_mapper=None, **kwargs): ... - def to_numpy(self, zero_copy_only=False): ... - def __array__(self, dtype=None, copy=None): ... - def cast(self, target_type: object | None=None, safe=None, options=None): ... - def dictionary_encode(self, null_encoding='mask'): ... - def flatten(self, memory_pool: MemoryPool | None=None): ... - def combine_chunks(self, memory_pool: MemoryPool | None=None): ... - def unique(self): ... - def value_counts(self): ... - def slice(self, offset=0, length=None): ... - def filter(self, mask, null_selection_behavior: object='drop'): ... - def index(self, value, start=None, end=None, *, memory_pool=None): ... - def take(self, indices: object): ... - def drop_null(self): ... - def sort(self, order='ascending', null_placement='at_end', **kwargs): ... - def unify_dictionaries(self, memory_pool: MemoryPool | None=None): ... - @property - def num_chunks(self): ... - def chunk(self, i): ... - @property - def chunks(self): ... - def iterchunks(self): ... - def to_pylist(self, *, maps_as_pydicts=None): ... - def __arrow_c_stream__(self, requested_schema=None): ... - @staticmethod - def _import_from_c_capsule(stream): ... - @property - def is_cpu(self): ... - def _assert_cpu(self): ... + def __init__(self): + ... + + def __reduce__(self) -> tuple: + ... + + @property + def data(self) -> ChunkedArray: + ... + + @property + def type(self) -> DataType: + ... + + def length(self): + ... + + def __len__(self): + ... + + def __repr__(self) -> str: + ... + + def to_string(self, *, indent: int=0, window: int=5, container_window: int=2, skip_new_lines: bool=False, element_size_limit: int=100) -> str: + ... + + def format(self, **kwargs) -> str: + ... + + def __str__(self): + ... + + def validate(self, *, full: bool=False) -> None: + ... + + @property + def null_count(self) -> int: + ... + + @property + def nbytes(self): + ... + + def get_total_buffer_size(self) -> int: + ... + + def __sizeof__(self): + ... + + def __iter__(self) -> Iterator[Any]: + ... + + def __getitem__(self, key) -> Scalar | ChunkedArray: + ... + + def is_null(self, *, nan_is_null: bool=False): + ... + + def is_nan(self): + ... + + def is_valid(self): + ... + + def __eq__(self, other): + ... + + def fill_null(self, fill_value: Any) -> Array | ChunkedArray: + ... + + def equals(self, other: ChunkedArray) -> bool: + ... + + def _to_pandas(self, options, types_mapper=None, **kwargs): + ... + + def to_numpy(self, zero_copy_only: bool=False): + ... + + def __array__(self, dtype=None, copy=None): + ... + + def cast(self, target_type: object | None=None, safe: bool | None=None, options: pyarrow.compute.CastOptions | None=None) -> Array | ChunkedArray: + ... + + def dictionary_encode(self, null_encoding: str='mask') -> ChunkedArray: + ... + + def flatten(self, memory_pool: MemoryPool | None=None) -> list: + ... + + def combine_chunks(self, memory_pool: MemoryPool | None=None) -> Array: + ... + + def unique(self) -> Array: + ... + + def value_counts(self): + ... + + def slice(self, offset: int | Scalar=0, length: int | Scalar | None=None) -> ChunkedArray: + ... + + def filter(self, mask, null_selection_behavior: object='drop') -> Array | ChunkedArray: + ... + + def index(self, value: Scalar | object, start: int | None=None, end: int | None=None, *, memory_pool: MemoryPool | None=None) -> Int64Scalar: + ... + + def take(self, indices: object) -> Array | ChunkedArray: + ... + + def drop_null(self): + ... + + def sort(self, order: str='ascending', null_placement: str='at_end', **kwargs) -> ChunkedArray: + ... + + def unify_dictionaries(self, memory_pool: MemoryPool | None=None) -> ChunkedArray: + ... + + @property + def num_chunks(self) -> int: + ... + + def chunk(self, i: int) -> Array: + ... + + @property + def chunks(self) -> list: + ... + + def iterchunks(self) -> Iterator[Any]: + ... + + def to_pylist(self, *, maps_as_pydicts: str | None=None) -> list: + ... + + def __arrow_c_stream__(self, requested_schema=None): + ... + + @staticmethod + def _import_from_c_capsule(stream) -> ChunkedArray: + ... + + @property + def is_cpu(self): + ... + + def _assert_cpu(self) -> None: + ... class _Tabular(_PandasConvertible): - def __init__(self): ... - def __array__(self, dtype=None, copy=None): ... - def __dataframe__(self, nan_as_null: bool=False, allow_copy: bool=True): ... - def __eq__(self, other): ... - def __getitem__(self, key): ... - def __len__(self): ... - def __repr__(self): ... - def _column(self, i: int): ... - def _ensure_integer_index(self, i): ... - def _is_initialized(self): ... - def column(self, i): ... - @property - def column_names(self): ... - @property - def columns(self): ... - def drop_null(self): ... - def field(self, i): ... + + def __init__(self): + ... + + def __array__(self, dtype=None, copy=None): + ... + + def __dataframe__(self, nan_as_null: bool=False, allow_copy: bool=True): + ... + + def __eq__(self, other): + ... + + def __getitem__(self, key): + ... + + def __len__(self): + ... + + def __repr__(self): + ... + + def _column(self, i: int): + ... + + def _ensure_integer_index(self, i): + ... + + def _is_initialized(self): + ... + + def column(self, i: int | str) -> Array | ChunkedArray: + ... + + @property + def column_names(self) -> list: + ... + + @property + def columns(self) -> list: + ... + + def drop_null(self) -> Table | RecordBatch: + ... + + def field(self, i: int | str) -> Field: + ... + @classmethod - def from_pydict(cls, mapping, schema=None, metadata=None): ... + def from_pydict(cls, mapping: dict | Mapping[Any, Any], schema: Schema | None=None, metadata: dict | Mapping[Any, Any] | None=None) -> Table | RecordBatch: + ... + @classmethod - def from_pylist(cls, mapping, schema=None, metadata=None): ... - def itercolumns(self): ... + def from_pylist(cls, mapping, schema: Schema | None=None, metadata: dict | Mapping[Any, Any] | None=None) -> Table | RecordBatch: + ... + + def itercolumns(self) -> Iterator[Any]: + ... + @property - def num_columns(self): ... + def num_columns(self): + ... + @property - def num_rows(self): ... + def num_rows(self): + ... + @property - def shape(self): ... + def shape(self) -> tuple: + ... + @property - def schema(self): ... - def sort_by(self, sorting, **kwargs): ... - def take(self, indices: object): ... - def filter(self, mask, null_selection_behavior: object='drop'): ... - def to_pydict(self, *, maps_as_pydicts=None): ... - def to_pylist(self, *, maps_as_pydicts=None): ... - def to_string(self, *, show_metadata=False, preview_cols=0): ... - def remove_column(self, i: int): ... - def drop_columns(self, columns): ... - def add_column(self, i: int, field_, column): ... - def append_column(self, field_, column): ... + def schema(self): + ... + + def sort_by(self, sorting, **kwargs) -> Table | RecordBatch: + ... + + def take(self, indices: object) -> Table | RecordBatch: + ... + + def filter(self, mask, null_selection_behavior: object='drop') -> Table | RecordBatch: + ... + + def to_pydict(self, *, maps_as_pydicts: str | None=None) -> dict: + ... + + def to_pylist(self, *, maps_as_pydicts: str | None=None) -> list: + ... + + def to_string(self, *, show_metadata: bool=False, preview_cols: int=0) -> str: + ... + + def remove_column(self, i: int): + ... + + def drop_columns(self, columns: str | list[str]) -> _Tabular: + ... + + def add_column(self, i: int, field_, column): + ... + + def append_column(self, field_: str | Field, column) -> Table | RecordBatch: + ... class RecordBatch(_Tabular): - def __init__(self): ... - def _is_initialized(self): ... - def __reduce__(self): ... - def validate(self, *, full=False): ... - def replace_schema_metadata(self, metadata=None): ... - @property - def num_columns(self): ... - @property - def num_rows(self): ... - @property - def schema(self): ... - def _column(self, i: int): ... - @property - def nbytes(self): ... - def get_total_buffer_size(self): ... - def __sizeof__(self): ... - def add_column(self, i: int, field_, column): ... - def remove_column(self, i: int): ... - def set_column(self, i: int, field_, column): ... - def rename_columns(self, names): ... - def serialize(self, memory_pool=None): ... - def slice(self, offset=0, length=None): ... - def equals(self, other: object, check_metadata: bool=False): ... - def select(self, columns: object): ... - def cast(self, target_schema: Schema, safe=None, options=None): ... - def _to_pandas(self, options, **kwargs): ... + + def __init__(self): + ... + + def _is_initialized(self) -> bool: + ... + + def __reduce__(self) -> tuple: + ... + + def validate(self, *, full: bool=False) -> None: + ... + + def replace_schema_metadata(self, metadata: dict | None=None) -> RecordBatch: + ... + + @property + def num_columns(self) -> int: + ... + + @property + def num_rows(self) -> int: + ... + + @property + def schema(self) -> Schema: + ... + + def _column(self, i: int) -> Array: + ... + + @property + def nbytes(self): + ... + + def get_total_buffer_size(self) -> int: + ... + + def __sizeof__(self): + ... + + def add_column(self, i: int, field_: str | Field, column) -> RecordBatch: + ... + + def remove_column(self, i: int) -> RecordBatch: + ... + + def set_column(self, i: int, field_: str | Field, column) -> RecordBatch: + ... + + def rename_columns(self, names) -> RecordBatch: + ... + + def serialize(self, memory_pool: MemoryPool | None=None) -> Buffer: + ... + + def slice(self, offset: int | Scalar=0, length: int | Scalar | None=None) -> RecordBatch: + ... + + def equals(self, other: object, check_metadata: bool=False) -> bool: + ... + + def select(self, columns: object) -> RecordBatch: + ... + + def cast(self, target_schema: Schema, safe: bool | None=None, options: pyarrow.compute.CastOptions | None=None) -> RecordBatch: + ... + + def _to_pandas(self, options, **kwargs): + ... + @classmethod - def from_pandas(cls, df, schema: Schema | None=None, preserve_index=None, nthreads=None, columns=None): ... + def from_pandas(cls, df, schema: Schema | None=None, preserve_index: bool | None=None, nthreads: int | None=None, columns: list | None=None) -> RecordBatch | Self: + ... + @staticmethod - def from_arrays(arrays: list[Any], names=None, schema=None, metadata=None): ... + def from_arrays(arrays: list[Any], names: list[str] | None=None, schema: Schema | None=None, metadata: dict | Mapping[Any, Any] | None=None) -> RecordBatch: + ... + @staticmethod - def from_struct_array(struct_array: StructArray): ... - def to_struct_array(self): ... - def to_tensor(self, null_to_nan: c_bool=False, row_major: c_bool=True, memory_pool: MemoryPool | None=None): ... - def copy_to(self, destination): ... - def _export_to_c(self, out_ptr, out_schema_ptr=0): ... + def from_struct_array(struct_array: StructArray) -> RecordBatch: + ... + + def to_struct_array(self) -> Array: + ... + + def to_tensor(self, null_to_nan: bool=False, row_major: bool=True, memory_pool: MemoryPool | None=None) -> Tensor: + ... + + def copy_to(self, destination: MemoryManager | Device) -> RecordBatch: + ... + + def _export_to_c(self, out_ptr, out_schema_ptr: int=0) -> None: + ... + @staticmethod - def _import_from_c(in_ptr, schema): ... - def __arrow_c_array__(self, requested_schema=None): ... - def __arrow_c_stream__(self, requested_schema=None): ... + def _import_from_c(in_ptr, schema) -> RecordBatch: + ... + + def __arrow_c_array__(self, requested_schema=None) -> tuple: + ... + + def __arrow_c_stream__(self, requested_schema=None): + ... + @staticmethod - def _import_from_c_capsule(schema_capsule, array_capsule): ... - def _export_to_c_device(self, out_ptr, out_schema_ptr=0): ... + def _import_from_c_capsule(schema_capsule, array_capsule) -> RecordBatch: + ... + + def _export_to_c_device(self, out_ptr, out_schema_ptr: int=0) -> None: + ... + @staticmethod - def _import_from_c_device(in_ptr, schema): ... - def __arrow_c_device_array__(self, requested_schema=None, **kwargs): ... + def _import_from_c_device(in_ptr, schema) -> RecordBatch: + ... + + def __arrow_c_device_array__(self, requested_schema=None, **kwargs) -> tuple: + ... + @staticmethod - def _import_from_c_device_capsule(schema_capsule, array_capsule): ... + def _import_from_c_device_capsule(schema_capsule, array_capsule) -> RecordBatch: + ... + @property - def device_type(self): ... + def device_type(self) -> DeviceAllocationType: + ... + @property - def is_cpu(self): ... + def is_cpu(self) -> bool: + ... class Table(_Tabular): - def __init__(self): ... - def _is_initialized(self): ... - def validate(self, *, full=False): ... - def __reduce__(self): ... - def slice(self, offset=0, length=None): ... - def select(self, columns: object): ... - def replace_schema_metadata(self, metadata=None): ... - def flatten(self, memory_pool: MemoryPool | None=None): ... - def combine_chunks(self, memory_pool: MemoryPool | None=None): ... - def unify_dictionaries(self, memory_pool: MemoryPool | None=None): ... - def equals(self, other: Table, check_metadata: bool=False): ... - def cast(self, target_schema: Schema, safe=None, options=None): ... + + def __init__(self): + ... + + def _is_initialized(self) -> bool: + ... + + def validate(self, *, full: bool=False) -> None: + ... + + def __reduce__(self) -> tuple: + ... + + def slice(self, offset: int | Scalar=0, length: int | Scalar | None=None) -> Table: + ... + + def select(self, columns: object) -> Table: + ... + + def replace_schema_metadata(self, metadata: dict | None=None) -> Table: + ... + + def flatten(self, memory_pool: MemoryPool | None=None) -> Table: + ... + + def combine_chunks(self, memory_pool: MemoryPool | None=None) -> Table: + ... + + def unify_dictionaries(self, memory_pool: MemoryPool | None=None) -> Table: + ... + + def equals(self, other: Table, check_metadata: bool=False) -> bool: + ... + + def cast(self, target_schema: Schema, safe: bool | None=None, options: pyarrow.compute.CastOptions | None=None) -> Table: + ... + @classmethod - def from_pandas(cls, df, schema: Schema | None=None, preserve_index=None, nthreads=None, columns=None, safe: bool=True): ... + def from_pandas(cls, df, schema: Schema | None=None, preserve_index: bool | None=None, nthreads: int | None=None, columns: list | None=None, safe: bool=True) -> Table | Self: + ... + @staticmethod - def from_arrays(arrays, names=None, schema=None, metadata=None): ... + def from_arrays(arrays: list[Array] | ChunkedArray, names: list[str] | None=None, schema: Schema | None=None, metadata: dict | Mapping[Any, Any] | None=None) -> Table: + ... + @staticmethod - def from_struct_array(struct_array): ... - def to_struct_array(self, max_chunksize=None): ... + def from_struct_array(struct_array: StructArray | ChunkedArray) -> Table: + ... + + def to_struct_array(self, max_chunksize: int | None=None) -> ChunkedArray: + ... + @staticmethod - def from_batches(batches, schema: Schema | None=None): ... - def to_batches(self, max_chunksize=None): ... - def to_tensor(self, null_to_nan: c_bool=False, row_major: c_bool=True, memory_pool: MemoryPool | None=None): ... - def to_reader(self, max_chunksize=None): ... - def _to_pandas(self, options, categories=None, ignore_metadata=False, types_mapper=None): ... + def from_batches(batches: Sequence[Any] | Iterator[RecordBatch], schema: Schema | None=None) -> Table: + ... + + def to_batches(self, max_chunksize: int | None=None) -> list[RecordBatch]: + ... + + def to_tensor(self, null_to_nan: bool=False, row_major: bool=True, memory_pool: MemoryPool | None=None) -> Tensor: + ... + + def to_reader(self, max_chunksize: int | None=None) -> RecordBatchReader: + ... + + def _to_pandas(self, options, categories=None, ignore_metadata: bool=False, types_mapper=None): + ... + @property - def schema(self): ... - def _column(self, i: int): ... + def schema(self) -> Schema: + ... + + def _column(self, i: int) -> ChunkedArray: + ... + @property - def num_columns(self): ... + def num_columns(self) -> int: + ... + @property - def num_rows(self): ... + def num_rows(self) -> int: + ... + @property - def nbytes(self): ... - def get_total_buffer_size(self): ... - def __sizeof__(self): ... - def add_column(self, i: int, field_, column): ... - def remove_column(self, i: int): ... - def set_column(self, i: int, field_, column): ... - def rename_columns(self, names): ... - def drop(self, columns): ... - def group_by(self, keys, use_threads=True): ... - def join(self, right_table, keys, right_keys=None, join_type='left outer', left_suffix=None, right_suffix=None, coalesce_keys=True, use_threads=True, filter_expression=None): ... - def join_asof(self, right_table, on, by, tolerance, right_on=None, right_by=None): ... - def __arrow_c_stream__(self, requested_schema=None): ... + def nbytes(self): + ... + + def get_total_buffer_size(self) -> int: + ... + + def __sizeof__(self): + ... + + def add_column(self, i: int, field_: str | Field, column) -> Table: + ... + + def remove_column(self, i: int) -> Table: + ... + + def set_column(self, i: int, field_: str | Field, column) -> Table: + ... + + def rename_columns(self, names) -> Table: + ... + + def drop(self, columns: str | list[str]) -> Table: + ... + + def group_by(self, keys: str | list[str], use_threads: bool=True) -> TableGroupBy: + ... + + def join(self, right_table: Table, keys: str | list[str], right_keys: str | list[str] | None=None, join_type: str='left outer', left_suffix: str | None=None, right_suffix: str | None=None, coalesce_keys: bool=True, use_threads: bool=True, filter_expression: pyarrow.compute.Expression | None=None) -> Table: + ... + + def join_asof(self, right_table: Table, on: str, by: str | list[str], tolerance: int, right_on: str | list[str] | None=None, right_by: str | list[str] | None=None) -> Table: + ... + + def __arrow_c_stream__(self, requested_schema=None): + ... + @property - def is_cpu(self): ... + def is_cpu(self): + ... class TableGroupBy: - def __init__(self, table, keys, use_threads=True): ... - def aggregate(self, aggregations): ... + + def __init__(self, table, keys, use_threads: bool=True): + ... + + def aggregate(self, aggregations) -> Table: + ... class Tensor(_Weakrefable): type: DataType _ssize_t_shape: bytes _ssize_t_strides: bytes - def __init__(self): ... - def _make_shape_or_strides_buffer(self, values): ... - def __repr__(self): ... + def __init__(self): + ... + + def _make_shape_or_strides_buffer(self, values): + ... + + def __repr__(self) -> str: + ... + @staticmethod - def from_numpy(obj, dim_names=None): ... - def to_numpy(self): ... - def equals(self, other: Tensor): ... - def __eq__(self, other): ... - def dim_name(self, i): ... + def from_numpy(obj, dim_names: list | None=None) -> Tensor: + ... + + def to_numpy(self): + ... + + def equals(self, other: Tensor): + ... + + def __eq__(self, other): + ... + + def dim_name(self, i: int) -> str: + ... + @property - def dim_names(self): ... + def dim_names(self) -> list: + ... + @property - def is_mutable(self): ... + def is_mutable(self): + ... + @property - def is_contiguous(self): ... + def is_contiguous(self): + ... + @property - def ndim(self): ... + def ndim(self): + ... + @property - def size(self): ... + def size(self): + ... + @property - def shape(self): ... + def shape(self) -> tuple: + ... + @property - def strides(self): ... + def strides(self) -> tuple: + ... + @staticmethod - def from_dlpack(x, /, *, device=None, copy=None): ... - def __dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None): ... - def __dlpack_device__(self): ... + def from_dlpack(x: object, /, *, device=None, copy: bool | None=None) -> Tensor: + ... + + def __dlpack__(self, *, stream: int | None=None, max_version=None, dl_device=None, copy: bool | None=None): + ... + + def __dlpack_device__(self) -> tuple: + ... class SparseCOOTensor(_Weakrefable): type: DataType - def __init__(self): ... - def __repr__(self): ... + def __init__(self): + ... + + def __repr__(self) -> str: + ... + @classmethod - def from_dense_numpy(cls, obj, dim_names=None): ... + def from_dense_numpy(cls, obj, dim_names: list[str] | None=None) -> Self: + ... + @staticmethod - def from_numpy(data, coords, shape, dim_names=None): ... + def from_numpy(data, coords, shape: tuple, dim_names: list | None=None) -> SparseCOOTensor: + ... + @staticmethod - def from_scipy(obj, dim_names=None): ... + def from_scipy(obj, dim_names: list | None=None) -> SparseCOOTensor: + ... + @staticmethod - def from_pydata_sparse(obj, dim_names=None): ... + def from_pydata_sparse(obj, dim_names: list | None=None) -> SparseCOOTensor: + ... + @staticmethod - def from_tensor(obj): ... - def to_numpy(self): ... - def to_scipy(self): ... - def to_pydata_sparse(self): ... - def to_tensor(self): ... - def equals(self, other: SparseCOOTensor): ... - def __eq__(self, other): ... + def from_tensor(obj: Tensor) -> SparseCOOTensor: + ... + + def to_numpy(self) -> tuple: + ... + + def to_scipy(self): + ... + + def to_pydata_sparse(self): + ... + + def to_tensor(self) -> Tensor: + ... + + def equals(self, other: SparseCOOTensor): + ... + + def __eq__(self, other): + ... + @property - def is_mutable(self): ... + def is_mutable(self): + ... + @property - def ndim(self): ... + def ndim(self): + ... + @property - def shape(self): ... + def shape(self) -> tuple: + ... + @property - def size(self): ... - def dim_name(self, i): ... + def size(self): + ... + + def dim_name(self, i: int) -> str: + ... + @property - def dim_names(self): ... + def dim_names(self) -> tuple: + ... + @property - def non_zero_length(self): ... + def non_zero_length(self): + ... + @property - def has_canonical_format(self): ... + def has_canonical_format(self): + ... class SparseCSRMatrix(_Weakrefable): type: DataType - def __init__(self): ... - def __repr__(self): ... + def __init__(self): + ... + + def __repr__(self) -> str: + ... + @classmethod - def from_dense_numpy(cls, obj, dim_names=None): ... + def from_dense_numpy(cls, obj, dim_names: list | None=None) -> Self: + ... + @staticmethod - def from_numpy(data, indptr, indices, shape, dim_names=None): ... + def from_numpy(data, indptr, indices, shape: tuple, dim_names: list | None=None) -> SparseCSRMatrix: + ... + @staticmethod - def from_scipy(obj, dim_names=None): ... + def from_scipy(obj, dim_names: list | None=None) -> SparseCSRMatrix: + ... + @staticmethod - def from_tensor(obj): ... - def to_numpy(self): ... - def to_scipy(self): ... - def to_tensor(self): ... - def equals(self, other: SparseCSRMatrix): ... - def __eq__(self, other): ... + def from_tensor(obj: Tensor) -> SparseCSRMatrix: + ... + + def to_numpy(self) -> tuple: + ... + + def to_scipy(self): + ... + + def to_tensor(self) -> Tensor: + ... + + def equals(self, other: SparseCSRMatrix): + ... + + def __eq__(self, other): + ... + @property - def is_mutable(self): ... + def is_mutable(self): + ... + @property - def ndim(self): ... + def ndim(self): + ... + @property - def shape(self): ... + def shape(self) -> tuple: + ... + @property - def size(self): ... - def dim_name(self, i): ... + def size(self): + ... + + def dim_name(self, i: int) -> str: + ... + @property - def dim_names(self): ... + def dim_names(self) -> tuple: + ... + @property - def non_zero_length(self): ... + def non_zero_length(self): + ... class SparseCSCMatrix(_Weakrefable): type: DataType - def __init__(self): ... - def __repr__(self): ... + def __init__(self): + ... + + def __repr__(self) -> str: + ... + @classmethod - def from_dense_numpy(cls, obj, dim_names=None): ... + def from_dense_numpy(cls, obj, dim_names: list[str] | None=None) -> Self: + ... + @staticmethod - def from_numpy(data, indptr, indices, shape, dim_names=None): ... + def from_numpy(data, indptr, indices, shape: tuple, dim_names: list | None=None) -> SparseCSCMatrix: + ... + @staticmethod - def from_scipy(obj, dim_names=None): ... + def from_scipy(obj, dim_names: list | None=None) -> SparseCSCMatrix: + ... + @staticmethod - def from_tensor(obj): ... - def to_numpy(self): ... - def to_scipy(self): ... - def to_tensor(self): ... - def equals(self, other: SparseCSCMatrix): ... - def __eq__(self, other): ... + def from_tensor(obj: Tensor) -> SparseCSCMatrix: + ... + + def to_numpy(self) -> tuple: + ... + + def to_scipy(self): + ... + + def to_tensor(self) -> Tensor: + ... + + def equals(self, other: SparseCSCMatrix): + ... + + def __eq__(self, other): + ... + @property - def is_mutable(self): ... + def is_mutable(self): + ... + @property - def ndim(self): ... + def ndim(self): + ... + @property - def shape(self): ... + def shape(self) -> tuple: + ... + @property - def size(self): ... - def dim_name(self, i): ... + def size(self): + ... + + def dim_name(self, i: int) -> str: + ... + @property - def dim_names(self): ... + def dim_names(self) -> tuple: + ... + @property - def non_zero_length(self): ... + def non_zero_length(self): + ... class SparseCSFTensor(_Weakrefable): type: DataType - def __init__(self): ... - def __repr__(self): ... + def __init__(self): + ... + + def __repr__(self) -> str: + ... + @classmethod - def from_dense_numpy(cls, obj, dim_names=None): ... + def from_dense_numpy(cls, obj, dim_names: list[str] | None=None) -> Self: + ... + @staticmethod - def from_numpy(data, indptr, indices, shape, axis_order=None, dim_names=None): ... + def from_numpy(data, indptr, indices, shape: tuple, axis_order: list | None=None, dim_names: list | None=None) -> SparseCSFTensor: + ... + @staticmethod - def from_tensor(obj): ... - def to_numpy(self): ... - def to_tensor(self): ... - def equals(self, other: SparseCSFTensor): ... - def __eq__(self, other): ... + def from_tensor(obj: Tensor) -> SparseCSFTensor: + ... + + def to_numpy(self) -> tuple: + ... + + def to_tensor(self) -> Tensor: + ... + + def equals(self, other: SparseCSFTensor): + ... + + def __eq__(self, other): + ... + @property - def is_mutable(self): ... + def is_mutable(self): + ... + @property - def ndim(self): ... + def ndim(self): + ... + @property - def shape(self): ... + def shape(self) -> tuple: + ... + @property - def size(self): ... - def dim_name(self, i): ... + def size(self): + ... + + def dim_name(self, i: int) -> str: + ... + @property - def dim_names(self): ... + def dim_names(self) -> tuple: + ... + @property - def non_zero_length(self): ... + def non_zero_length(self): + ... class NativeFile(_Weakrefable): - _default_chunk_size = 256 * 1024 - - def __init__(self): ... - def __enter__(self): ... - def __exit__(self, exc_type, exc_value, tb): ... - def __repr__(self): ... - @property - def mode(self): ... - def readable(self): ... - def writable(self): ... - def seekable(self): ... - def isatty(self): ... - def fileno(self): ... - @property - def closed(self): ... - def close(self): ... - def _assert_open(self): ... - def _assert_readable(self): ... - def _assert_writable(self): ... - def _assert_seekable(self): ... - def size(self): ... - def metadata(self): ... - def tell(self): ... - def seek(self, position: int, whence: int=0): ... - def flush(self): ... - def write(self, data): ... - def read(self, nbytes=None): ... - def get_stream(self, file_offset, nbytes): ... - def read_at(self, nbytes, offset): ... - def read1(self, nbytes=None): ... - def readall(self): ... - def readinto(self, b): ... - def readline(self, size=None): ... - def readlines(self, hint=None): ... - def __iter__(self): ... - def __next__(self): ... - def read_buffer(self, nbytes=None): ... - def truncate(self): ... - def writelines(self, lines): ... - def download(self, stream_or_path, buffer_size=None): ... - def _download_nothreads(self, stream_or_path, buffer_size=None): ... - def upload(self, stream, buffer_size=None): ... - def _upload_nothreads(self, stream, buffer_size=None): ... + _default_chunk_size: int + + def __init__(self): + ... + + def __enter__(self) -> Self: + ... + + def __exit__(self, exc_type, exc_value, tb) -> None: + ... + + def __repr__(self) -> str: + ... + + @property + def mode(self) -> str: + ... + + def readable(self): + ... + + def writable(self): + ... + + def seekable(self): + ... + + def isatty(self) -> bool: + ... + + def fileno(self): + ... + + @property + def closed(self): + ... + + def close(self) -> None: + ... + + def _assert_open(self) -> None: + ... + + def _assert_readable(self) -> None: + ... + + def _assert_writable(self) -> None: + ... + + def _assert_seekable(self) -> None: + ... + + def size(self) -> int: + ... + + def metadata(self) -> dict: + ... + + def tell(self) -> int: + ... + + def seek(self, position: int, whence: int=0) -> int: + ... + + def flush(self) -> None: + ... + + def write(self, data) -> int: + ... + + def read(self, nbytes: int | None=None) -> bytes: + ... + + def get_stream(self, file_offset: int, nbytes: int) -> NativeFile: + ... + + def read_at(self, nbytes: int, offset: int) -> bytes: + ... + + def read1(self, nbytes: int | None=None) -> bytes: + ... + + def readall(self): + ... + + def readinto(self, b) -> int: + ... + + def readline(self, size: int | None=None): + ... + + def readlines(self, hint: int | None=None): + ... + + def __iter__(self) -> Self: + ... + + def __next__(self): + ... + + def read_buffer(self, nbytes: int | None=None): + ... + + def truncate(self): + ... + + def writelines(self, lines: Iterable[Any]) -> None: + ... + + def download(self, stream_or_path: str | IO[Any], buffer_size: int | None=None): + ... + + def _download_nothreads(self, stream_or_path, buffer_size=None) -> None: + ... + + def upload(self, stream: IO[Any], buffer_size: int | None=None): + ... + + def _upload_nothreads(self, stream, buffer_size=None) -> None: + ... class PythonFile(NativeFile): - def __init__(self, handle, mode=None): ... - def truncate(self, pos=None): ... - def readline(self, size=None): ... - def readlines(self, hint=None): ... + + def __init__(self, handle, mode=None): + ... + + def truncate(self, pos: int | None=None) -> None: + ... + + def readline(self, size: int | None=None): + ... + + def readlines(self, hint: int | None=None): + ... class MemoryMappedFile(NativeFile): + @staticmethod - def create(path, size): ... - def _open(self, path, mode='r'): ... - def resize(self, new_size): ... - def fileno(self): ... + def create(path: str, size: int) -> MemoryMappedFile: + ... + + def _open(self, path, mode: str='r') -> None: + ... + + def resize(self, new_size) -> None: + ... + + def fileno(self): + ... class OSFile(NativeFile): - def __init__(self, path, mode='r', memory_pool: MemoryPool | None=None): ... - def fileno(self): ... + + def __init__(self, path, mode: str='r', memory_pool: MemoryPool | None=None): + ... + + def fileno(self): + ... class FixedSizeBufferWriter(NativeFile): - def __init__(self, buffer: Buffer): ... - def set_memcopy_threads(self, num_threads: int): ... - def set_memcopy_blocksize(self, blocksize: int): ... - def set_memcopy_threshold(self, threshold: int): ... + + def __init__(self, buffer: Buffer): + ... + + def set_memcopy_threads(self, num_threads: int) -> None: + ... + + def set_memcopy_blocksize(self, blocksize: int) -> None: + ... + + def set_memcopy_threshold(self, threshold: int) -> None: + ... class Buffer(_Weakrefable): - def __init__(self): ... - def __len__(self): ... - def __repr__(self): ... - def _assert_cpu(self): ... + + def __init__(self): + ... + + def __len__(self): + ... + + def __repr__(self) -> str: + ... + + def _assert_cpu(self) -> None: + ... + @property - def size(self): ... + def size(self): + ... + @property - def address(self): ... - def hex(self): ... + def address(self): + ... + + def hex(self): + ... + @property - def is_mutable(self): ... + def is_mutable(self): + ... + @property - def is_cpu(self): ... + def is_cpu(self): + ... + @property - def device(self): ... + def device(self) -> Device: + ... + @property - def memory_manager(self): ... + def memory_manager(self) -> MemoryManager: + ... + @property - def device_type(self): ... + def device_type(self) -> DeviceAllocationType: + ... + @property - def parent(self): ... - def __getitem__(self, key): ... - def slice(self, offset=0, length=None): ... - def equals(self, other: Buffer): ... - def __eq__(self, other): ... - def __reduce_ex__(self, protocol): ... - def to_pybytes(self): ... + def parent(self) -> None | Buffer: + ... + + def __getitem__(self, key): + ... + + def slice(self, offset: int=0, length: int | None=None) -> Buffer: + ... + + def equals(self, other: Buffer) -> bool: + ... + + def __eq__(self, other): + ... + + def __reduce_ex__(self, protocol) -> tuple: + ... + + def to_pybytes(self): + ... class ResizableBuffer(Buffer): - def resize(self, new_size: int, shrink_to_fit=False): ... + + def resize(self, new_size: int, shrink_to_fit: bool=False) -> None: + ... class BufferOutputStream(NativeFile): - def __init__(self, memory_pool: MemoryPool | None=None): ... - def getvalue(self): ... + + def __init__(self, memory_pool: MemoryPool | None=None): + ... + + def getvalue(self) -> Buffer: + ... class MockOutputStream(NativeFile): - def __init__(self): ... - def size(self): ... + + def __init__(self): + ... + + def size(self): + ... class BufferReader(NativeFile): - def __init__(self, obj): ... + + def __init__(self, obj): + ... class CompressedInputStream(NativeFile): - def __init__(self, stream: object, compression: str): ... + + def __init__(self, stream: object, compression: str): + ... class CompressedOutputStream(NativeFile): - def __init__(self, stream: object, compression: str): ... + + def __init__(self, stream: object, compression: str): + ... class BufferedInputStream(NativeFile): - def __init__(self, stream: NativeFile, buffer_size: int, memory_pool: MemoryPool | None=None): ... - def detach(self): ... + + def __init__(self, stream: NativeFile, buffer_size: int, memory_pool: MemoryPool | None=None): + ... + + def detach(self) -> NativeFile: + ... class BufferedOutputStream(NativeFile): - def __init__(self, stream: NativeFile, buffer_size: int, memory_pool: MemoryPool | None=None): ... - def detach(self): ... + + def __init__(self, stream: NativeFile, buffer_size: int, memory_pool: MemoryPool | None=None): + ... + + def detach(self) -> NativeFile: + ... class TransformInputStream(NativeFile): - def __init__(self, stream: NativeFile, transform_func): ... + + def __init__(self, stream: NativeFile, transform_func): + ... class Transcoder: - def __init__(self, decoder, encoder): ... - def __call__(self, buf): ... + + def __init__(self, decoder, encoder): + ... + + def __call__(self, buf): + ... class CacheOptions(_Weakrefable): - def __init__(self, *, hole_size_limit=None, range_size_limit=None, lazy=None, prefetch_limit=None): ... + + def __init__(self, *, hole_size_limit=None, range_size_limit=None, lazy=None, prefetch_limit=None): + ... + @property - def hole_size_limit(self): ... + def hole_size_limit(self) -> None: + ... + @hole_size_limit.setter - def hole_size_limit(self, hole_size_limit): ... + def hole_size_limit(self, hole_size_limit) -> None: + ... + @property - def range_size_limit(self): ... + def range_size_limit(self) -> None: + ... + @range_size_limit.setter - def range_size_limit(self, range_size_limit): ... + def range_size_limit(self, range_size_limit) -> None: + ... + @property - def lazy(self): ... + def lazy(self) -> None: + ... + @lazy.setter - def lazy(self, lazy): ... + def lazy(self, lazy) -> None: + ... + @property - def prefetch_limit(self): ... + def prefetch_limit(self) -> None: + ... + @prefetch_limit.setter - def prefetch_limit(self, prefetch_limit): ... - def __eq__(self, other: CacheOptions): ... + def prefetch_limit(self, prefetch_limit) -> None: + ... + + def __eq__(self, other: CacheOptions): + ... + @staticmethod - def from_network_metrics(time_to_first_byte_millis, transfer_bandwidth_mib_per_sec, ideal_bandwidth_utilization_frac=0.9, max_ideal_request_size_mib=64): ... + def from_network_metrics(time_to_first_byte_millis: int, transfer_bandwidth_mib_per_sec: int, ideal_bandwidth_utilization_frac: int | float=0.9, max_ideal_request_size_mib: int=64) -> CacheOptions: + ... + @staticmethod - def _reconstruct(kwargs): ... - def __reduce__(self): ... + def _reconstruct(kwargs) -> CacheOptions: + ... + + def __reduce__(self) -> tuple: + ... class Codec(_Weakrefable): - def __init__(self, compression: str, compression_level=None): ... + + def __init__(self, compression: str, compression_level=None): + ... + @staticmethod - def detect(path): ... + def detect(path: str | os.PathLike[str]) -> Codec: + ... + @staticmethod - def is_available(compression: str): ... + def is_available(compression: str) -> bool: + ... + @staticmethod - def supports_compression_level(compression: str): ... + def supports_compression_level(compression: str): + ... + @staticmethod - def default_compression_level(compression: str): ... + def default_compression_level(compression: str): + ... + @staticmethod - def minimum_compression_level(compression: str): ... + def minimum_compression_level(compression: str): + ... + @staticmethod - def maximum_compression_level(compression: str): ... + def maximum_compression_level(compression: str): + ... + @property - def name(self): ... + def name(self) -> str: + ... + @property - def compression_level(self): ... - def compress(self, buf: object, asbytes=False, memory_pool=None): ... - def decompress(self, buf: object, decompressed_size=None, asbytes=False, memory_pool=None): ... - def __repr__(self): ... + def compression_level(self): + ... + + def compress(self, buf: object, asbytes: bool=False, memory_pool: MemoryPool | None=None) -> Buffer | bytes: + ... + + def decompress(self, buf: object, decompressed_size: int | None=None, asbytes: bool=False, memory_pool: MemoryPool | None=None) -> Buffer | bytes: + ... + + def __repr__(self) -> str: + ... class WriteStats(_WriteStats): __slots__ = () @@ -1740,310 +4060,327 @@ class ReadStats(_ReadStats): class IpcReadOptions(_Weakrefable): __slots__ = () - def __init__(self, *, ensure_native_endian: bool=True, ensure_alignment: Alignment=Alignment.Any, use_threads: bool=True, included_fields: list[Any] | None=None): ... + def __init__(self, *, ensure_native_endian: bool=True, ensure_alignment: Alignment=Alignment.Any, use_threads: bool=True, included_fields: list[Any] | None=None): + ... + @property - def ensure_native_endian(self): ... + def ensure_native_endian(self) -> None: + ... + @ensure_native_endian.setter - def ensure_native_endian(self, value: bool): ... + def ensure_native_endian(self, value: bool) -> None: + ... + @property - def ensure_alignment(self): ... + def ensure_alignment(self) -> None: + ... + @ensure_alignment.setter - def ensure_alignment(self, value: Alignment): ... + def ensure_alignment(self, value: Alignment) -> None: + ... + @property - def use_threads(self): ... + def use_threads(self) -> None: + ... + @use_threads.setter - def use_threads(self, value: bool): ... + def use_threads(self, value: bool) -> None: + ... + @property - def included_fields(self): ... + def included_fields(self) -> None: + ... + @included_fields.setter - def included_fields(self, value: list[Any]): ... - def __repr__(self): ... + def included_fields(self, value: list[Any]) -> None: + ... + + def __repr__(self) -> str: + ... class IpcWriteOptions(_Weakrefable): __slots__ = () - def __init__(self, *, metadata_version=MetadataVersion.V5, allow_64bit: bool=False, use_legacy_format=False, compression=None, use_threads: bool=True, emit_dictionary_deltas: bool=False, unify_dictionaries: bool=False): ... + def __init__(self, *, metadata_version=MetadataVersion.V5, allow_64bit: bool=False, use_legacy_format: bool=False, compression=None, use_threads: bool=True, emit_dictionary_deltas: bool=False, unify_dictionaries: bool=False): + ... + @property - def allow_64bit(self): ... + def allow_64bit(self) -> None: + ... + @allow_64bit.setter - def allow_64bit(self, value: bool): ... + def allow_64bit(self, value: bool) -> None: + ... + @property - def use_legacy_format(self): ... + def use_legacy_format(self) -> None: + ... + @use_legacy_format.setter - def use_legacy_format(self, value: bool): ... + def use_legacy_format(self, value: bool) -> None: + ... + @property - def metadata_version(self): ... + def metadata_version(self) -> None: + ... + @metadata_version.setter - def metadata_version(self, value): ... + def metadata_version(self, value) -> None: + ... + @property - def compression(self): ... + def compression(self) -> None: + ... + @compression.setter - def compression(self, value): ... + def compression(self, value) -> None: + ... + @property - def use_threads(self): ... + def use_threads(self) -> None: + ... + @use_threads.setter - def use_threads(self, value: bool): ... + def use_threads(self, value: bool) -> None: + ... + @property - def emit_dictionary_deltas(self): ... + def emit_dictionary_deltas(self) -> None: + ... + @emit_dictionary_deltas.setter - def emit_dictionary_deltas(self, value: bool): ... + def emit_dictionary_deltas(self, value: bool) -> None: + ... + @property - def unify_dictionaries(self): ... + def unify_dictionaries(self) -> None: + ... + @unify_dictionaries.setter - def unify_dictionaries(self, value: bool): ... - def __repr__(self): ... + def unify_dictionaries(self, value: bool) -> None: + ... + + def __repr__(self) -> str: + ... class Message(_Weakrefable): - def __init__(self): ... + + def __init__(self): + ... + @property - def type(self): ... + def type(self) -> str: + ... + @property - def metadata(self): ... + def metadata(self) -> Buffer: + ... + @property - def metadata_version(self): ... + def metadata_version(self): + ... + @property - def body(self): ... - def equals(self, other: Message): ... - def serialize_to(self, sink: NativeFile, alignment=8, memory_pool=None): ... - def serialize(self, alignment=8, memory_pool=None): ... - def __repr__(self): ... + def body(self) -> None | Buffer: + ... + + def equals(self, other: Message) -> bool: + ... + + def serialize_to(self, sink: NativeFile, alignment: int=8, memory_pool: MemoryPool | None=None) -> None: + ... + + def serialize(self, alignment: int=8, memory_pool: MemoryPool | None=None) -> Buffer: + ... + + def __repr__(self) -> str: + ... class MessageReader(_Weakrefable): - def __init__(self): ... + + def __init__(self): + ... + @staticmethod - def open_stream(source): ... - def __iter__(self): ... - def __next__(self): ... - def read_next_message(self): ... + def open_stream(source) -> MessageReader: + ... + + def __iter__(self) -> Self: + ... + + def __next__(self): + ... + + def read_next_message(self) -> Message: + ... class _CRecordBatchWriter(_Weakrefable): - def write(self, table_or_batch): ... - def write_batch(self, batch: RecordBatch, custom_metadata=None): ... - def write_table(self, table: Table, max_chunksize=None): ... - def close(self): ... - def __enter__(self): ... - def __exit__(self, exc_type, exc_val, exc_tb): ... + + def write(self, table_or_batch) -> None: + ... + + def write_batch(self, batch: RecordBatch, custom_metadata: Mapping[Any, Any] | KeyValueMetadata | None=None) -> None: + ... + + def write_table(self, table: Table, max_chunksize: int | None=None) -> None: + ... + + def close(self) -> None: + ... + + def __enter__(self) -> Self: + ... + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + ... + @property - def stats(self): ... + def stats(self): + ... class _RecordBatchStreamWriter(_CRecordBatchWriter): - def __init__(self): ... + + def __init__(self): + ... + @property - def _use_legacy_format(self): ... + def _use_legacy_format(self): + ... + @property - def _metadata_version(self): ... - def _open(self, sink, schema: Schema, options: IpcWriteOptions=IpcWriteOptions()): ... + def _metadata_version(self): + ... + + def _open(self, sink, schema: Schema, options: IpcWriteOptions=IpcWriteOptions()) -> None: + ... class _ReadPandasMixin: - def read_pandas(self, **options): ... + + def read_pandas(self, **options): + ... class RecordBatchReader(_Weakrefable): read_pandas = _ReadPandasMixin.read_pandas - def __init__(self): ... - def __iter__(self): ... - def __next__(self): ... + def __init__(self): + ... + + def __iter__(self) -> Self: + ... + + def __next__(self): + ... + @property - def schema(self): ... - def read_next_batch(self): ... - def read_next_batch_with_custom_metadata(self): ... - def iter_batches_with_custom_metadata(self): ... - def read_all(self): ... - def close(self): ... - def __enter__(self): ... - def __exit__(self, exc_type, exc_val, exc_tb): ... - def cast(self, target_schema): ... - def _export_to_c(self, out_ptr): ... + def schema(self) -> Schema: + ... + + def read_next_batch(self) -> RecordBatch: + ... + + def read_next_batch_with_custom_metadata(self): + ... + + def iter_batches_with_custom_metadata(self) -> Iterator[Any]: + ... + + def read_all(self) -> Table: + ... + + def close(self) -> None: + ... + + def __enter__(self) -> Self: + ... + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + ... + + def cast(self, target_schema: Schema) -> RecordBatchReader: + ... + + def _export_to_c(self, out_ptr) -> None: + ... + @staticmethod - def _import_from_c(in_ptr): ... - def __arrow_c_stream__(self, requested_schema=None): ... + def _import_from_c(in_ptr) -> RecordBatchReader: + ... + + def __arrow_c_stream__(self, requested_schema=None): + ... + @staticmethod - def _import_from_c_capsule(stream): ... + def _import_from_c_capsule(stream) -> RecordBatchReader: + ... + @staticmethod - def from_stream(data, schema=None): ... + def from_stream(data, schema: Schema | None=None) -> RecordBatchReader: + ... + @staticmethod - def from_batches(schema: Schema, batches): ... + def from_batches(schema: Schema, batches: Iterable[RecordBatch]) -> RecordBatchReader: + ... class _RecordBatchStreamReader(RecordBatchReader): - def __init__(self): ... - def _open(self, source, options: IpcReadOptions=IpcReadOptions(), memory_pool: MemoryPool | None=None): ... + + def __init__(self): + ... + + def _open(self, source, options: IpcReadOptions=IpcReadOptions(), memory_pool: MemoryPool | None=None) -> None: + ... + @property - def stats(self): ... + def stats(self): + ... class _RecordBatchFileWriter(_RecordBatchStreamWriter): - def _open(self, sink, schema: Schema, options: IpcWriteOptions=IpcWriteOptions(), metadata=None): ... + + def _open(self, sink, schema: Schema, options: IpcWriteOptions=IpcWriteOptions(), metadata=None) -> None: + ... class RecordBatchWithMetadata(_RecordBatchWithMetadata): __slots__ = () class _RecordBatchFileReader(_Weakrefable): - get_record_batch = get_batch read_pandas = _ReadPandasMixin.read_pandas schema: Schema - def __init__(self): ... - def _open(self, source, footer_offset=None, options: IpcReadOptions=IpcReadOptions(), memory_pool: MemoryPool | None=None): ... - @property - def num_record_batches(self): ... - def count_rows(self): ... - def get_batch(self, i: int): ... - def get_batch_with_custom_metadata(self, i: int): ... - def read_all(self): ... - def __enter__(self): ... - def __exit__(self, exc_type, exc_value, traceback): ... - @property - def stats(self): ... - @property - def metadata(self): ... - -class _Weakrefable: ... - -class _Metadata(_Weakrefable): ... - -class Decimal32Array(FixedSizeBinaryArray): ... - -def cpu_count(): ... -def set_cpu_count(count: int): ... -def is_opentelemetry_enabled() -> bool: ... -def is_threading_enabled() -> bool: ... -def _pc(): ... -def _pac(): ... -def _ensure_cuda_loaded(): ... -def _gdb_test_session(): ... -def encode_file_path(path): ... -def tobytes(o): ... -def frombytes(o, *, safe=False): ... -def enable_signal_handlers(enable: c_bool): ... -def runtime_info(): ... -def _build_info(): ... -def set_timezone_db_path(path): ... -def default_memory_pool(): ... -def proxy_memory_pool(parent: MemoryPool): ... -def logging_memory_pool(parent: MemoryPool): ... -def system_memory_pool(): ... -def jemalloc_memory_pool(): ... -def mimalloc_memory_pool(): ... -def set_memory_pool(pool: MemoryPool): ... -def log_memory_allocations(enable=True): ... -def total_allocated_bytes(): ... -def jemalloc_set_decay_ms(decay_ms): ... -def supported_memory_backends(): ... -def default_cpu_memory_manager(): ... -def _get_pandas_type_map(): ... -def _is_primitive(type: Type): ... -def _is_nested(type: Type): ... -def _get_pandas_type(arrow_type, coerce_to_ns=False): ... -def _get_pandas_tz_type(arrow_type, coerce_to_ns=False): ... -def _to_pandas_dtype(arrow_type, options=None): ... -def register_extension_type(ext_type): ... -def unregister_extension_type(type_name): ... -def ensure_metadata(meta: object, allow_none: c_bool=False) -> KeyValueMetadata: ... -def unify_schemas(schemas, *, promote_options='default'): ... -def field(name, type=None, nullable=None, metadata=None): ... -def null(): ... -def bool_(): ... -def uint8(): ... -def int8(): ... -def uint16(): ... -def int16(): ... -def uint32(): ... -def int32(): ... -def uint64(): ... -def int64(): ... -def tzinfo_to_string(tz): ... -def string_to_tzinfo(name, *, prefer_zoneinfo=True): ... -def timestamp(unit, tz=None): ... -def time32(unit): ... -def time64(unit): ... -def duration(unit): ... -def month_day_nano_interval(): ... -def date32(): ... -def date64(): ... -def float16(): ... -def float32(): ... -def float64(): ... -def decimal32(precision: int, scale: int=0) -> DataType: ... -def decimal64(precision: int, scale: int=0) -> DataType: ... -def decimal128(precision: int, scale: int=0) -> DataType: ... -def decimal256(precision: int, scale: int=0) -> DataType: ... -def string(): ... -def utf8(): ... -def binary(length: int=-1): ... -def large_binary(): ... -def large_string(): ... -def large_utf8(): ... -def binary_view(): ... -def string_view(): ... -def list_(value_type, list_size: int=-1): ... -def large_list(value_type) -> LargeListType: ... -def list_view(value_type) -> ListViewType: ... -def large_list_view(value_type) -> LargeListViewType: ... -def map_(key_type, item_type, keys_sorted=False) -> MapType: ... -def dictionary(index_type, value_type, ordered: bool=False) -> DictionaryType: ... -def struct(fields): ... -def sparse_union(child_fields, type_codes=None): ... -def dense_union(child_fields, type_codes=None): ... -def union(child_fields, mode, type_codes=None): ... -def run_end_encoded(run_end_type, value_type): ... -def json_(storage_type: DataType=utf8()): ... -def uuid(): ... -def fixed_shape_tensor(value_type: DataType, shape, dim_names=None, permutation=None): ... -def bool8(): ... -def opaque(storage_type: DataType, type_name: str, vendor_name: str): ... -def type_for_alias(name): ... -def ensure_type(ty: object, allow_none: bool=False) -> DataType: ... -def schema(fields, metadata=None): ... -def from_numpy_dtype(dtype: object): ... -def is_boolean_value(obj: object): ... -def is_integer_value(obj: object): ... -def is_float_value(obj: object): ... -def _datetime_from_int(value: int, unit: TimeUnit, tzinfo=None): ... -def scalar(value, type=None, *, from_pandas=None, memory_pool: MemoryPool | None=None): ... -def _ndarray_to_arrow_type(values: object, type: DataType): ... -def _handle_arrow_array_protocol(obj, type, mask, size): ... -def array(obj: object, type=None, mask=None, size=None, from_pandas=None, safe: bool=True, memory_pool: MemoryPool | None=None): ... -def asarray(values, type=None): ... -def nulls(size, type=None, memory_pool: MemoryPool | None=None): ... -def repeat(value, size, memory_pool: MemoryPool | None=None): ... -def infer_type(values, mask=None, from_pandas=False): ... -def arange(start: int, stop: int, step: int=1, *, memory_pool=None): ... -def _normalize_slice(arrow_obj: object, key: slice): ... -def _restore_array(data): ... -def _compute_binary_op(func_name, left, right): ... -def concat_arrays(arrays, memory_pool: MemoryPool | None=None): ... -def _empty_array(type: DataType): ... -def chunked_array(arrays, type=None): ... -def _reconstruct_record_batch(columns, schema): ... -def table_to_blocks(options, table: Table, categories, extension_columns): ... -def _reconstruct_table(arrays, schema): ... -def record_batch(data, names=None, schema=None, metadata=None): ... -def table(data, names=None, schema=None, metadata=None, nthreads=None): ... -def concat_tables(tables, memory_pool: MemoryPool | None=None, promote_options: str='none', **kwargs): ... -def concat_batches(recordbatches, memory_pool: MemoryPool | None=None): ... -def _from_pydict(cls, mapping, schema, metadata): ... -def _from_pylist(cls, mapping, schema, metadata): ... -def io_thread_count(): ... -def set_io_thread_count(count: int): ... -def memory_map(path, mode='r'): ... -def create_memory_map(path, size): ... -def allocate_buffer(size: int, memory_pool: MemoryPool | None=None, resizable=False): ... -def transcoding_input_stream(stream, src_encoding, dest_encoding): ... -def py_buffer(obj: object): ... -def foreign_buffer(address, size, base=None): ... -def as_buffer(o: object): ... -def _detect_compression(path): ... -def compress(buf: object, codec='lz4', asbytes=False, memory_pool=None): ... -def decompress(buf: object, decompressed_size=None, codec='lz4', asbytes=False, memory_pool=None): ... -def input_stream(source, compression='detect', buffer_size=None): ... -def output_stream(source, compression='detect', buffer_size=None): ... -def get_tensor_size(tensor: Tensor): ... -def get_record_batch_size(batch: RecordBatch): ... -def write_tensor(tensor: Tensor, dest: NativeFile): ... -def read_tensor(source): ... -def read_message(source): ... -def read_schema(obj, dictionary_memo: DictionaryMemo | None=None): ... -def read_record_batch(obj, schema: Schema, dictionary_memo: DictionaryMemo | None=None): ... -def benchmark_PandasObjectIsNull(obj: list[Any]): ... + def __init__(self): + ... -_NULL: NullScalar -NA: NullScalar + def _open(self, source, footer_offset=None, options: IpcReadOptions=IpcReadOptions(), memory_pool: MemoryPool | None=None) -> None: + ... + + @property + def num_record_batches(self): + ... + + def count_rows(self) -> int: + ... + + def get_batch(self, i: int) -> RecordBatch: + ... + get_record_batch = get_batch + + def get_batch_with_custom_metadata(self, i: int): + ... + + def read_all(self) -> Table: + ... + + def __enter__(self) -> Self: + ... + + def __exit__(self, exc_type, exc_value, traceback) -> None: + ... + + @property + def stats(self): + ... + + @property + def metadata(self): + ... + +class Decimal32Array(FixedSizeBinaryArray): + ... diff --git a/python/pyarrow-stubs/pyarrow/orc.pyi b/python/pyarrow-stubs/pyarrow/orc.pyi index 9b1abb8b2b3c..79a8751e51ca 100644 --- a/python/pyarrow-stubs/pyarrow/orc.pyi +++ b/python/pyarrow-stubs/pyarrow/orc.pyi @@ -15,64 +15,126 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +import pyarrow.fs +from pyarrow.lib import __doc__ as __doc__ import types from _typeshed import Incomplete from pyarrow.fs import _resolve_filesystem_and_path as _resolve_filesystem_and_path from pyarrow.lib import Table as Table +_orc_writer_args_docs: str + +def read_table(source: str | pyarrow.NativeFile | IO[Any], columns: list | None=None, filesystem: pyarrow.fs.FileSystem | None=None): + ... + +def write_table(table: Table, where, *, file_version: str='0.12', batch_size: int=1024, stripe_size: int=..., compression: str='uncompressed', compression_block_size: int=65536, compression_strategy: str='speed', row_index_stride: int=10000, padding_tolerance: float=0.0, dictionary_key_size_threshold: float=0.0, bloom_filter_columns=None, bloom_filter_fpp: float=0.05) -> None: + ... class ORCFile: reader: Incomplete - def __init__(self, source) -> None: ... + + def __init__(self, source) -> None: + ... + @property - def metadata(self): ... + def metadata(self): + ... + @property - def schema(self): ... + def schema(self): + ... + @property - def nrows(self): ... + def nrows(self): + ... + @property - def nstripes(self): ... + def nstripes(self): + ... + @property - def file_version(self): ... + def file_version(self): + ... + @property - def software_version(self): ... + def software_version(self): + ... + @property - def compression(self): ... + def compression(self): + ... + @property - def compression_size(self): ... + def compression_size(self): + ... + @property - def writer(self): ... + def writer(self): + ... + @property - def writer_version(self): ... + def writer_version(self): + ... + @property - def row_index_stride(self): ... + def row_index_stride(self): + ... + @property - def nstripe_statistics(self): ... + def nstripe_statistics(self): + ... + @property - def content_length(self): ... + def content_length(self): + ... + @property - def stripe_statistics_length(self): ... + def stripe_statistics_length(self): + ... + @property - def file_footer_length(self): ... + def file_footer_length(self): + ... + @property - def file_postscript_length(self): ... + def file_postscript_length(self): + ... + @property - def file_length(self): ... - def _select_names(self, columns=None): ... - def read_stripe(self, n, columns=None): ... - def read(self, columns=None): ... + def file_length(self): + ... -_orc_writer_args_docs: str + def _select_names(self, columns=None): + ... + + def read_stripe(self, n: int, columns: list | None=None) -> pyarrow.RecordBatch: + ... + + def read(self, columns: list | None=None) -> Table: + ... class ORCWriter: __doc__: Incomplete is_open: bool writer: Incomplete - def __init__(self, where, *, file_version: str = '0.12', batch_size: int = 1024, stripe_size=..., compression: str = 'uncompressed', compression_block_size: int = 65536, compression_strategy: str = 'speed', row_index_stride: int = 10000, padding_tolerance: float = 0.0, dictionary_key_size_threshold: float = 0.0, bloom_filter_columns=None, bloom_filter_fpp: float = 0.05) -> None: ... - def __del__(self) -> None: ... - def __enter__(self): ... - def __exit__(self, *args, **kwargs) -> None: ... - def write(self, table) -> None: ... - def close(self) -> None: ... - -def read_table(source, columns=None, filesystem=None): ... -def write_table(table, where, *, file_version: str = '0.12', batch_size: int = 1024, stripe_size=..., compression: str = 'uncompressed', compression_block_size: int = 65536, compression_strategy: str = 'speed', row_index_stride: int = 10000, padding_tolerance: float = 0.0, dictionary_key_size_threshold: float = 0.0, bloom_filter_columns=None, bloom_filter_fpp: float = 0.05) -> None: ... + + def __init__(self, where, *, file_version: str='0.12', batch_size: int=1024, stripe_size=..., compression: str='uncompressed', compression_block_size: int=65536, compression_strategy: str='speed', row_index_stride: int=10000, padding_tolerance: float=0.0, dictionary_key_size_threshold: float=0.0, bloom_filter_columns=None, bloom_filter_fpp: float=0.05) -> None: + ... + + def __del__(self) -> None: + ... + + def __enter__(self): + ... + + def __exit__(self, *args, **kwargs) -> None: + ... + + def write(self, table: Table) -> None: + ... + + def close(self) -> None: + ... diff --git a/python/pyarrow-stubs/pyarrow/pandas_compat.pyi b/python/pyarrow-stubs/pyarrow/pandas_compat.pyi index 9e04065d39f7..0b31852eac86 100644 --- a/python/pyarrow-stubs/pyarrow/pandas_compat.pyi +++ b/python/pyarrow-stubs/pyarrow/pandas_compat.pyi @@ -15,48 +15,118 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence from _typeshed import Incomplete from pyarrow.lib import _pandas_api as _pandas_api, frombytes as frombytes, is_threading_enabled as is_threading_enabled - _logical_type_map: Incomplete _numpy_logical_type_map: Incomplete _pandas_logical_type_map: Incomplete -def get_logical_type_map(): ... -def get_logical_type(arrow_type): ... -def get_numpy_logical_type_map(): ... -def get_logical_type_from_numpy(pandas_collection): ... -def get_extension_dtype_info(column): ... -def get_column_metadata(column, name, arrow_type, field_name): ... -def construct_metadata(columns_to_convert, df, column_names, index_levels, index_descriptors, preserve_index, types, column_field_names=None): ... -def _get_simple_index_descriptor(level, name): ... -def _column_name_to_strings(name): ... -def _index_level_name(index, i, column_names): ... -def _get_columns_to_convert(df, schema, preserve_index, columns): ... -def _get_columns_to_convert_given_schema(df, schema, preserve_index): ... -def _get_index_level(df, name): ... -def _level_name(name): ... -def _get_range_index_descriptor(level): ... -def _get_index_level_values(index): ... -def _resolve_columns_of_interest(df, schema, columns): ... -def dataframe_to_types(df, preserve_index, columns=None): ... -def dataframe_to_arrays(df, schema, preserve_index, nthreads: int = 1, columns=None, safe: bool = True): ... -def get_datetimetz_type(values, dtype, type_): ... -def _reconstruct_block(item, columns=None, extension_columns=None, return_block: bool = True): ... -def make_datetimetz(unit, tz): ... -def table_to_dataframe(options, table, categories=None, ignore_metadata: bool = False, types_mapper=None): ... +def get_logical_type_map(): + ... + +def get_logical_type(arrow_type): + ... + +def get_numpy_logical_type_map(): + ... + +def get_logical_type_from_numpy(pandas_collection): + ... + +def get_extension_dtype_info(column): + ... + +def get_column_metadata(column, name: str, arrow_type: pyarrow.DataType, field_name: str) -> dict: + ... + +def construct_metadata(columns_to_convert, df, column_names, index_levels, index_descriptors: list[dict], preserve_index: bool, types: list[pyarrow.DataType], column_field_names=None) -> dict: + ... + +def _get_simple_index_descriptor(level, name): + ... + +def _column_name_to_strings(name: str | tuple) -> str | tuple: + ... + +def _index_level_name(index, i: int, column_names) -> str: + ... + +def _get_columns_to_convert(df, schema, preserve_index, columns): + ... + +def _get_columns_to_convert_given_schema(df, schema, preserve_index): + ... + +def _get_index_level(df, name): + ... + +def _level_name(name): + ... + +def _get_range_index_descriptor(level): + ... + +def _get_index_level_values(index): + ... + +def _resolve_columns_of_interest(df, schema, columns): + ... +def dataframe_to_types(df, preserve_index, columns=None): + ... + +def dataframe_to_arrays(df, schema, preserve_index, nthreads: int=1, columns=None, safe: bool=True): + ... + +def get_datetimetz_type(values, dtype, type_): + ... + +def _reconstruct_block(item: dict, columns=None, extension_columns: dict | None=None, return_block: bool=True): + ... + +def make_datetimetz(unit, tz): + ... + +def table_to_dataframe(options, table, categories=None, ignore_metadata: bool=False, types_mapper=None): + ... _pandas_supported_numpy_types: Incomplete -def _get_extension_dtypes(table, columns_metadata, types_mapper, options, categories): ... -def _check_data_column_metadata_consistency(all_columns) -> None: ... -def _deserialize_column_index(block_table, all_columns, column_indexes): ... -def _reconstruct_index(table, index_descriptors, all_columns, types_mapper=None): ... -def _extract_index_level(table, result_table, field_name, field_name_to_metadata, types_mapper=None): ... -def _backwards_compatible_index_name(raw_name, logical_name): ... -def _is_generated_index_name(name): ... -def get_pandas_logical_type_map(): ... -def _pandas_type_to_numpy_type(pandas_type): ... -def _reconstruct_columns_from_metadata(columns, column_indexes): ... -def _add_any_metadata(table, pandas_metadata): ... -def make_tz_aware(series, tz): ... +def _get_extension_dtypes(table, columns_metadata, types_mapper, options, categories): + ... + +def _check_data_column_metadata_consistency(all_columns) -> None: + ... + +def _deserialize_column_index(block_table, all_columns, column_indexes): + ... + +def _reconstruct_index(table, index_descriptors, all_columns, types_mapper=None): + ... + +def _extract_index_level(table, result_table, field_name, field_name_to_metadata, types_mapper=None): + ... + +def _backwards_compatible_index_name(raw_name: str, logical_name: str) -> str: + ... + +def _is_generated_index_name(name): + ... + +def get_pandas_logical_type_map(): + ... + +def _pandas_type_to_numpy_type(pandas_type: str): + ... + +def _reconstruct_columns_from_metadata(columns, column_indexes): + ... + +def _add_any_metadata(table, pandas_metadata): + ... + +def make_tz_aware(series, tz): + ... diff --git a/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi b/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi index 14d5874f5c46..a6655e8ae5de 100644 --- a/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi +++ b/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete from .core import * diff --git a/python/pyarrow-stubs/pyarrow/parquet/core.pyi b/python/pyarrow-stubs/pyarrow/parquet/core.pyi index 1fe5b27ea5a6..4289fbcec48f 100644 --- a/python/pyarrow-stubs/pyarrow/parquet/core.pyi +++ b/python/pyarrow-stubs/pyarrow/parquet/core.pyi @@ -15,42 +15,103 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +import pyarrow.compute +import pyarrow.dataset +import pyarrow.fs +from pyarrow.lib import __doc__ as __doc__ import types from _typeshed import Incomplete from pyarrow._parquet import ColumnChunkMetaData as ColumnChunkMetaData, ColumnSchema as ColumnSchema, FileDecryptionProperties as FileDecryptionProperties, FileEncryptionProperties as FileEncryptionProperties, FileMetaData as FileMetaData, ParquetLogicalType as ParquetLogicalType, ParquetReader as ParquetReader, ParquetSchema as ParquetSchema, RowGroupMetaData as RowGroupMetaData, SortingColumn as SortingColumn, Statistics as Statistics - __all__ = ['ParquetReader', 'Statistics', 'FileMetaData', 'RowGroupMetaData', 'ColumnChunkMetaData', 'ParquetSchema', 'ColumnSchema', 'ParquetLogicalType', 'FileEncryptionProperties', 'FileDecryptionProperties', 'SortingColumn', 'filters_to_expression', '_filters_to_expression', 'ParquetFile', 'ParquetWriter', 'ParquetDataset', 'read_table', 'read_pandas', 'write_table', 'write_to_dataset', 'write_metadata', 'read_metadata', 'read_schema'] -def filters_to_expression(filters): ... - +def filters_to_expression(filters: list[tuple] | list[list[tuple]]) -> pyarrow.compute.Expression: + ... _filters_to_expression: Incomplete +def read_table(source: str | list[str] | pyarrow.NativeFile | IO[Any], *, columns: list | None=None, use_threads: bool=True, schema: pyarrow.Schema | None=None, use_pandas_metadata: bool=False, read_dictionary: list | None=None, binary_type: pyarrow.DataType | None=None, list_type=None, memory_map: bool=False, buffer_size: int=0, partitioning: str='hive', filesystem: pyarrow.fs.FileSystem | None=None, filters: pyarrow.compute.Expression | list[tuple] | list[list[tuple]] | None=None, ignore_prefixes: list | None=None, pre_buffer: bool=True, coerce_int96_timestamp_unit: str | None=None, decryption_properties: FileDecryptionProperties | None=None, thrift_string_size_limit: int | None=None, thrift_container_size_limit: int | None=None, schema_depth_limit: int | None=None, page_checksum_verification: bool=False, arrow_extensions_enabled: bool=True) -> pyarrow.Table: + ... + +def read_pandas(source: str | list[str] | pyarrow.NativeFile | IO[Any], columns: list | None=None, **kwargs) -> pyarrow.Table: + ... + +def write_table(table: pyarrow.Table, where: str | pyarrow.NativeFile, row_group_size: int | None=None, version: str='2.6', use_dictionary: bool=True, compression: str='snappy', write_statistics: bool=True, use_deprecated_int96_timestamps: bool | None=None, coerce_timestamps: str | None=None, allow_truncated_timestamps: bool=False, data_page_size: int | None=None, flavor: Literal['spark'] | None=None, filesystem: pyarrow.fs.FileSystem | None=None, compression_level: int | dict | None=None, use_byte_stream_split: bool=False, column_encoding: str | dict | None=None, data_page_version: str='1.0', use_compliant_nested_type: bool=True, encryption_properties: FileEncryptionProperties | None=None, write_batch_size: int | None=None, dictionary_pagesize_limit: int | None=None, store_schema: bool=True, write_page_index: bool=False, write_page_checksum: bool=False, sorting_columns: Sequence[SortingColumn] | None=None, store_decimal_as_integer: bool=False, write_time_adjusted_to_utc: bool=False, max_rows_per_page: int | None=None, bloom_filter_options: dict | None=None, use_content_defined_chunking: bool=False, **kwargs) -> None: + ... + +def write_to_dataset(table: pyarrow.Table, root_path, partition_cols: list | None=None, filesystem: pyarrow.fs.FileSystem | None=None, schema: pyarrow.Schema | None=None, partitioning: pyarrow.dataset.Partitioning | list[str] | None=None, basename_template: str | None=None, use_threads: bool | None=None, file_visitor: Callable[..., Any] | None=None, existing_data_behavior: Literal['overwrite_or_ignore'] | Literal['error'] | Literal['delete_matching'] | None=None, **kwargs) -> None: + ... + +def write_metadata(schema: pyarrow.Schema, where: str | pyarrow.NativeFile, metadata_collector: list | None=None, filesystem: pyarrow.fs.FileSystem | None=None, **kwargs) -> None: + ... + +def read_metadata(where: str | IO[Any], memory_map: bool=False, decryption_properties: FileDecryptionProperties | None=None, filesystem: pyarrow.fs.FileSystem | None=None, arrow_extensions_enabled: bool=True) -> FileMetaData: + ... + +def read_schema(where: str | IO[Any], memory_map: bool=False, decryption_properties: FileDecryptionProperties | None=None, filesystem: pyarrow.fs.FileSystem | None=None, arrow_extensions_enabled: bool=True) -> pyarrow.Schema: + ... +EXCLUDED_PARQUET_PATHS: set[str] + class ParquetFile: _close_source: Incomplete reader: Incomplete common_metadata: Incomplete _nested_paths_by_prefix: Incomplete - def __init__(self, source, *, metadata=None, common_metadata=None, read_dictionary=None, binary_type=None, list_type=None, memory_map: bool = False, buffer_size: int = 0, pre_buffer: bool = True, coerce_int96_timestamp_unit=None, decryption_properties=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, filesystem=None, page_checksum_verification: bool = False, arrow_extensions_enabled: bool = True) -> None: ... - def __enter__(self): ... - def __exit__(self, *args, **kwargs) -> None: ... - def _build_nested_paths(self): ... + + def __init__(self, source, *, metadata=None, common_metadata=None, read_dictionary=None, binary_type=None, list_type=None, memory_map: bool=False, buffer_size: int=0, pre_buffer: bool=True, coerce_int96_timestamp_unit=None, decryption_properties=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, filesystem=None, page_checksum_verification: bool=False, arrow_extensions_enabled: bool=True) -> None: + ... + + def __enter__(self): + ... + + def __exit__(self, *args, **kwargs) -> None: + ... + + def _build_nested_paths(self): + ... + @property - def metadata(self): ... + def metadata(self): + ... + @property - def schema(self): ... + def schema(self): + ... + @property - def schema_arrow(self): ... + def schema_arrow(self): + ... + @property - def num_row_groups(self): ... - def close(self, force: bool = False): ... + def num_row_groups(self): + ... + + def close(self, force: bool=False): + ... + @property - def closed(self) -> bool: ... - def read_row_group(self, i, columns=None, use_threads: bool = True, use_pandas_metadata: bool = False): ... - def read_row_groups(self, row_groups, columns=None, use_threads: bool = True, use_pandas_metadata: bool = False): ... - def iter_batches(self, batch_size: int = 65536, row_groups=None, columns=None, use_threads: bool = True, use_pandas_metadata: bool = False): ... - def read(self, columns=None, use_threads: bool = True, use_pandas_metadata: bool = False): ... - def scan_contents(self, columns=None, batch_size: int = 65536): ... - def _get_column_indices(self, column_names, use_pandas_metadata: bool = False): ... + def closed(self) -> bool: + ... + + def read_row_group(self, i: int, columns: list | None=None, use_threads: bool=True, use_pandas_metadata: bool=False): + ... + + def read_row_groups(self, row_groups: list, columns: list | None=None, use_threads: bool=True, use_pandas_metadata: bool=False): + ... + + def iter_batches(self, batch_size: int=65536, row_groups: list | None=None, columns: list | None=None, use_threads: bool=True, use_pandas_metadata: bool=False): + ... + + def read(self, columns: list | None=None, use_threads: bool=True, use_pandas_metadata: bool=False): + ... + + def scan_contents(self, columns: list[int] | None=None, batch_size: int=65536) -> int: + ... + + def _get_column_indices(self, column_names, use_pandas_metadata: bool=False): + ... class ParquetWriter: __doc__: Incomplete @@ -62,43 +123,74 @@ class ParquetWriter: _metadata_collector: Incomplete writer: Incomplete is_open: bool - def __init__(self, where, schema, filesystem=None, flavor=None, version: str = '2.6', use_dictionary: bool = True, compression: str = 'snappy', write_statistics: bool = True, use_deprecated_int96_timestamps=None, compression_level=None, use_byte_stream_split: bool = False, column_encoding=None, writer_engine_version=None, data_page_version: str = '1.0', use_compliant_nested_type: bool = True, encryption_properties=None, write_batch_size=None, dictionary_pagesize_limit=None, store_schema: bool = True, write_page_index: bool = False, write_page_checksum: bool = False, sorting_columns=None, store_decimal_as_integer: bool = False, write_time_adjusted_to_utc: bool = False, max_rows_per_page=None, bloom_filter_options=None, use_content_defined_chunking: bool = False, **options) -> None: ... - def __del__(self) -> None: ... - def __enter__(self): ... - def __exit__(self, *args, **kwargs): ... - def write(self, table_or_batch, row_group_size=None) -> None: ... - def write_batch(self, batch, row_group_size=None) -> None: ... - def write_table(self, table, row_group_size=None) -> None: ... - def close(self) -> None: ... - def add_key_value_metadata(self, key_value_metadata) -> None: ... + + def __init__(self, where, schema, filesystem=None, flavor=None, version: str='2.6', use_dictionary: bool=True, compression: str='snappy', write_statistics: bool=True, use_deprecated_int96_timestamps=None, compression_level=None, use_byte_stream_split: bool=False, column_encoding=None, writer_engine_version=None, data_page_version: str='1.0', use_compliant_nested_type: bool=True, encryption_properties=None, write_batch_size=None, dictionary_pagesize_limit=None, store_schema: bool=True, write_page_index: bool=False, write_page_checksum: bool=False, sorting_columns=None, store_decimal_as_integer: bool=False, write_time_adjusted_to_utc: bool=False, max_rows_per_page=None, bloom_filter_options=None, use_content_defined_chunking: bool=False, **options) -> None: + ... + + def __del__(self) -> None: + ... + + def __enter__(self): + ... + + def __exit__(self, *args, **kwargs): + ... + + def write(self, table_or_batch, row_group_size: int | None=None) -> None: + ... + + def write_batch(self, batch: pyarrow.RecordBatch, row_group_size: int | None=None) -> None: + ... + + def write_table(self, table: pyarrow.Table, row_group_size: int | None=None) -> None: + ... + + def close(self) -> None: + ... + + def add_key_value_metadata(self, key_value_metadata: dict) -> None: + ... class ParquetDataset: __doc__: Incomplete _filter_expression: Incomplete _base_dir: Incomplete _dataset: Incomplete - def __init__(self, path_or_paths, filesystem=None, schema=None, *, filters=None, read_dictionary=None, binary_type=None, list_type=None, memory_map: bool = False, buffer_size=None, partitioning: str = 'hive', ignore_prefixes=None, pre_buffer: bool = True, coerce_int96_timestamp_unit=None, decryption_properties=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, page_checksum_verification: bool = False, arrow_extensions_enabled: bool = True) -> None: ... - def equals(self, other): ... - def __eq__(self, other): ... + + def __init__(self, path_or_paths, filesystem=None, schema=None, *, filters=None, read_dictionary=None, binary_type=None, list_type=None, memory_map: bool=False, buffer_size=None, partitioning: str='hive', ignore_prefixes=None, pre_buffer: bool=True, coerce_int96_timestamp_unit=None, decryption_properties=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, page_checksum_verification: bool=False, arrow_extensions_enabled: bool=True) -> None: + ... + + def equals(self, other): + ... + + def __eq__(self, other): + ... + @property - def schema(self): ... - def read(self, columns=None, use_threads: bool = True, use_pandas_metadata: bool = False): ... - def _get_common_pandas_metadata(self): ... - def read_pandas(self, **kwargs): ... + def schema(self): + ... + + def read(self, columns: list[str] | None=None, use_threads: bool=True, use_pandas_metadata: bool=False) -> pyarrow.Table: + ... + + def _get_common_pandas_metadata(self): + ... + + def read_pandas(self, **kwargs): + ... + @property - def fragments(self): ... + def fragments(self): + ... + @property - def files(self): ... + def files(self): + ... + @property - def filesystem(self): ... + def filesystem(self): + ... + @property - def partitioning(self): ... - -def read_table(source, *, columns=None, use_threads: bool = True, schema=None, use_pandas_metadata: bool = False, read_dictionary=None, binary_type=None, list_type=None, memory_map: bool = False, buffer_size: int = 0, partitioning: str = 'hive', filesystem=None, filters=None, ignore_prefixes=None, pre_buffer: bool = True, coerce_int96_timestamp_unit=None, decryption_properties=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, page_checksum_verification: bool = False, arrow_extensions_enabled: bool = True): ... -def read_pandas(source, columns=None, **kwargs): ... -def write_table(table, where, row_group_size=None, version: str = '2.6', use_dictionary: bool = True, compression: str = 'snappy', write_statistics: bool = True, use_deprecated_int96_timestamps=None, coerce_timestamps=None, allow_truncated_timestamps: bool = False, data_page_size=None, flavor=None, filesystem=None, compression_level=None, use_byte_stream_split: bool = False, column_encoding=None, data_page_version: str = '1.0', use_compliant_nested_type: bool = True, encryption_properties=None, write_batch_size=None, dictionary_pagesize_limit=None, store_schema: bool = True, write_page_index: bool = False, write_page_checksum: bool = False, sorting_columns=None, store_decimal_as_integer: bool = False, write_time_adjusted_to_utc: bool = False, max_rows_per_page=None, bloom_filter_options=None, use_content_defined_chunking: bool = False, **kwargs) -> None: ... -def write_to_dataset(table, root_path, partition_cols=None, filesystem=None, schema=None, partitioning=None, basename_template=None, use_threads=None, file_visitor=None, existing_data_behavior=None, **kwargs) -> None: ... -def write_metadata(schema, where, metadata_collector=None, filesystem=None, **kwargs) -> None: ... -def read_metadata(where, memory_map: bool = False, decryption_properties=None, filesystem=None, arrow_extensions_enabled: bool = True): ... -def read_schema(where, memory_map: bool = False, decryption_properties=None, filesystem=None, arrow_extensions_enabled: bool = True): ... -EXCLUDED_PARQUET_PATHS: set[str] + def partitioning(self): + ... diff --git a/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi b/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi index 04211f468f6e..09df26e59b21 100644 --- a/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi +++ b/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete from pyarrow._parquet_encryption import CryptoFactory as CryptoFactory, DecryptionConfiguration as DecryptionConfiguration, EncryptionConfiguration as EncryptionConfiguration, KmsClient as KmsClient, KmsConnectionConfig as KmsConnectionConfig, create_decryption_properties as create_decryption_properties, create_encryption_properties as create_encryption_properties diff --git a/python/pyarrow-stubs/pyarrow/substrait.pyi b/python/pyarrow-stubs/pyarrow/substrait.pyi index bf0b5922e1d8..095f981d3b97 100644 --- a/python/pyarrow-stubs/pyarrow/substrait.pyi +++ b/python/pyarrow-stubs/pyarrow/substrait.pyi @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete from pyarrow._substrait import BoundExpressions as BoundExpressions, SubstraitSchema as SubstraitSchema, deserialize_expressions as deserialize_expressions, deserialize_schema as deserialize_schema, get_supported_functions as get_supported_functions, run_query as run_query, serialize_expressions as serialize_expressions, serialize_schema as serialize_schema diff --git a/python/pyarrow-stubs/pyarrow/types.pyi b/python/pyarrow-stubs/pyarrow/types.pyi index 863d12a3e370..5e140bbc9722 100644 --- a/python/pyarrow-stubs/pyarrow/types.pyi +++ b/python/pyarrow-stubs/pyarrow/types.pyi @@ -15,14 +15,17 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence from _typeshed import Incomplete from enum import IntEnum from pyarrow.lib import is_boolean_value as is_boolean_value, is_float_value as is_float_value, is_integer_value as is_integer_value from pyarrow.util import doc as doc - _SIGNED_INTEGER_TYPES: Incomplete _UNSIGNED_INTEGER_TYPES: Incomplete -_INTEGER_TYPES = _SIGNED_INTEGER_TYPES | _UNSIGNED_INTEGER_TYPES +_INTEGER_TYPES: set[Incomplete] _FLOATING_TYPES: Incomplete _DECIMAL_TYPES: Incomplete _DATE_TYPES: Incomplete @@ -31,6 +34,165 @@ _INTERVAL_TYPES: Incomplete _TEMPORAL_TYPES: Incomplete _UNION_TYPES: Incomplete +def is_null(t: pyarrow.DataType): + ... + +def is_boolean(t: pyarrow.DataType): + ... + +def is_integer(t: pyarrow.DataType): + ... + +def is_signed_integer(t: pyarrow.DataType): + ... + +def is_unsigned_integer(t: pyarrow.DataType): + ... + +def is_int8(t: pyarrow.DataType): + ... + +def is_int16(t: pyarrow.DataType): + ... + +def is_int32(t: pyarrow.DataType): + ... + +def is_int64(t: pyarrow.DataType): + ... + +def is_uint8(t: pyarrow.DataType): + ... + +def is_uint16(t: pyarrow.DataType): + ... + +def is_uint32(t: pyarrow.DataType): + ... + +def is_uint64(t: pyarrow.DataType): + ... + +def is_floating(t: pyarrow.DataType): + ... + +def is_float16(t: pyarrow.DataType): + ... + +def is_float32(t: pyarrow.DataType): + ... + +def is_float64(t: pyarrow.DataType): + ... + +def is_list(t: pyarrow.DataType): + ... + +def is_large_list(t: pyarrow.DataType): + ... + +def is_fixed_size_list(t: pyarrow.DataType): + ... + +def is_list_view(t: pyarrow.DataType): + ... + +def is_large_list_view(t: pyarrow.DataType): + ... + +def is_struct(t: pyarrow.DataType): + ... + +def is_union(t: pyarrow.DataType): + ... + +def is_nested(t: pyarrow.DataType): + ... + +def is_run_end_encoded(t: pyarrow.DataType): + ... + +def is_temporal(t: pyarrow.DataType): + ... + +def is_timestamp(t: pyarrow.DataType): + ... + +def is_duration(t: pyarrow.DataType): + ... + +def is_time(t: pyarrow.DataType): + ... + +def is_time32(t: pyarrow.DataType): + ... + +def is_time64(t: pyarrow.DataType): + ... + +def is_binary(t: pyarrow.DataType): + ... + +def is_large_binary(t: pyarrow.DataType): + ... + +def is_unicode(t: pyarrow.DataType): + ... + +def is_string(t: pyarrow.DataType): + ... + +def is_large_unicode(t: pyarrow.DataType): + ... + +def is_large_string(t: pyarrow.DataType): + ... + +def is_fixed_size_binary(t: pyarrow.DataType): + ... + +def is_binary_view(t: pyarrow.DataType): + ... + +def is_string_view(t: pyarrow.DataType): + ... + +def is_date(t: pyarrow.DataType): + ... + +def is_date32(t: pyarrow.DataType): + ... + +def is_date64(t: pyarrow.DataType): + ... + +def is_map(t: pyarrow.DataType): + ... + +def is_decimal(t: pyarrow.DataType): + ... + +def is_decimal32(t: pyarrow.DataType): + ... + +def is_decimal64(t: pyarrow.DataType): + ... + +def is_decimal128(t: pyarrow.DataType): + ... + +def is_decimal256(t: pyarrow.DataType): + ... + +def is_dictionary(t: pyarrow.DataType): + ... + +def is_interval(t: pyarrow.DataType): + ... + +def is_primitive(t: pyarrow.DataType): + ... + class TypesEnum(IntEnum): NA = ... BOOL = ... @@ -76,57 +238,3 @@ class TypesEnum(IntEnum): INTERVAL_MONTH_DAY_NANO = ... DURATION = ... DICTIONARY = ... - -def is_null(t): ... -def is_boolean(t): ... -def is_integer(t): ... -def is_signed_integer(t): ... -def is_unsigned_integer(t): ... -def is_int8(t): ... -def is_int16(t): ... -def is_int32(t): ... -def is_int64(t): ... -def is_uint8(t): ... -def is_uint16(t): ... -def is_uint32(t): ... -def is_uint64(t): ... -def is_floating(t): ... -def is_float16(t): ... -def is_float32(t): ... -def is_float64(t): ... -def is_list(t): ... -def is_large_list(t): ... -def is_fixed_size_list(t): ... -def is_list_view(t): ... -def is_large_list_view(t): ... -def is_struct(t): ... -def is_union(t): ... -def is_nested(t): ... -def is_run_end_encoded(t): ... -def is_temporal(t): ... -def is_timestamp(t): ... -def is_duration(t): ... -def is_time(t): ... -def is_time32(t): ... -def is_time64(t): ... -def is_binary(t): ... -def is_large_binary(t): ... -def is_unicode(t): ... -def is_string(t): ... -def is_large_unicode(t): ... -def is_large_string(t): ... -def is_fixed_size_binary(t): ... -def is_binary_view(t): ... -def is_string_view(t): ... -def is_date(t): ... -def is_date32(t): ... -def is_date64(t): ... -def is_map(t): ... -def is_decimal(t): ... -def is_decimal32(t): ... -def is_decimal64(t): ... -def is_decimal128(t): ... -def is_decimal256(t): ... -def is_dictionary(t): ... -def is_interval(t): ... -def is_primitive(t): ... diff --git a/python/pyarrow-stubs/pyarrow/util.pyi b/python/pyarrow-stubs/pyarrow/util.pyi index 9cc1eedabab6..1ecd4fa95454 100644 --- a/python/pyarrow-stubs/pyarrow/util.pyi +++ b/python/pyarrow-stubs/pyarrow/util.pyi @@ -15,19 +15,51 @@ # specific language governing permissions and limitations # under the License. +import os +import re +import pyarrow +from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from _typeshed import Incomplete _DEPR_MSG: str -def doc(*docstrings, **params): ... -def _deprecate_api(old_name, new_name, api, next_version, type=...): ... -def _deprecate_class(old_name, new_class, next_version, instancecheck: bool = True): ... -def _is_iterable(obj): ... -def _is_path_like(path): ... -def _stringify_path(path): ... -def product(seq): ... -def get_contiguous_span(shape, strides, itemsize): ... -def find_free_port(): ... -def guid(): ... -def _break_traceback_cycle_from_frame(frame) -> None: ... -def _download_urllib(url, out_path) -> None: ... -def _download_requests(url, out_path) -> None: ... -def download_tzdata_on_windows() -> None: ... +def doc(*docstrings, **params): + ... + +def _deprecate_api(old_name, new_name, api, next_version, type=...): + ... + +def _deprecate_class(old_name, new_class, next_version, instancecheck: bool=True): + ... + +def _is_iterable(obj): + ... + +def _is_path_like(path): + ... + +def _stringify_path(path): + ... + +def product(seq): + ... + +def get_contiguous_span(shape: tuple, strides: tuple, itemsize: int) -> int: + ... + +def find_free_port(): + ... + +def guid(): + ... + +def _break_traceback_cycle_from_frame(frame) -> None: + ... + +def _download_urllib(url, out_path) -> None: + ... + +def _download_requests(url, out_path) -> None: + ... + +def download_tzdata_on_windows() -> None: + ... From 4dd31d79be8e1aef4e732733c6143b221cd352a6 Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Fri, 9 Oct 2026 18:20:52 +0100 Subject: [PATCH 7/8] [Python] Annotate the stubs from MonkeyType traces of the test suite The second pass of scripts/add_types_to_stubs.py adds what tracing the test suite under MonkeyType observed and the sources did not type. --- python/pyarrow-stubs/pyarrow/__init__.pyi | 4 +- python/pyarrow-stubs/pyarrow/_compute.pyi | 14 +-- python/pyarrow-stubs/pyarrow/_csv.pyi | 12 +- python/pyarrow-stubs/pyarrow/_flight.pyi | 12 +- python/pyarrow-stubs/pyarrow/acero.pyi | 11 +- python/pyarrow-stubs/pyarrow/compute.pyi | 16 ++- python/pyarrow-stubs/pyarrow/dataset.pyi | 31 ++--- python/pyarrow-stubs/pyarrow/feather.pyi | 16 ++- python/pyarrow-stubs/pyarrow/fs.pyi | 11 +- .../pyarrow/interchange/column.pyi | 1 + .../pyarrow/interchange/dataframe.pyi | 1 + .../pyarrow/interchange/from_dataframe.pyi | 6 +- python/pyarrow-stubs/pyarrow/ipc.pyi | 24 ++-- python/pyarrow-stubs/pyarrow/lib.pyi | 88 +++++++------- python/pyarrow-stubs/pyarrow/orc.pyi | 25 ++-- .../pyarrow-stubs/pyarrow/pandas_compat.pyi | 59 +++++----- python/pyarrow-stubs/pyarrow/parquet/core.pyi | 48 ++++---- python/pyarrow-stubs/pyarrow/types.pyi | 108 +++++++++--------- python/pyarrow-stubs/pyarrow/util.pyi | 20 ++-- 19 files changed, 281 insertions(+), 226 deletions(-) diff --git a/python/pyarrow-stubs/pyarrow/__init__.pyi b/python/pyarrow-stubs/pyarrow/__init__.pyi index b617162a9377..843d3dc07db4 100644 --- a/python/pyarrow-stubs/pyarrow/__init__.pyi +++ b/python/pyarrow-stubs/pyarrow/__init__.pyi @@ -37,13 +37,13 @@ def _module_is_available(module): def _filesystem_is_available(fs): ... -def have_libhdfs(): +def have_libhdfs() -> bool: ... def show_info() -> None: ... -def get_include(): +def get_include() -> str: ... def _get_pkg_config_executable(): diff --git a/python/pyarrow-stubs/pyarrow/_compute.pyi b/python/pyarrow-stubs/pyarrow/_compute.pyi index cb4715d597b6..665644920841 100644 --- a/python/pyarrow-stubs/pyarrow/_compute.pyi +++ b/python/pyarrow-stubs/pyarrow/_compute.pyi @@ -30,13 +30,6 @@ from typing_extensions import TypeAlias __pas: Incomplete _substrait_msg: str SUPPORTED_INPUT_ARR_TYPES: tuple[Incomplete, ...] - -class FunctionDoc(NamedTuple): - summary: Incomplete - description: Incomplete - arg_names: Incomplete - options_class: Incomplete - options_required: Incomplete _deserialize: Incomplete def _pas(): @@ -87,6 +80,13 @@ def _register_user_defined_function(register_func, func: Callable[..., Any], fun def call_tabular_function(function_name: str, args: Iterable[Any] | None=None, func_registry: FunctionRegistry | None=None) -> pyarrow.RecordBatchReader: ... +class FunctionDoc(NamedTuple): + summary: Incomplete + description: Incomplete + arg_names: Incomplete + options_class: Incomplete + options_required: Incomplete + class Kernel(_Weakrefable): def __init__(self): diff --git a/python/pyarrow-stubs/pyarrow/_csv.pyi b/python/pyarrow-stubs/pyarrow/_csv.pyi index a8bcb022b59f..6c6c8dfb1ff9 100644 --- a/python/pyarrow-stubs/pyarrow/_csv.pyi +++ b/python/pyarrow-stubs/pyarrow/_csv.pyi @@ -25,12 +25,6 @@ from pyarrow.lib import __name__ as __name__ from _typeshed import Incomplete from collections import namedtuple from pyarrow.lib import MemoryPool, RecordBatchReader, Schema, _CRecordBatchWriter, _Weakrefable - -class _InvalidRow(NamedTuple): - expected_columns: Incomplete - actual_columns: Incomplete - number: Incomplete - text: Incomplete ISO8601: _ISO8601 def read_csv(input_file, read_options: pyarrow.csv.ReadOptions | None=None, parse_options: pyarrow.csv.ParseOptions | None=None, convert_options: pyarrow.csv.ConvertOptions | None=None, memory_pool: MemoryPool | None=None) -> pyarrow.Table: @@ -45,6 +39,12 @@ def _raise_invalid_function_option(value, description, *, exception_class=ValueE def write_csv(data: pyarrow.RecordBatch | pyarrow.Table, output_file, write_options: pyarrow.csv.WriteOptions | None=None, memory_pool: MemoryPool | None=None) -> None: ... +class _InvalidRow(NamedTuple): + expected_columns: Incomplete + actual_columns: Incomplete + number: Incomplete + text: Incomplete + class InvalidRow(_InvalidRow): __slots__ = () diff --git a/python/pyarrow-stubs/pyarrow/_flight.pyi b/python/pyarrow-stubs/pyarrow/_flight.pyi index 225c8ed34711..e4bc8a887400 100644 --- a/python/pyarrow-stubs/pyarrow/_flight.pyi +++ b/python/pyarrow-stubs/pyarrow/_flight.pyi @@ -31,6 +31,12 @@ from pyarrow.lib import * from pyarrow.lib import ArrowCancelled as ArrowCancelled, ArrowException as ArrowException, ArrowInvalid as ArrowInvalid _FLIGHT_SERVER_ERROR_REGEX: re.Pattern[str] +def _munge_grpc_python_error(message): + ... + +def connect(location: str | tuple | Location, **kwargs) -> FlightClient: + ... + class _CertKeyPair(NamedTuple): cert: Incomplete key: Incomplete @@ -42,12 +48,6 @@ class _ActionType(NamedTuple): class _CallInfo(NamedTuple): method: Incomplete -def _munge_grpc_python_error(message): - ... - -def connect(location: str | tuple | Location, **kwargs) -> FlightClient: - ... - class FlightCallOptions(_Weakrefable): def __init__(self, timeout: float | None=None, write_options: pyarrow.ipc.IpcWriteOptions | None=None, headers=None, read_options: IpcReadOptions | None=None): diff --git a/python/pyarrow-stubs/pyarrow/acero.pyi b/python/pyarrow-stubs/pyarrow/acero.pyi index 4e3b177f7980..c8f84d4846f4 100644 --- a/python/pyarrow-stubs/pyarrow/acero.pyi +++ b/python/pyarrow-stubs/pyarrow/acero.pyi @@ -15,6 +15,9 @@ # specific language governing permissions and limitations # under the License. +from pyarrow._compute import CountOptions, PivotWiderOptions +from pyarrow._dataset import FileSystemDataset, InMemoryDataset, UnionDataset +from typing import List, Optional, Tuple, Type, Union import os import re import pyarrow @@ -28,22 +31,22 @@ from pyarrow.compute import Expression as Expression, field as field from pyarrow.lib import RecordBatch as RecordBatch, Table as Table, array as array ds: Incomplete -def _dataset_to_decl(dataset, use_threads: bool=True, implicit_ordering: bool=False): +def _dataset_to_decl(dataset: FileSystemDataset | UnionDataset | InMemoryDataset, use_threads: bool=True, implicit_ordering: bool=False) -> Declaration: ... def _perform_join(join_type: str, left_operand: Table | pyarrow.dataset.Dataset, left_keys: str | list[str], right_operand: Table | pyarrow.dataset.Dataset, right_keys: str | list[str], left_suffix: str | None=None, right_suffix: str | None=None, use_threads: bool=True, coalesce_keys: bool=False, output_type=..., filter_expression: pyarrow.compute.Expression | None=None) -> Table | pyarrow.dataset.InMemoryDataset: ... -def _perform_join_asof(left_operand: Table | pyarrow.dataset.Dataset, left_on: str, left_by, right_operand: Table | pyarrow.dataset.Dataset, right_on: str | list[str], right_by, tolerance: int, use_threads: bool=True, output_type=...) -> Table | pyarrow.dataset.InMemoryDataset: +def _perform_join_asof(left_operand: Table | pyarrow.dataset.Dataset, left_on: str, left_by: list[Any] | str | list[str], right_operand: Table | pyarrow.dataset.Dataset, right_on: str | list[str], right_by: list[Any] | str | list[str], tolerance: int, use_threads: bool=True, output_type=...) -> Table | pyarrow.dataset.InMemoryDataset: ... def _filter_table(table: Table | RecordBatch, expression: Expression) -> Table | RecordBatch: ... -def _sort_source(table_or_dataset, sort_keys, output_type=..., **kwargs): +def _sort_source(table_or_dataset: FileSystemDataset | InMemoryDataset, sort_keys: list[tuple[str, str, str]], output_type=..., **kwargs) -> InMemoryDataset: ... -def _group_by(table, aggregates, keys, use_threads: bool=True): +def _group_by(table: Table, aggregates: list[tuple[tuple[str, str], str, None, str] | tuple[list[Any], str, None, str] | tuple[list[str], str, CountOptions, str] | Any | tuple[list[str], str, None, str] | tuple[tuple[str, str], str, PivotWiderOptions, str]], keys: list[str], use_threads: bool=True) -> Table: ... class DatasetModuleStub: diff --git a/python/pyarrow-stubs/pyarrow/compute.pyi b/python/pyarrow-stubs/pyarrow/compute.pyi index 246ed53633ad..231cbf040bec 100644 --- a/python/pyarrow-stubs/pyarrow/compute.pyi +++ b/python/pyarrow-stubs/pyarrow/compute.pyi @@ -15,6 +15,10 @@ # specific language governing permissions and limitations # under the License. +from pyarrow._compute import MetaFunction +from pyarrow.lib import BinaryArray, BinaryScalar, BinaryViewArray, Bool8Array, Bool8Type, BooleanArray, ChunkedArray, DataType, Date32Array, Date32Scalar, Date64Array, Date64Scalar, Decimal128Array, Decimal128Scalar, Decimal128Type, Decimal256Array, Decimal256Scalar, Decimal256Type, Decimal32Scalar, Decimal32Type, Decimal64Scalar, Decimal64Type, DenseUnionType, DictionaryArray, DictionaryType, DoubleArray, DoubleScalar, DurationArray, DurationType, ExtensionArray, FixedShapeTensorArray, FixedShapeTensorType, FixedSizeBinaryArray, FixedSizeBinaryType, FixedSizeListArray, FixedSizeListType, FloatArray, FloatScalar, HalfFloatArray, Int16Array, Int16Scalar, Int32Array, Int64Array, Int64Scalar, Int8Array, Int8Scalar, JsonArray, JsonType, LargeBinaryArray, LargeListArray, LargeListType, LargeStringArray, LargeStringScalar, ListArray, ListType, MapArray, MapType, MonthDayNanoIntervalArray, NullArray, NullScalar, OpaqueArray, OpaqueType, RecordBatch, SparseUnionType, StringArray, StringScalar, StringViewArray, StructArray, StructType, Table, Time32Array, Time64Array, TimestampArray, TimestampScalar, TimestampType, UInt16Array, UInt16Scalar, UInt32Array, UInt32Scalar, UInt64Array, UInt64Scalar, UInt8Array, UInt8Scalar +from pyarrow.tests.test_extension_type import ExampleUuidType, ExampleUuidType2, IntegerEmbeddedType, IntegerType, MyFixedListType, MyListType, PeriodType, PeriodTypeWithClass, PeriodTypeWithToPandasDtype, TinyIntType +from typing import Dict, List, Optional, Tuple, Type, Union import os import re from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence @@ -209,7 +213,7 @@ def bit_wise_xor(x, y, *, memory_pool: Incomplete | None=...): def case_when(cond, *cases, memory_pool: Incomplete | None=...): ... -def cast(arr, target_type: Incomplete | None=..., safe: Incomplete | None=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...) -> pyarrow.Array: +def cast(arr: Table | Int16Array | StringViewArray | UInt64Array | Date32Array | JsonArray | DurationArray | UInt16Array | HalfFloatArray | Date64Array | Int64Scalar | ChunkedArray | StringScalar | FloatScalar | FixedShapeTensorArray | DoubleScalar | Int32Array | DictionaryArray | FloatArray | TimestampArray | Decimal128Array | LargeStringArray | ListArray | NullArray | StringArray | OpaqueArray | BinaryScalar | Int8Array | UInt32Array | MapArray | DoubleArray | Time32Array | LargeListArray | BooleanArray | ExtensionArray | BinaryArray | TimestampScalar | Bool8Array | UInt8Array | Int64Array | FixedSizeBinaryArray | Time64Array | StructArray | FixedSizeListArray | Int8Scalar, target_type: Incomplete | None=..., safe: Incomplete | None=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...) -> pyarrow.Array: ... def ceil(x, *, memory_pool: Incomplete | None=...): @@ -359,7 +363,7 @@ def hypot(x, y, *, memory_pool: Incomplete | None=...): def if_else(cond, left, right, *, memory_pool: Incomplete | None=...): ... -def index(data, value, start: Incomplete | None=..., end: Incomplete | None=..., *, memory_pool: Incomplete | None=...) -> int: +def index(data: ChunkedArray | Int64Array, value: Int64Scalar | int | Int8Scalar | None, start: Incomplete | None=..., end: Incomplete | None=..., *, memory_pool: Incomplete | None=...) -> int: ... def index_in(values, value_set: pyarrow.Array, *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): @@ -599,7 +603,7 @@ def quarter(values, *, memory_pool: Incomplete | None=...): def quarters_between(start, end, *, memory_pool: Incomplete | None=...): ... -def random(n: int, *, initializer: str=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): +def random(n: int, *, initializer: str=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...) -> DoubleArray: ... def rank(input, sort_keys: str=..., *, null_placement: Incomplete | None=..., tiebreaker: str=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): @@ -731,7 +735,7 @@ def subtract_checked(x, y, *, memory_pool: Incomplete | None=...): def sum(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): ... -def take(data: pyarrow.Array | pyarrow.ChunkedArray | pyarrow.RecordBatch | pyarrow.Table, indices: pyarrow.Array | pyarrow.ChunkedArray, *, boundscheck: bool=..., memory_pool: Incomplete | None=...): +def take(data: pyarrow.Array | pyarrow.ChunkedArray | pyarrow.RecordBatch | pyarrow.Table, indices: pyarrow.Array | pyarrow.ChunkedArray, *, boundscheck: bool=..., memory_pool: Incomplete | None=...) -> Table | Int16Array | UInt64Array | UInt16Array | ChunkedArray | Int32Array | DictionaryArray | FloatArray | ListArray | NullArray | StringArray | Int8Array | UInt32Array | DoubleArray | LargeBinaryArray | LargeListArray | BooleanArray | ExtensionArray | BinaryArray | UInt8Array | Int64Array | FixedSizeBinaryArray | RecordBatch | StructArray: ... def tan(x, *, memory_pool: Incomplete | None=...): @@ -884,13 +888,13 @@ def years_between(start, end, *, memory_pool: Incomplete | None=...): def utf8_zfill(strings, width: int, padding: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): ... -def fill_null(values, fill_value): +def fill_null(values: ChunkedArray | Int16Array | UInt64Array | StringArray | NullArray | UInt8Array | Int64Array | Int8Array | DoubleArray | Int32Array | FloatArray | LargeBinaryArray | UInt16Array | BooleanArray, fill_value: Int64Scalar | NullScalar | FloatScalar | bool | str | Int8Array | UInt16Scalar | UInt8Scalar | UInt64Scalar | DoubleScalar | Int16Scalar | int | Int8Scalar | None) -> ChunkedArray | Int16Array | UInt64Array | StringArray | NullArray | UInt8Array | Int64Array | Int8Array | DoubleArray | Int32Array | FloatArray | LargeBinaryArray | UInt16Array | BooleanArray: ... def top_k_unstable(values: pyarrow.Array | pyarrow.ChunkedArray | pyarrow.RecordBatch | pyarrow.Table, k: int, sort_keys: Incomplete | None=..., null_placements: Incomplete | None=..., *, memory_pool: Incomplete | None=...) -> pyarrow.Array: ... -def bottom_k_unstable(values: pyarrow.Array | pyarrow.ChunkedArray | pyarrow.RecordBatch | pyarrow.Table, k: int, sort_keys: Incomplete | None=..., null_placements: Incomplete | None=..., *, memory_pool: Incomplete | None=...): +def bottom_k_unstable(values: pyarrow.Array | pyarrow.ChunkedArray | pyarrow.RecordBatch | pyarrow.Table, k: int, sort_keys: Incomplete | None=..., null_placements: Incomplete | None=..., *, memory_pool: Incomplete | None=...) -> UInt64Array: ... def field(*name_or_index) -> Expression: diff --git a/python/pyarrow-stubs/pyarrow/dataset.pyi b/python/pyarrow-stubs/pyarrow/dataset.pyi index 80d65fd8ec9a..e9cbe7950502 100644 --- a/python/pyarrow-stubs/pyarrow/dataset.pyi +++ b/python/pyarrow-stubs/pyarrow/dataset.pyi @@ -15,6 +15,11 @@ # specific language governing permissions and limitations # under the License. +from pathlib import Path +from pyarrow._fs import FileSelector, LocalFileSystem, PyFileSystem, SubTreeFileSystem, _MockFileSystem +from pyarrow.lib import RecordBatch, RecordBatchReader, Schema, Table +from pyarrow.tests.util import FSProtocolClass +from typing import List, Optional, Tuple, Union import os import re import pyarrow @@ -33,41 +38,41 @@ _orc_msg: str _parquet_available: bool _parquet_msg: str -def __getattr__(name) -> None: +def __getattr__(name: str) -> None: ... -def partitioning(schema: pyarrow.Schema | None=None, field_names: list[str] | None=None, flavor: str | None=None, dictionaries=None) -> Partitioning | PartitioningFactory: +def partitioning(schema: pyarrow.Schema | None=None, field_names: list[str] | None=None, flavor: str | None=None, dictionaries: str | None=None) -> Partitioning | PartitioningFactory: ... -def _ensure_partitioning(scheme): +def _ensure_partitioning(scheme: str | list[str] | PartitioningFactory | FilenamePartitioning | DirectoryPartitioning | int | HivePartitioning | None) -> FilenamePartitioning | PartitioningFactory | HivePartitioning | DirectoryPartitioning | None: ... -def _ensure_format(obj): +def _ensure_format(obj: str | OrcFileFormat | JsonFileFormat | IpcFileFormat | CsvFileFormat | ParquetFileFormat) -> OrcFileFormat | JsonFileFormat | IpcFileFormat | CsvFileFormat | FeatherFileFormat | ParquetFileFormat: ... -def _ensure_multiple_sources(paths: list[os.PathLike[str]], filesystem: pyarrow.fs.FileSystem | str | None=None): +def _ensure_multiple_sources(paths: list[os.PathLike[str]], filesystem: pyarrow.fs.FileSystem | str | None=None) -> tuple[_MockFileSystem, list[str]] | tuple[LocalFileSystem, list[Any]] | tuple[LocalFileSystem, list[str]] | tuple[SubTreeFileSystem, list[str]]: ... -def _ensure_single_source(path: os.PathLike[str], filesystem: pyarrow.fs.FileSystem | str | None=None): +def _ensure_single_source(path: os.PathLike[str], filesystem: pyarrow.fs.FileSystem | str | None=None) -> tuple[PyFileSystem, FileSelector] | tuple[_MockFileSystem, FileSelector] | tuple[LocalFileSystem, list[str]] | tuple[LocalFileSystem, FileSelector] | tuple[SubTreeFileSystem, FileSelector] | tuple[SubTreeFileSystem, list[str]]: ... -def _filesystem_dataset(source, schema=None, filesystem=None, partitioning=None, format=None, partition_base_dir=None, exclude_invalid_files=None, selector_ignore_prefixes=None) -> FileSystemDataset: +def _filesystem_dataset(source, schema: Schema | None=None, filesystem: _MockFileSystem | PyFileSystem | str | SubTreeFileSystem | LocalFileSystem | None=None, partitioning: FilenamePartitioning | str | list[str] | PartitioningFactory | HivePartitioning | DirectoryPartitioning | int | None=None, format: str | OrcFileFormat | JsonFileFormat | IpcFileFormat | CsvFileFormat | ParquetFileFormat | None=None, partition_base_dir=None, exclude_invalid_files=None, selector_ignore_prefixes: list[str] | None=None) -> FileSystemDataset: ... -def _in_memory_dataset(source, schema=None, **kwargs): +def _in_memory_dataset(source: Table | list[RecordBatch] | RecordBatch | list[Table] | RecordBatchReader, schema=None, **kwargs) -> InMemoryDataset: ... -def _union_dataset(children, schema=None, **kwargs): +def _union_dataset(children: list[FileSystemDataset] | tuple[InMemoryDataset, InMemoryDataset] | tuple[FileSystemDataset, FileSystemDataset], schema: Schema | None=None, **kwargs) -> UnionDataset: ... -def parquet_dataset(metadata_path, schema: pyarrow.Schema | None=None, filesystem=None, format: ParquetFileFormat | None=None, partitioning: Partitioning | PartitioningFactory | str | list[str] | None=None, partition_base_dir: str | None=None) -> FileSystemDataset: +def parquet_dataset(metadata_path: str, schema: pyarrow.Schema | None=None, filesystem=None, format: ParquetFileFormat | None=None, partitioning: Partitioning | PartitioningFactory | str | list[str] | None=None, partition_base_dir: str | None=None) -> FileSystemDataset: ... -def dataset(source, schema: pyarrow.Schema | None=None, format: FileFormat | str | None=None, filesystem=None, partitioning: Partitioning | PartitioningFactory | str | list[str] | None=None, partition_base_dir: str | None=None, exclude_invalid_files: bool | None=None, ignore_prefixes: list | None=None) -> Dataset: +def dataset(source, schema: pyarrow.Schema | None=None, format: FileFormat | str | None=None, filesystem: _MockFileSystem | PyFileSystem | str | SubTreeFileSystem | LocalFileSystem | None=None, partitioning: Partitioning | PartitioningFactory | str | list[str] | None=None, partition_base_dir: str | None=None, exclude_invalid_files: bool | None=None, ignore_prefixes: list | None=None) -> Dataset: ... -def _ensure_write_partitioning(part, schema, flavor): +def _ensure_write_partitioning(part: list[str] | FilenamePartitioning | DirectoryPartitioning | HivePartitioning | None, schema: Schema, flavor: str | None) -> DirectoryPartitioning | HivePartitioning | FilenamePartitioning: ... -def write_dataset(data, base_dir: str, *, basename_template: str | None=None, format: FileFormat | str | None=None, partitioning: Partitioning | list[str] | None=None, partitioning_flavor: str | None=None, schema: pyarrow.Schema | None=None, filesystem: pyarrow.fs.FileSystem | None=None, file_options: pyarrow.dataset.FileWriteOptions | None=None, use_threads: bool=True, preserve_order: bool=False, max_partitions: int | None=None, max_open_files: int | None=None, max_rows_per_file: int | None=None, min_rows_per_group: int | None=None, max_rows_per_group: int | None=None, file_visitor: Callable[..., Any] | None=None, existing_data_behavior: str='error', create_dir: bool=True) -> None: +def write_dataset(data: Table | FileSystemDataset | Scanner | list[Table] | InMemoryDataset, base_dir: str, *, basename_template: str | None=None, format: FileFormat | str | None=None, partitioning: Partitioning | list[str] | None=None, partitioning_flavor: str | None=None, schema: pyarrow.Schema | None=None, filesystem: pyarrow.fs.FileSystem | None=None, file_options: pyarrow.dataset.FileWriteOptions | None=None, use_threads: bool=True, preserve_order: bool=False, max_partitions: int | None=None, max_open_files: int | None=None, max_rows_per_file: int | None=None, min_rows_per_group: int | None=None, max_rows_per_group: int | None=None, file_visitor: Callable[..., Any] | None=None, existing_data_behavior: str='error', create_dir: bool=True) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/feather.pyi b/python/pyarrow-stubs/pyarrow/feather.pyi index cf741d5adc74..8e824a3a4a53 100644 --- a/python/pyarrow-stubs/pyarrow/feather.pyi +++ b/python/pyarrow-stubs/pyarrow/feather.pyi @@ -15,6 +15,10 @@ # specific language governing permissions and limitations # under the License. +from io import BytesIO +from pathlib import Path +from pyarrow.lib import ChunkedArray +from typing import List, Optional, Union import os import re import pyarrow @@ -25,27 +29,27 @@ from pyarrow._feather import FeatherError as FeatherError from pyarrow.lib import Codec as Codec, Table as Table, concat_tables as concat_tables, schema as schema from pyarrow.pandas_compat import _pandas_api as _pandas_api -def check_chunked_overflow(name, col) -> None: +def check_chunked_overflow(name: str, col: ChunkedArray) -> None: ... _FEATHER_SUPPORTED_CODECS: Incomplete def write_feather(df, dest: str, compression: str | None=None, compression_level: int | None=None, chunksize: int | None=None, version: int=2) -> None: ... -def read_feather(source, columns: Sequence[Any] | None=None, use_threads: bool=True, memory_map: bool=False, **kwargs): +def read_feather(source: BytesIO | Path | str, columns: Sequence[Any] | None=None, use_threads: bool=True, memory_map: bool=False, **kwargs): ... -def _read_table_internal(source, columns=None, memory_map: bool=False, use_threads: bool=True): +def _read_table_internal(source: Path | BytesIO | str, columns: list[int] | Iterator[Any] | list[str] | None=None, memory_map: bool=False, use_threads: bool=True) -> Table: ... -def read_table(source, columns: Sequence[Any] | None=None, memory_map: bool=False, use_threads: bool=True) -> Table: +def read_table(source: Path | str, columns: Sequence[Any] | None=None, memory_map: bool=False, use_threads: bool=True) -> Table: ... class FeatherDataset: paths: Incomplete validate_schema: Incomplete - def __init__(self, path_or_paths, validate_schema: bool=True) -> None: + def __init__(self, path_or_paths: list[str], validate_schema: bool=True) -> None: ... _tables: Incomplete schema: Incomplete @@ -53,7 +57,7 @@ class FeatherDataset: def read_table(self, columns: list[str] | None=None) -> Table: ... - def validate_schemas(self, piece, table) -> None: + def validate_schemas(self, piece: str, table: Table) -> None: ... def read_pandas(self, columns: list[str] | None=None, use_threads: bool=True): diff --git a/python/pyarrow-stubs/pyarrow/fs.pyi b/python/pyarrow-stubs/pyarrow/fs.pyi index 58ddf41f685f..e9040396d0cb 100644 --- a/python/pyarrow-stubs/pyarrow/fs.pyi +++ b/python/pyarrow-stubs/pyarrow/fs.pyi @@ -15,6 +15,11 @@ # specific language governing permissions and limitations # under the License. +from io import BufferedReader, BufferedWriter, BytesIO +from pathlib import Path +from pyarrow.lib import Buffer, BufferOutputStream, BufferReader, NativeFile, OSFile +from pyarrow.tests.util import FSProtocolClass +from typing import Optional, Tuple, Union import os import re import pyarrow @@ -29,13 +34,13 @@ from pyarrow.util import _is_path_like as _is_path_like, _stringify_path as _str FileStats = FileInfo _not_imported: Incomplete -def __getattr__(name) -> None: +def __getattr__(name: str) -> None: ... -def _ensure_filesystem(filesystem, *, use_mmap: bool=False): +def _ensure_filesystem(filesystem: PyFileSystem | _MockFileSystem | str | SubTreeFileSystem | LocalFileSystem, *, use_mmap: bool=False) -> PyFileSystem | SubTreeFileSystem | _MockFileSystem | LocalFileSystem: ... -def _resolve_filesystem_and_path(path, filesystem=None, *, memory_map: bool=False): +def _resolve_filesystem_and_path(path, filesystem: _MockFileSystem | PyFileSystem | str | SubTreeFileSystem | LocalFileSystem | None=None, *, memory_map: bool=False) -> tuple[PyFileSystem, str] | tuple[None, BufferedReader] | tuple[None, NativeFile] | tuple[None, BufferOutputStream] | tuple[LocalFileSystem, str] | tuple[_MockFileSystem, str] | tuple[None, None] | tuple[None, BytesIO] | tuple[None, OSFile] | tuple[SubTreeFileSystem, str] | tuple[None, Buffer] | tuple[None, BufferedWriter] | tuple[None, BufferReader]: ... def copy_files(source: str, destination: str, source_filesystem: FileSystem | None=None, destination_filesystem: FileSystem | None=None, *, chunk_size: int=..., use_threads: bool=True) -> None: diff --git a/python/pyarrow-stubs/pyarrow/interchange/column.pyi b/python/pyarrow-stubs/pyarrow/interchange/column.pyi index 780ae251cc65..6461bfafa1ca 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/column.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/column.pyi @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +from typing import Optional import os import re import pyarrow diff --git a/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi b/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi index 693e7fc9c5ec..82c922def5d4 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +from typing import Optional import os import re import pyarrow diff --git a/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi b/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi index f299d3c4578f..56300ed43591 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +from pyarrow.lib import DataType, Table, TimestampType +from typing import Tuple, Union import os import re import pyarrow @@ -46,10 +48,10 @@ def bool_column_to_array(col: ColumnObject, allow_copy: bool=True) -> pa.Array: def categorical_column_to_dictionary(col: ColumnObject, allow_copy: bool=True) -> pa.DictionaryArray: ... -def parse_datetime_format_str(format_str): +def parse_datetime_format_str(format_str: str) -> tuple[str, str]: ... -def map_date_type(data_type): +def map_date_type(data_type: tuple[DtypeKind, int, str, str] | tuple[DtypeKind, int, str, str]) -> DataType | TimestampType: ... def buffers_to_array(buffers: ColumnBuffers, data_type: tuple[DtypeKind, int, str, str], length: int, describe_null: ColumnNullType, offset: int=0, allow_copy: bool=True) -> pa.Array: diff --git a/python/pyarrow-stubs/pyarrow/ipc.pyi b/python/pyarrow-stubs/pyarrow/ipc.pyi index 73a20d7f1088..31eba3df47a4 100644 --- a/python/pyarrow-stubs/pyarrow/ipc.pyi +++ b/python/pyarrow-stubs/pyarrow/ipc.pyi @@ -15,6 +15,10 @@ # specific language governing permissions and limitations # under the License. +from io import BufferedWriter, BytesIO +from pathlib import Path +from pyarrow.lib import Buffer, BufferOutputStream, BufferReader, CompressedInputStream, CompressedOutputStream, MockOutputStream, NativeFile, OSFile, Schema +from typing import Optional, Union import os import re import pyarrow @@ -27,48 +31,48 @@ from pyarrow.lib import Alignment as Alignment, IpcReadOptions as IpcReadOptions _ipc_writer_class_doc: str _ipc_file_writer_class_doc: Incomplete -def _get_legacy_format_default(options): +def _get_legacy_format_default(options: IpcWriteOptions | None) -> IpcWriteOptions: ... -def _ensure_default_ipc_read_options(options): +def _ensure_default_ipc_read_options(options: IpcReadOptions | bool | None) -> IpcReadOptions: ... def new_stream(sink: str | pyarrow.NativeFile | IO[Any], schema: pyarrow.Schema, *, options: pyarrow.ipc.IpcWriteOptions | None=None) -> RecordBatchStreamWriter: ... -def open_stream(source, *, options: pyarrow.ipc.IpcReadOptions | None=None, memory_pool: pyarrow.MemoryPool | None=None) -> RecordBatchStreamReader: +def open_stream(source: BytesIO | bytes | BufferReader | Buffer, *, options: pyarrow.ipc.IpcReadOptions | None=None, memory_pool: pyarrow.MemoryPool | None=None) -> RecordBatchStreamReader: ... def new_file(sink: str | pyarrow.NativeFile | IO[Any], schema: pyarrow.Schema, *, options: pyarrow.ipc.IpcWriteOptions | None=None, metadata: dict | pyarrow.KeyValueMetadata | None=None) -> RecordBatchFileWriter: ... -def open_file(source, footer_offset: int | None=None, *, options: pyarrow.ipc.IpcReadOptions | None=None, memory_pool: pyarrow.MemoryPool | None=None) -> RecordBatchFileReader: +def open_file(source: Path | BufferReader | Buffer | BytesIO | OSFile | bytes, footer_offset: int | None=None, *, options: pyarrow.ipc.IpcReadOptions | None=None, memory_pool: pyarrow.MemoryPool | None=None) -> RecordBatchFileReader: ... -def serialize_pandas(df, *, nthreads: int | None=None, preserve_index: bool | None=None): +def serialize_pandas(df, *, nthreads: int | None=None, preserve_index: bool | None=None) -> Buffer: ... -def deserialize_pandas(buf, *, use_threads: bool=True): +def deserialize_pandas(buf: Buffer, *, use_threads: bool=True): ... class RecordBatchStreamReader(lib._RecordBatchStreamReader): - def __init__(self, source, *, options=None, memory_pool=None) -> None: + def __init__(self, source: BufferReader | CompressedInputStream | Buffer | BytesIO | bytes, *, options=None, memory_pool=None) -> None: ... class RecordBatchStreamWriter(lib._RecordBatchStreamWriter): __doc__: Incomplete - def __init__(self, sink, schema, *, options=None) -> None: + def __init__(self, sink: BufferOutputStream | BufferedWriter | MockOutputStream | CompressedOutputStream | BytesIO, schema: Schema, *, options=None) -> None: ... class RecordBatchFileReader(lib._RecordBatchFileReader): - def __init__(self, source, footer_offset=None, *, options=None, memory_pool=None) -> None: + def __init__(self, source: Path | BufferReader | Buffer | BytesIO | OSFile | bytes, footer_offset=None, *, options=None, memory_pool=None) -> None: ... class RecordBatchFileWriter(lib._RecordBatchFileWriter): __doc__: Incomplete - def __init__(self, sink, schema, *, options=None, metadata=None) -> None: + def __init__(self, sink: BufferOutputStream | MockOutputStream | str | BytesIO | NativeFile | OSFile, schema: Schema, *, options=None, metadata=None) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/lib.pyi b/python/pyarrow-stubs/pyarrow/lib.pyi index 91037d4a5e34..1359f11e6e98 100644 --- a/python/pyarrow-stubs/pyarrow/lib.pyi +++ b/python/pyarrow-stubs/pyarrow/lib.pyi @@ -81,32 +81,6 @@ __cuda_loaded: Incomplete ordered_dict = dict ArrowIOError = IOError have_signal_refcycle: bool - -class VersionInfo(NamedTuple): - major: Incomplete - minor: Incomplete - patch: Incomplete - -class RuntimeInfo(NamedTuple): - simd_level: Incomplete - detected_simd_level: Incomplete - -class BuildInfo(NamedTuple): - build_type: Incomplete - cpp_build_info: Incomplete - -class CppBuildInfo(NamedTuple): - version: Incomplete - version_info: Incomplete - so_version: Incomplete - full_so_version: Incomplete - compiler_id: Incomplete - compiler_version: Incomplete - compiler_flags: Incomplete - git_id: Incomplete - git_description: Incomplete - package_kind: Incomplete - build_type: Incomplete build_info: BuildInfo cpp_build_info: CppBuildInfo cpp_version: str @@ -116,24 +90,6 @@ _python_extension_types_registry: list[Incomplete] _registry_nanny: _ExtensionRegistryNanny DEFAULT_BUFFER_SIZE: int -class _WriteStats(NamedTuple): - num_messages: Incomplete - num_record_batches: Incomplete - num_dictionary_batches: Incomplete - num_dictionary_deltas: Incomplete - num_replaced_dictionaries: Incomplete - -class _ReadStats(NamedTuple): - num_messages: Incomplete - num_record_batches: Incomplete - num_dictionary_batches: Incomplete - num_dictionary_deltas: Incomplete - num_replaced_dictionaries: Incomplete - -class _RecordBatchWithMetadata(NamedTuple): - batch: Incomplete - custom_metadata: Incomplete - def cpu_count(): ... @@ -562,6 +518,50 @@ def benchmark_PandasObjectIsNull(obj: list[Any]) -> None: _NULL: NullScalar NA: NullScalar +class VersionInfo(NamedTuple): + major: Incomplete + minor: Incomplete + patch: Incomplete + +class RuntimeInfo(NamedTuple): + simd_level: Incomplete + detected_simd_level: Incomplete + +class BuildInfo(NamedTuple): + build_type: Incomplete + cpp_build_info: Incomplete + +class CppBuildInfo(NamedTuple): + version: Incomplete + version_info: Incomplete + so_version: Incomplete + full_so_version: Incomplete + compiler_id: Incomplete + compiler_version: Incomplete + compiler_flags: Incomplete + git_id: Incomplete + git_description: Incomplete + package_kind: Incomplete + build_type: Incomplete + +class _WriteStats(NamedTuple): + num_messages: Incomplete + num_record_batches: Incomplete + num_dictionary_batches: Incomplete + num_dictionary_deltas: Incomplete + num_replaced_dictionaries: Incomplete + +class _ReadStats(NamedTuple): + num_messages: Incomplete + num_record_batches: Incomplete + num_dictionary_batches: Incomplete + num_dictionary_deltas: Incomplete + num_replaced_dictionaries: Incomplete + +class _RecordBatchWithMetadata(NamedTuple): + batch: Incomplete + custom_metadata: Incomplete + class DeviceAllocationType(IntEnum): CPU = ... CUDA = ... diff --git a/python/pyarrow-stubs/pyarrow/orc.pyi b/python/pyarrow-stubs/pyarrow/orc.pyi index 79a8751e51ca..fcf77076b182 100644 --- a/python/pyarrow-stubs/pyarrow/orc.pyi +++ b/python/pyarrow-stubs/pyarrow/orc.pyi @@ -15,6 +15,11 @@ # specific language governing permissions and limitations # under the License. +from io import BytesIO +from pathlib import Path +from pyarrow._fs import LocalFileSystem +from pyarrow.lib import BufferOutputStream, BufferReader, NativeFile, RecordBatch, Schema +from typing import List, Optional, Union import os import re import pyarrow @@ -27,7 +32,7 @@ from pyarrow.fs import _resolve_filesystem_and_path as _resolve_filesystem_and_p from pyarrow.lib import Table as Table _orc_writer_args_docs: str -def read_table(source: str | pyarrow.NativeFile | IO[Any], columns: list | None=None, filesystem: pyarrow.fs.FileSystem | None=None): +def read_table(source: str | pyarrow.NativeFile | IO[Any], columns: list | None=None, filesystem: pyarrow.fs.FileSystem | None=None) -> Table: ... def write_table(table: Table, where, *, file_version: str='0.12', batch_size: int=1024, stripe_size: int=..., compression: str='uncompressed', compression_block_size: int=65536, compression_strategy: str='speed', row_index_stride: int=10000, padding_tolerance: float=0.0, dictionary_key_size_threshold: float=0.0, bloom_filter_columns=None, bloom_filter_fpp: float=0.05) -> None: @@ -36,7 +41,7 @@ def write_table(table: Table, where, *, file_version: str='0.12', batch_size: in class ORCFile: reader: Incomplete - def __init__(self, source) -> None: + def __init__(self, source: Path | BufferReader | str | BytesIO | NativeFile) -> None: ... @property @@ -44,19 +49,19 @@ class ORCFile: ... @property - def schema(self): + def schema(self) -> Schema: ... @property - def nrows(self): + def nrows(self) -> int: ... @property - def nstripes(self): + def nstripes(self) -> int: ... @property - def file_version(self): + def file_version(self) -> str: ... @property @@ -64,11 +69,11 @@ class ORCFile: ... @property - def compression(self): + def compression(self) -> str: ... @property - def compression_size(self): + def compression_size(self) -> int: ... @property @@ -80,7 +85,7 @@ class ORCFile: ... @property - def row_index_stride(self): + def row_index_stride(self) -> int: ... @property @@ -107,7 +112,7 @@ class ORCFile: def file_length(self): ... - def _select_names(self, columns=None): + def _select_names(self, columns: list[int] | list[str] | None=None) -> list[str]: ... def read_stripe(self, n: int, columns: list | None=None) -> pyarrow.RecordBatch: diff --git a/python/pyarrow-stubs/pyarrow/pandas_compat.pyi b/python/pyarrow-stubs/pyarrow/pandas_compat.pyi index 0b31852eac86..b629a0496282 100644 --- a/python/pyarrow-stubs/pyarrow/pandas_compat.pyi +++ b/python/pyarrow-stubs/pyarrow/pandas_compat.pyi @@ -15,6 +15,9 @@ # specific language governing permissions and limitations # under the License. +from pyarrow.lib import BinaryArray, BooleanArray, ChunkedArray, DataType, Date32Array, Decimal128Array, Decimal128Type, DictionaryArray, DictionaryType, DoubleArray, DurationArray, DurationType, ExtensionArray, Field, FixedSizeBinaryType, FixedSizeListType, FloatArray, HalfFloatArray, Int16Array, Int32Array, Int64Array, Int8Array, LargeListType, LargeListViewType, LargeStringArray, ListArray, ListType, ListViewType, MapArray, MapType, NullArray, RunEndEncodedType, Schema, StringArray, StringViewArray, StructArray, StructType, Table, Time32Type, Time64Type, TimestampArray, TimestampType, UInt16Array, UInt32Array, UInt64Array, UInt8Array, UuidArray, UuidType +from pyarrow.tests.test_pandas import DummyExtensionType +from typing import Dict, List, Optional, Tuple, Type, Union import os import re import pyarrow @@ -25,95 +28,95 @@ _logical_type_map: Incomplete _numpy_logical_type_map: Incomplete _pandas_logical_type_map: Incomplete -def get_logical_type_map(): +def get_logical_type_map() -> dict[int, str]: ... -def get_logical_type(arrow_type): +def get_logical_type(arrow_type) -> str: ... def get_numpy_logical_type_map(): ... -def get_logical_type_from_numpy(pandas_collection): +def get_logical_type_from_numpy(pandas_collection) -> str: ... -def get_extension_dtype_info(column): +def get_extension_dtype_info(column) -> tuple[str, dict[str, str]] | tuple[str, dict[str, int | bool]] | tuple[str, None]: ... def get_column_metadata(column, name: str, arrow_type: pyarrow.DataType, field_name: str) -> dict: ... -def construct_metadata(columns_to_convert, df, column_names, index_levels, index_descriptors: list[dict], preserve_index: bool, types: list[pyarrow.DataType], column_field_names=None) -> dict: +def construct_metadata(columns_to_convert, df, column_names: list[Any | str], index_levels, index_descriptors: list[dict], preserve_index: bool, types: list[pyarrow.DataType], column_field_names: list[Any | str] | None=None) -> dict: ... -def _get_simple_index_descriptor(level, name): +def _get_simple_index_descriptor(level, name: str | None) -> dict[str, str | dict[str, str] | None] | dict[str, str | dict[str, str]] | dict[str, str | None]: ... def _column_name_to_strings(name: str | tuple) -> str | tuple: ... -def _index_level_name(index, i: int, column_names) -> str: +def _index_level_name(index, i: int, column_names: list[Any | str]) -> str: ... -def _get_columns_to_convert(df, schema, preserve_index, columns): +def _get_columns_to_convert(df, schema: Schema | None, preserve_index: bool | None, columns: list[str] | None): ... -def _get_columns_to_convert_given_schema(df, schema, preserve_index): +def _get_columns_to_convert_given_schema(df, schema: Schema, preserve_index: bool | None): ... -def _get_index_level(df, name): +def _get_index_level(df, name: str): ... -def _level_name(name): +def _level_name(name) -> int | str | None: ... -def _get_range_index_descriptor(level): +def _get_range_index_descriptor(level) -> dict[str, str | int] | dict[str, str | int | None]: ... def _get_index_level_values(index): ... -def _resolve_columns_of_interest(df, schema, columns): +def _resolve_columns_of_interest(df, schema: Schema | None, columns: list[str] | None): ... -def dataframe_to_types(df, preserve_index, columns=None): +def dataframe_to_types(df, preserve_index: bool | None, columns=None): ... -def dataframe_to_arrays(df, schema, preserve_index, nthreads: int=1, columns=None, safe: bool=True): +def dataframe_to_arrays(df, schema: Schema | None, preserve_index: bool | None, nthreads: int=1, columns: list[str] | None=None, safe: bool=True) -> tuple[list[MapArray | StringArray], Schema, None] | tuple[list[Int64Array | FloatArray | DoubleArray | BooleanArray | LargeStringArray], Schema, None] | tuple[list[LargeStringArray | Int64Array | TimestampArray], Schema, None] | tuple[list[DoubleArray], Schema, None] | tuple[list[DoubleArray | LargeStringArray], Schema, None] | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array | Int16Array | Int32Array | Int64Array | HalfFloatArray | FloatArray | DoubleArray | BooleanArray | TimestampArray | DurationArray | LargeStringArray | NullArray | ListArray], Schema, None] | tuple[list[TimestampArray | DictionaryArray | LargeStringArray | BooleanArray | DoubleArray | Int64Array | ExtensionArray], Schema, None] | tuple[list[Date32Array], Schema, None] | tuple[list[StringArray | Int64Array | DoubleArray], Schema, None] | tuple[list[UInt32Array], Schema, None] | tuple[list[Int16Array], Schema, None] | tuple[list[Int64Array | LargeStringArray], Schema, None] | tuple[list[Date32Array | Int64Array | DoubleArray | StringArray], Schema, None] | tuple[list[Any], Schema, None] | tuple[list[ExtensionArray], Schema, None] | tuple[list[ListArray], Schema, None] | tuple[list[StructArray], Schema, None] | tuple[list[UInt8Array | UInt32Array | Int16Array | Int32Array | HalfFloatArray | DoubleArray | TimestampArray | LargeStringArray | ListArray], Schema, None] | tuple[list[Int64Array | DoubleArray], Schema, None] | tuple[list[TimestampArray], Schema, None] | tuple[list[Int64Array | UInt32Array | DoubleArray | LargeStringArray | BooleanArray], Schema, None] | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array | Int8Array | Int16Array | Int32Array | Int64Array | FloatArray | DoubleArray | BooleanArray], Schema, None] | tuple[list[StructArray | LargeStringArray], Schema, None] | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array], Schema, None] | tuple[list[DurationArray], Schema, None] | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array | Int16Array | Int32Array | Int64Array | FloatArray | DoubleArray | BooleanArray | TimestampArray | DurationArray | LargeStringArray], Schema, None] | tuple[list[LargeStringArray | ListArray], Schema, None] | tuple[list[UInt64Array], Schema, None] | tuple[list[DoubleArray | ListArray | Int32Array], Schema, None] | tuple[list[LargeStringArray | Decimal128Array], Schema, None] | tuple[list[StringViewArray], Schema, None] | tuple[list[Int64Array | StructArray], Schema, None] | tuple[list[DoubleArray | LargeStringArray | Int64Array], Schema, None] | tuple[list[UInt8Array], Schema, None] | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array | Int8Array | Int16Array | Int32Array | Int64Array | FloatArray | DoubleArray | BooleanArray | LargeStringArray | NullArray], Schema, None] | tuple[list[UInt32Array | DoubleArray | BooleanArray | LargeStringArray], Schema, None] | tuple[list[FloatArray | DoubleArray], Schema, None] | tuple[list[Int16Array | FloatArray | StringArray], Schema, None] | tuple[list[UInt8Array | UInt16Array | Int64Array | UInt64Array | Int8Array | Int16Array | Int32Array | FloatArray | DoubleArray | BooleanArray | LargeStringArray | NullArray], Schema, None] | tuple[list[Int64Array | BooleanArray], Schema, None] | tuple[list[LargeStringArray | Int64Array | UInt8Array | DoubleArray | BooleanArray | DictionaryArray | TimestampArray], Schema, None] | tuple[list[Int64Array | ListArray | DoubleArray], Schema, None] | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array | Int16Array | Int32Array | Int64Array | FloatArray | DoubleArray | BooleanArray], Schema, None] | tuple[list[LargeStringArray], Schema, None] | tuple[list[NullArray], Schema, None] | tuple[list[DictionaryArray], Schema, None] | tuple[list[Int32Array | StringArray], Schema, None] | tuple[list[DictionaryArray | Int64Array | LargeStringArray | BinaryArray], Schema, None] | tuple[list[FloatArray | TimestampArray], Schema, None] | tuple[list[StringArray | Int64Array | Int32Array | TimestampArray], Schema, None] | tuple[list[BooleanArray], Schema, None] | tuple[list[LargeStringArray | Int64Array | DoubleArray | TimestampArray], Schema, None] | tuple[list[Int64Array | ListArray], Schema, None] | tuple[list[DictionaryArray | Int64Array], Schema, None] | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array | Int16Array | Int32Array | Int64Array | FloatArray | DoubleArray | BooleanArray | LargeStringArray], Schema, None] | tuple[list[StringArray], Schema, None] | tuple[list[Any], Schema, int] | tuple[list[Int8Array], Schema, None] | tuple[list[Decimal128Array], Schema, None] | tuple[list[NullArray | DoubleArray], Schema, None] | tuple[list[UInt16Array], Schema, None] | tuple[list[UuidArray], Schema, None] | tuple[list[BinaryArray], Schema, None] | tuple[list[LargeStringArray | DictionaryArray], Schema, None] | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array | Int16Array | Int32Array | Int64Array | HalfFloatArray | FloatArray | DoubleArray | BooleanArray | TimestampArray | DurationArray | LargeStringArray | NullArray | ListArray | DictionaryArray], Schema, None] | tuple[list[Int32Array], Schema, None] | tuple[list[DoubleArray | DictionaryArray], Schema, None] | tuple[list[StringArray | FloatArray], Schema, None] | tuple[list[Int8Array | Int16Array | Int32Array | Int64Array | UInt8Array | UInt16Array | UInt32Array | UInt64Array], Schema, None] | tuple[list[FloatArray], Schema, None] | tuple[list[Int64Array], Schema, None] | tuple[list[ChunkedArray | StringArray], Schema, None] | tuple[list[TimestampArray | Int64Array], Schema, None]: ... -def get_datetimetz_type(values, dtype, type_): +def get_datetimetz_type(values, dtype, type_: Decimal128Type | LargeListViewType | StructType | LargeListType | MapType | RunEndEncodedType | DictionaryType | ListType | TimestampType | FixedSizeListType | DataType | FixedSizeBinaryType | DurationType | Time64Type | Time32Type | ListViewType | None): ... -def _reconstruct_block(item: dict, columns=None, extension_columns: dict | None=None, return_block: bool=True): +def _reconstruct_block(item: dict, columns: list[str] | None=None, extension_columns: dict | None=None, return_block: bool=True): ... -def make_datetimetz(unit, tz): +def make_datetimetz(unit: str, tz: str): ... -def table_to_dataframe(options, table, categories=None, ignore_metadata: bool=False, types_mapper=None): +def table_to_dataframe(options: dict[str, bool | None], table: Table, categories=None, ignore_metadata: bool=False, types_mapper=None): ... _pandas_supported_numpy_types: Incomplete -def _get_extension_dtypes(table, columns_metadata, types_mapper, options, categories): +def _get_extension_dtypes(table: Table, columns_metadata: list[dict[str, str | dict[str, int]] | dict[str, str | None] | dict[str, str | dict[str, int | bool]] | Any | dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None]], types_mapper, options: dict[str, bool | None], categories): ... -def _check_data_column_metadata_consistency(all_columns) -> None: +def _check_data_column_metadata_consistency(all_columns: list[dict[str, str | dict[str, int]] | dict[str, str | None] | dict[str, str | dict[str, int | bool]] | Any | dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None]]) -> None: ... -def _deserialize_column_index(block_table, all_columns, column_indexes): +def _deserialize_column_index(block_table: Table, all_columns: list[dict[str, str | dict[str, int]] | dict[str, str | None] | dict[str, str | dict[str, int | bool]] | Any | dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None]], column_indexes: list[dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None] | dict[str, str | None] | Any]): ... -def _reconstruct_index(table, index_descriptors, all_columns, types_mapper=None): +def _reconstruct_index(table: Table, index_descriptors: list[Any | dict[str, str | int] | dict[str, str | int | None] | str], all_columns: list[dict[str, str | dict[str, int]] | dict[str, str | None] | dict[str, str | dict[str, int | bool]] | Any | dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None]], types_mapper=None): ... -def _extract_index_level(table, result_table, field_name, field_name_to_metadata, types_mapper=None): +def _extract_index_level(table: Table, result_table: Table, field_name: str, field_name_to_metadata: dict[str, dict[str, str | None] | dict[str, str | dict[str, int | bool]]] | dict[str, dict[str, str | None] | dict[str, str | dict[str, str]]] | dict[str, dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None]] | dict[str, dict[str, str | None]], types_mapper=None): ... def _backwards_compatible_index_name(raw_name: str, logical_name: str) -> str: ... -def _is_generated_index_name(name): +def _is_generated_index_name(name: str) -> bool: ... def get_pandas_logical_type_map(): @@ -122,11 +125,11 @@ def get_pandas_logical_type_map(): def _pandas_type_to_numpy_type(pandas_type: str): ... -def _reconstruct_columns_from_metadata(columns, column_indexes): +def _reconstruct_columns_from_metadata(columns, column_indexes: list[dict[str, str | dict[str, str]] | dict[str, str | None] | dict[str, str | dict[str, str] | None]]): ... -def _add_any_metadata(table, pandas_metadata): +def _add_any_metadata(table: Table, pandas_metadata: dict[str, list[str] | str | list[dict[str, str | None] | dict[str, str | dict[str, str]]]] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | None] | dict[str, str | dict[str, str]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | dict[str, int]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[Any] | list[dict[str, str | None] | dict[str, str | dict[str, int | bool]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[Any] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[str] | list[dict[str, str | None]] | str] | dict[str, list[str] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | None]] | list[dict[str, str | int]] | str] | dict[str, list[str] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | None]] | str] | dict[str, list[Any] | list[dict[str, str | dict[str, int | bool]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | dict[str, str]]] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[str] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | None] | dict[str, str | dict[str, int | bool]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[Any] | list[dict[str, str | None] | dict[str, str | dict[str, int]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[str] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[str] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | dict[str, str]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | None] | dict[str, str | dict[str, int | bool]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[Any] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | None] | dict[str, str | dict[str, str]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | dict[str, int | bool]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | None]] | list[Any] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[Any] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[Any] | list[dict[str, str | dict[str, str]]] | dict[Any, Any] | dict[str, str] | str]) -> Table: ... -def make_tz_aware(series, tz): +def make_tz_aware(series, tz: str): ... diff --git a/python/pyarrow-stubs/pyarrow/parquet/core.pyi b/python/pyarrow-stubs/pyarrow/parquet/core.pyi index 4289fbcec48f..c8a9d7f8f817 100644 --- a/python/pyarrow-stubs/pyarrow/parquet/core.pyi +++ b/python/pyarrow-stubs/pyarrow/parquet/core.pyi @@ -15,6 +15,14 @@ # specific language governing permissions and limitations # under the License. +from io import BufferedReader, BufferedWriter, BytesIO +from datetime import datetime +from pathlib import Path +from pyarrow._compute import Expression +from pyarrow._fs import LocalFileSystem as LocalFileSystem, SubTreeFileSystem +from pyarrow.lib import Buffer, BufferOutputStream, BufferReader, NativeFile, OSFile, PythonFile, RecordBatch, Schema, Table +from pyarrow.tests.util import FSProtocolClass +from typing import DefaultDict, Dict, List, Optional, Set, Tuple, Union import os import re import pyarrow @@ -41,7 +49,7 @@ def read_pandas(source: str | list[str] | pyarrow.NativeFile | IO[Any], columns: def write_table(table: pyarrow.Table, where: str | pyarrow.NativeFile, row_group_size: int | None=None, version: str='2.6', use_dictionary: bool=True, compression: str='snappy', write_statistics: bool=True, use_deprecated_int96_timestamps: bool | None=None, coerce_timestamps: str | None=None, allow_truncated_timestamps: bool=False, data_page_size: int | None=None, flavor: Literal['spark'] | None=None, filesystem: pyarrow.fs.FileSystem | None=None, compression_level: int | dict | None=None, use_byte_stream_split: bool=False, column_encoding: str | dict | None=None, data_page_version: str='1.0', use_compliant_nested_type: bool=True, encryption_properties: FileEncryptionProperties | None=None, write_batch_size: int | None=None, dictionary_pagesize_limit: int | None=None, store_schema: bool=True, write_page_index: bool=False, write_page_checksum: bool=False, sorting_columns: Sequence[SortingColumn] | None=None, store_decimal_as_integer: bool=False, write_time_adjusted_to_utc: bool=False, max_rows_per_page: int | None=None, bloom_filter_options: dict | None=None, use_content_defined_chunking: bool=False, **kwargs) -> None: ... -def write_to_dataset(table: pyarrow.Table, root_path, partition_cols: list | None=None, filesystem: pyarrow.fs.FileSystem | None=None, schema: pyarrow.Schema | None=None, partitioning: pyarrow.dataset.Partitioning | list[str] | None=None, basename_template: str | None=None, use_threads: bool | None=None, file_visitor: Callable[..., Any] | None=None, existing_data_behavior: Literal['overwrite_or_ignore'] | Literal['error'] | Literal['delete_matching'] | None=None, **kwargs) -> None: +def write_to_dataset(table: pyarrow.Table, root_path: Path | str, partition_cols: list | None=None, filesystem: pyarrow.fs.FileSystem | None=None, schema: pyarrow.Schema | None=None, partitioning: pyarrow.dataset.Partitioning | list[str] | None=None, basename_template: str | None=None, use_threads: bool | None=None, file_visitor: Callable[..., Any] | None=None, existing_data_behavior: Literal['overwrite_or_ignore'] | Literal['error'] | Literal['delete_matching'] | None=None, **kwargs) -> None: ... def write_metadata(schema: pyarrow.Schema, where: str | pyarrow.NativeFile, metadata_collector: list | None=None, filesystem: pyarrow.fs.FileSystem | None=None, **kwargs) -> None: @@ -69,23 +77,23 @@ class ParquetFile: def __exit__(self, *args, **kwargs) -> None: ... - def _build_nested_paths(self): + def _build_nested_paths(self) -> DefaultDict[str, list[int]] | DefaultDict[Any, Any]: ... @property - def metadata(self): + def metadata(self) -> FileMetaData: ... @property - def schema(self): + def schema(self) -> ParquetSchema: ... @property - def schema_arrow(self): + def schema_arrow(self) -> Schema: ... @property - def num_row_groups(self): + def num_row_groups(self) -> int: ... def close(self, force: bool=False): @@ -95,22 +103,22 @@ class ParquetFile: def closed(self) -> bool: ... - def read_row_group(self, i: int, columns: list | None=None, use_threads: bool=True, use_pandas_metadata: bool=False): + def read_row_group(self, i: int, columns: list | None=None, use_threads: bool=True, use_pandas_metadata: bool=False) -> Table: ... - def read_row_groups(self, row_groups: list, columns: list | None=None, use_threads: bool=True, use_pandas_metadata: bool=False): + def read_row_groups(self, row_groups: list, columns: list | None=None, use_threads: bool=True, use_pandas_metadata: bool=False) -> Table: ... def iter_batches(self, batch_size: int=65536, row_groups: list | None=None, columns: list | None=None, use_threads: bool=True, use_pandas_metadata: bool=False): ... - def read(self, columns: list | None=None, use_threads: bool=True, use_pandas_metadata: bool=False): + def read(self, columns: list | None=None, use_threads: bool=True, use_pandas_metadata: bool=False) -> Table: ... def scan_contents(self, columns: list[int] | None=None, batch_size: int=65536) -> int: ... - def _get_column_indices(self, column_names, use_pandas_metadata: bool=False): + def _get_column_indices(self, column_names, use_pandas_metadata: bool=False) -> list[int]: ... class ParquetWriter: @@ -124,7 +132,7 @@ class ParquetWriter: writer: Incomplete is_open: bool - def __init__(self, where, schema, filesystem=None, flavor=None, version: str='2.6', use_dictionary: bool=True, compression: str='snappy', write_statistics: bool=True, use_deprecated_int96_timestamps=None, compression_level=None, use_byte_stream_split: bool=False, column_encoding=None, writer_engine_version=None, data_page_version: str='1.0', use_compliant_nested_type: bool=True, encryption_properties=None, write_batch_size=None, dictionary_pagesize_limit=None, store_schema: bool=True, write_page_index: bool=False, write_page_checksum: bool=False, sorting_columns=None, store_decimal_as_integer: bool=False, write_time_adjusted_to_utc: bool=False, max_rows_per_page=None, bloom_filter_options=None, use_content_defined_chunking: bool=False, **options) -> None: + def __init__(self, where, schema: Schema | None, filesystem: LocalFileSystem | None=None, flavor: str | None=None, version: str='2.6', use_dictionary: bool=True, compression: str='snappy', write_statistics: bool=True, use_deprecated_int96_timestamps: bool | None=None, compression_level: int | dict[str, int] | None=None, use_byte_stream_split: bool=False, column_encoding: dict[str, str] | bool | str | None=None, writer_engine_version=None, data_page_version: str='1.0', use_compliant_nested_type: bool=True, encryption_properties: FileEncryptionProperties | None=None, write_batch_size: int | None=None, dictionary_pagesize_limit: int | str | None=None, store_schema: bool=True, write_page_index: bool=False, write_page_checksum: bool=False, sorting_columns: tuple[SortingColumn, SortingColumn] | None=None, store_decimal_as_integer: bool=False, write_time_adjusted_to_utc: bool=False, max_rows_per_page: int | None=None, bloom_filter_options: bool | dict[str, dict[str, str | int]] | dict[str, dict[Any, Any]] | dict[str, dict[str, float]] | dict[str, dict[str, str | float]] | dict[str, dict[str, int]] | dict[str, bool] | dict[str, dict[str, int | float]] | None=None, use_content_defined_chunking: bool=False, **options) -> None: ... def __del__(self) -> None: @@ -136,7 +144,7 @@ class ParquetWriter: def __exit__(self, *args, **kwargs): ... - def write(self, table_or_batch, row_group_size: int | None=None) -> None: + def write(self, table_or_batch: Table | RecordBatch, row_group_size: int | None=None) -> None: ... def write_batch(self, batch: pyarrow.RecordBatch, row_group_size: int | None=None) -> None: @@ -157,26 +165,26 @@ class ParquetDataset: _base_dir: Incomplete _dataset: Incomplete - def __init__(self, path_or_paths, filesystem=None, schema=None, *, filters=None, read_dictionary=None, binary_type=None, list_type=None, memory_map: bool=False, buffer_size=None, partitioning: str='hive', ignore_prefixes=None, pre_buffer: bool=True, coerce_int96_timestamp_unit=None, decryption_properties=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, page_checksum_verification: bool=False, arrow_extensions_enabled: bool=True) -> None: + def __init__(self, path_or_paths, filesystem: SubTreeFileSystem | LocalFileSystem | str | None=None, schema: Schema | None=None, *, filters=None, read_dictionary=None, binary_type=None, list_type=None, memory_map: bool=False, buffer_size=None, partitioning: str='hive', ignore_prefixes=None, pre_buffer: bool=True, coerce_int96_timestamp_unit=None, decryption_properties=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, page_checksum_verification: bool=False, arrow_extensions_enabled: bool=True) -> None: ... - def equals(self, other): + def equals(self, other) -> bool: ... - def __eq__(self, other): + def __eq__(self, other) -> bool: ... @property - def schema(self): + def schema(self) -> Schema: ... def read(self, columns: list[str] | None=None, use_threads: bool=True, use_pandas_metadata: bool=False) -> pyarrow.Table: ... - def _get_common_pandas_metadata(self): + def _get_common_pandas_metadata(self) -> dict[bytes, bytes]: ... - def read_pandas(self, **kwargs): + def read_pandas(self, **kwargs) -> Table: ... @property @@ -184,11 +192,11 @@ class ParquetDataset: ... @property - def files(self): + def files(self) -> list[str]: ... @property - def filesystem(self): + def filesystem(self) -> LocalFileSystem: ... @property diff --git a/python/pyarrow-stubs/pyarrow/types.pyi b/python/pyarrow-stubs/pyarrow/types.pyi index 5e140bbc9722..28de0bc22f86 100644 --- a/python/pyarrow-stubs/pyarrow/types.pyi +++ b/python/pyarrow-stubs/pyarrow/types.pyi @@ -15,6 +15,10 @@ # specific language governing permissions and limitations # under the License. +from pyarrow.lib import DataType, Decimal128Type, Decimal256Type, Decimal32Type, Decimal64Type, DenseUnionType, DictionaryType, DurationType, FixedSizeBinaryType, FixedSizeListType, LargeListType, LargeListViewType, ListType, ListViewType, MapType, RunEndEncodedType, SparseUnionType, StructType, Time32Type, Time64Type, TimestampType, UuidType +from pyarrow.tests.test_extension_type import PeriodType, PeriodTypeWithClass, PeriodTypeWithToPandasDtype +from pyarrow.tests.test_pandas import DummyExtensionType, MyCustomIntegerType +from typing import Union import os import re import pyarrow @@ -34,163 +38,163 @@ _INTERVAL_TYPES: Incomplete _TEMPORAL_TYPES: Incomplete _UNION_TYPES: Incomplete -def is_null(t: pyarrow.DataType): +def is_null(t: pyarrow.DataType) -> bool: ... -def is_boolean(t: pyarrow.DataType): +def is_boolean(t: pyarrow.DataType) -> bool: ... -def is_integer(t: pyarrow.DataType): +def is_integer(t: pyarrow.DataType) -> bool: ... -def is_signed_integer(t: pyarrow.DataType): +def is_signed_integer(t: pyarrow.DataType) -> bool: ... -def is_unsigned_integer(t: pyarrow.DataType): +def is_unsigned_integer(t: pyarrow.DataType) -> bool: ... -def is_int8(t: pyarrow.DataType): +def is_int8(t: pyarrow.DataType) -> bool: ... -def is_int16(t: pyarrow.DataType): +def is_int16(t: pyarrow.DataType) -> bool: ... -def is_int32(t: pyarrow.DataType): +def is_int32(t: pyarrow.DataType) -> bool: ... -def is_int64(t: pyarrow.DataType): +def is_int64(t: pyarrow.DataType) -> bool: ... -def is_uint8(t: pyarrow.DataType): +def is_uint8(t: pyarrow.DataType) -> bool: ... -def is_uint16(t: pyarrow.DataType): +def is_uint16(t: pyarrow.DataType) -> bool: ... -def is_uint32(t: pyarrow.DataType): +def is_uint32(t: pyarrow.DataType) -> bool: ... -def is_uint64(t: pyarrow.DataType): +def is_uint64(t: pyarrow.DataType) -> bool: ... -def is_floating(t: pyarrow.DataType): +def is_floating(t: pyarrow.DataType) -> bool: ... -def is_float16(t: pyarrow.DataType): +def is_float16(t: pyarrow.DataType) -> bool: ... -def is_float32(t: pyarrow.DataType): +def is_float32(t: pyarrow.DataType) -> bool: ... -def is_float64(t: pyarrow.DataType): +def is_float64(t: pyarrow.DataType) -> bool: ... -def is_list(t: pyarrow.DataType): +def is_list(t: pyarrow.DataType) -> bool: ... -def is_large_list(t: pyarrow.DataType): +def is_large_list(t: pyarrow.DataType) -> bool: ... -def is_fixed_size_list(t: pyarrow.DataType): +def is_fixed_size_list(t: pyarrow.DataType) -> bool: ... -def is_list_view(t: pyarrow.DataType): +def is_list_view(t: pyarrow.DataType) -> bool: ... -def is_large_list_view(t: pyarrow.DataType): +def is_large_list_view(t: pyarrow.DataType) -> bool: ... -def is_struct(t: pyarrow.DataType): +def is_struct(t: pyarrow.DataType) -> bool: ... -def is_union(t: pyarrow.DataType): +def is_union(t: pyarrow.DataType) -> bool: ... -def is_nested(t: pyarrow.DataType): +def is_nested(t: pyarrow.DataType) -> bool: ... -def is_run_end_encoded(t: pyarrow.DataType): +def is_run_end_encoded(t: pyarrow.DataType) -> bool: ... -def is_temporal(t: pyarrow.DataType): +def is_temporal(t: pyarrow.DataType) -> bool: ... -def is_timestamp(t: pyarrow.DataType): +def is_timestamp(t: pyarrow.DataType) -> bool: ... -def is_duration(t: pyarrow.DataType): +def is_duration(t: pyarrow.DataType) -> bool: ... -def is_time(t: pyarrow.DataType): +def is_time(t: pyarrow.DataType) -> bool: ... -def is_time32(t: pyarrow.DataType): +def is_time32(t: pyarrow.DataType) -> bool: ... -def is_time64(t: pyarrow.DataType): +def is_time64(t: pyarrow.DataType) -> bool: ... -def is_binary(t: pyarrow.DataType): +def is_binary(t: pyarrow.DataType) -> bool: ... -def is_large_binary(t: pyarrow.DataType): +def is_large_binary(t: pyarrow.DataType) -> bool: ... -def is_unicode(t: pyarrow.DataType): +def is_unicode(t: pyarrow.DataType) -> bool: ... -def is_string(t: pyarrow.DataType): +def is_string(t: pyarrow.DataType) -> bool: ... def is_large_unicode(t: pyarrow.DataType): ... -def is_large_string(t: pyarrow.DataType): +def is_large_string(t: pyarrow.DataType) -> bool: ... -def is_fixed_size_binary(t: pyarrow.DataType): +def is_fixed_size_binary(t: pyarrow.DataType) -> bool: ... -def is_binary_view(t: pyarrow.DataType): +def is_binary_view(t: pyarrow.DataType) -> bool: ... -def is_string_view(t: pyarrow.DataType): +def is_string_view(t: pyarrow.DataType) -> bool: ... -def is_date(t: pyarrow.DataType): +def is_date(t: pyarrow.DataType) -> bool: ... -def is_date32(t: pyarrow.DataType): +def is_date32(t: pyarrow.DataType) -> bool: ... -def is_date64(t: pyarrow.DataType): +def is_date64(t: pyarrow.DataType) -> bool: ... -def is_map(t: pyarrow.DataType): +def is_map(t: pyarrow.DataType) -> bool: ... -def is_decimal(t: pyarrow.DataType): +def is_decimal(t: pyarrow.DataType) -> bool: ... -def is_decimal32(t: pyarrow.DataType): +def is_decimal32(t: pyarrow.DataType) -> bool: ... -def is_decimal64(t: pyarrow.DataType): +def is_decimal64(t: pyarrow.DataType) -> bool: ... -def is_decimal128(t: pyarrow.DataType): +def is_decimal128(t: pyarrow.DataType) -> bool: ... -def is_decimal256(t: pyarrow.DataType): +def is_decimal256(t: pyarrow.DataType) -> bool: ... -def is_dictionary(t: pyarrow.DataType): +def is_dictionary(t: pyarrow.DataType) -> bool: ... -def is_interval(t: pyarrow.DataType): +def is_interval(t: pyarrow.DataType) -> bool: ... -def is_primitive(t: pyarrow.DataType): +def is_primitive(t: pyarrow.DataType) -> bool: ... class TypesEnum(IntEnum): diff --git a/python/pyarrow-stubs/pyarrow/util.pyi b/python/pyarrow-stubs/pyarrow/util.pyi index 1ecd4fa95454..7134014f25b3 100644 --- a/python/pyarrow-stubs/pyarrow/util.pyi +++ b/python/pyarrow-stubs/pyarrow/util.pyi @@ -15,6 +15,12 @@ # specific language governing permissions and limitations # under the License. +from io import BufferedReader, BufferedWriter, BytesIO, StringIO +from pathlib import Path +from pyarrow._dataset import FileSystemDataset, FileSystemDatasetFactory, InMemoryDataset, Scanner +from pyarrow.lib import Buffer, BufferOutputStream, BufferReader, BufferedInputStream, BufferedOutputStream, CompressedInputStream, CompressedOutputStream, MockOutputStream, NativeFile, OSFile, PythonFile, RecordBatch, RecordBatchReader, Table +from pyarrow.tests.util import FSProtocolClass +from typing import List, Optional, Tuple, Type, Union import os import re import pyarrow @@ -22,22 +28,22 @@ from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, Name from _typeshed import Incomplete _DEPR_MSG: str -def doc(*docstrings, **params): +def doc(*docstrings, **params) -> Callable: ... -def _deprecate_api(old_name, new_name, api, next_version, type=...): +def _deprecate_api(old_name: str, new_name: str, api: Callable, next_version: str, type=...) -> Callable: ... def _deprecate_class(old_name, new_class, next_version, instancecheck: bool=True): ... -def _is_iterable(obj): +def _is_iterable(obj: FileSystemDataset | Iterator[Any] | InMemoryDataset | Scanner): ... -def _is_path_like(path): +def _is_path_like(path) -> bool: ... -def _stringify_path(path): +def _stringify_path(path) -> str: ... def product(seq): @@ -46,10 +52,10 @@ def product(seq): def get_contiguous_span(shape: tuple, strides: tuple, itemsize: int) -> int: ... -def find_free_port(): +def find_free_port() -> int: ... -def guid(): +def guid() -> str: ... def _break_traceback_cycle_from_frame(frame) -> None: From 9965f8a10a3ccd0c6334a899f681ad6740e836d2 Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Fri, 9 Oct 2026 18:20:52 +0100 Subject: [PATCH 8/8] [Python] Fix and format the stubs with ruff ruff check --fix --unsafe-fixes and ruff format as configured in pyproject.toml: unused and duplicate imports dropped, typing spellings updated, the flake8-pyi conventions applied, and the typeshed layout. --- python/pyarrow-stubs/pyarrow/__init__.pyi | 361 +- python/pyarrow-stubs/pyarrow/_acero.pyi | 110 +- python/pyarrow-stubs/pyarrow/_azurefs.pyi | 30 +- python/pyarrow-stubs/pyarrow/_compute.pyi | 983 ++- .../pyarrow/_compute_docstrings.pyi | 5 +- python/pyarrow-stubs/pyarrow/_csv.pyi | 509 +- python/pyarrow-stubs/pyarrow/_cuda.pyi | 29 +- python/pyarrow-stubs/pyarrow/_dataset.pyi | 953 ++- python/pyarrow-stubs/pyarrow/_dataset_orc.pyi | 21 +- .../pyarrow/_dataset_parquet.pyi | 353 +- .../pyarrow/_dataset_parquet_encryption.pyi | 37 +- python/pyarrow-stubs/pyarrow/_feather.pyi | 37 +- python/pyarrow-stubs/pyarrow/_flight.pyi | 852 +-- python/pyarrow-stubs/pyarrow/_fs.pyi | 319 +- python/pyarrow-stubs/pyarrow/_gcsfs.pyi | 42 +- python/pyarrow-stubs/pyarrow/_hdfs.pyi | 35 +- python/pyarrow-stubs/pyarrow/_json.pyi | 115 +- python/pyarrow-stubs/pyarrow/_orc.pyi | 119 +- python/pyarrow-stubs/pyarrow/_parquet.pyi | 713 +-- .../pyarrow/_parquet_encryption.pyi | 281 +- .../pyarrow/_pyarrow_cpp_tests.pyi | 22 +- python/pyarrow-stubs/pyarrow/_s3fs.pyi | 79 +- python/pyarrow-stubs/pyarrow/_substrait.pyi | 68 +- python/pyarrow-stubs/pyarrow/acero.pyi | 95 +- python/pyarrow-stubs/pyarrow/benchmark.pyi | 5 - python/pyarrow-stubs/pyarrow/cffi.pyi | 5 +- python/pyarrow-stubs/pyarrow/compute.pyi | 2242 ++++--- python/pyarrow-stubs/pyarrow/csv.pyi | 19 +- python/pyarrow-stubs/pyarrow/cuda.pyi | 13 +- python/pyarrow-stubs/pyarrow/dataset.pyi | 260 +- python/pyarrow-stubs/pyarrow/feather.pyi | 74 +- python/pyarrow-stubs/pyarrow/flight.pyi | 53 +- python/pyarrow-stubs/pyarrow/fs.pyi | 176 +- python/pyarrow-stubs/pyarrow/gandiva.pyi | 29 +- .../pyarrow/interchange/__init__.pyi | 5 - .../pyarrow/interchange/buffer.pyi | 26 +- .../pyarrow/interchange/column.pyi | 78 +- .../pyarrow/interchange/dataframe.pyi | 68 +- .../pyarrow/interchange/from_dataframe.pyi | 93 +- python/pyarrow-stubs/pyarrow/ipc.pyi | 144 +- python/pyarrow-stubs/pyarrow/json.pyi | 12 +- python/pyarrow-stubs/pyarrow/jvm.pyi | 49 +- python/pyarrow-stubs/pyarrow/lib.pyi | 5637 ++++++----------- python/pyarrow-stubs/pyarrow/orc.pyi | 163 +- .../pyarrow-stubs/pyarrow/pandas_compat.pyi | 802 ++- .../pyarrow/parquet/__init__.pyi | 5 - python/pyarrow-stubs/pyarrow/parquet/core.pyi | 440 +- .../pyarrow/parquet/encryption.pyi | 15 +- python/pyarrow-stubs/pyarrow/substrait.pyi | 16 +- python/pyarrow-stubs/pyarrow/types.pyi | 225 +- python/pyarrow-stubs/pyarrow/util.pyi | 74 +- 51 files changed, 7623 insertions(+), 9273 deletions(-) diff --git a/python/pyarrow-stubs/pyarrow/__init__.pyi b/python/pyarrow-stubs/pyarrow/__init__.pyi index 843d3dc07db4..6dca7576eb02 100644 --- a/python/pyarrow-stubs/pyarrow/__init__.pyi +++ b/python/pyarrow-stubs/pyarrow/__init__.pyi @@ -15,51 +15,318 @@ # specific language governing permissions and limitations # under the License. -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete -from pyarrow.ipc import Message as Message, MessageReader as MessageReader, MetadataVersion as MetadataVersion, RecordBatchFileReader as RecordBatchFileReader, RecordBatchFileWriter as RecordBatchFileWriter, RecordBatchStreamReader as RecordBatchStreamReader, RecordBatchStreamWriter as RecordBatchStreamWriter, deserialize_pandas as deserialize_pandas, serialize_pandas as serialize_pandas -from pyarrow.lib import Array as Array, ArrowCancelled as ArrowCancelled, ArrowCapacityError as ArrowCapacityError, ArrowException as ArrowException, ArrowIOError as ArrowIOError, ArrowIndexError as ArrowIndexError, ArrowInvalid as ArrowInvalid, ArrowKeyError as ArrowKeyError, ArrowMemoryError as ArrowMemoryError, ArrowNotImplementedError as ArrowNotImplementedError, ArrowSerializationError as ArrowSerializationError, ArrowTypeError as ArrowTypeError, BaseExtensionType as BaseExtensionType, BinaryArray as BinaryArray, BinaryScalar as BinaryScalar, BinaryViewArray as BinaryViewArray, BinaryViewScalar as BinaryViewScalar, Bool8Array as Bool8Array, Bool8Scalar as Bool8Scalar, Bool8Type as Bool8Type, BooleanArray as BooleanArray, BooleanScalar as BooleanScalar, Buffer as Buffer, BufferOutputStream as BufferOutputStream, BufferReader as BufferReader, BufferedInputStream as BufferedInputStream, BufferedOutputStream as BufferedOutputStream, BuildInfo as BuildInfo, CacheOptions as CacheOptions, ChunkedArray as ChunkedArray, Codec as Codec, CompressedInputStream as CompressedInputStream, CompressedOutputStream as CompressedOutputStream, CppBuildInfo as CppBuildInfo, DataType as DataType, Date32Array as Date32Array, Date32Scalar as Date32Scalar, Date64Array as Date64Array, Date64Scalar as Date64Scalar, Decimal128Array as Decimal128Array, Decimal128Scalar as Decimal128Scalar, Decimal128Type as Decimal128Type, Decimal256Array as Decimal256Array, Decimal256Scalar as Decimal256Scalar, Decimal256Type as Decimal256Type, Decimal32Array as Decimal32Array, Decimal32Scalar as Decimal32Scalar, Decimal32Type as Decimal32Type, Decimal64Array as Decimal64Array, Decimal64Scalar as Decimal64Scalar, Decimal64Type as Decimal64Type, DenseUnionType as DenseUnionType, Device as Device, DeviceAllocationType as DeviceAllocationType, DictionaryArray as DictionaryArray, DictionaryMemo as DictionaryMemo, DictionaryScalar as DictionaryScalar, DictionaryType as DictionaryType, DoubleArray as DoubleArray, DoubleScalar as DoubleScalar, DurationArray as DurationArray, DurationScalar as DurationScalar, DurationType as DurationType, ExtensionArray as ExtensionArray, ExtensionScalar as ExtensionScalar, ExtensionType as ExtensionType, Field as Field, FixedShapeTensorArray as FixedShapeTensorArray, FixedShapeTensorScalar as FixedShapeTensorScalar, FixedShapeTensorType as FixedShapeTensorType, FixedSizeBinaryArray as FixedSizeBinaryArray, FixedSizeBinaryScalar as FixedSizeBinaryScalar, FixedSizeBinaryType as FixedSizeBinaryType, FixedSizeBufferWriter as FixedSizeBufferWriter, FixedSizeListArray as FixedSizeListArray, FixedSizeListScalar as FixedSizeListScalar, FixedSizeListType as FixedSizeListType, FloatArray as FloatArray, FloatScalar as FloatScalar, FloatingPointArray as FloatingPointArray, HalfFloatArray as HalfFloatArray, HalfFloatScalar as HalfFloatScalar, Int16Array as Int16Array, Int16Scalar as Int16Scalar, Int32Array as Int32Array, Int32Scalar as Int32Scalar, Int64Array as Int64Array, Int64Scalar as Int64Scalar, Int8Array as Int8Array, Int8Scalar as Int8Scalar, IntegerArray as IntegerArray, JsonArray as JsonArray, JsonScalar as JsonScalar, JsonType as JsonType, KeyValueMetadata as KeyValueMetadata, LargeBinaryArray as LargeBinaryArray, LargeBinaryScalar as LargeBinaryScalar, LargeListArray as LargeListArray, LargeListScalar as LargeListScalar, LargeListType as LargeListType, LargeListViewArray as LargeListViewArray, LargeListViewScalar as LargeListViewScalar, LargeListViewType as LargeListViewType, LargeStringArray as LargeStringArray, LargeStringScalar as LargeStringScalar, ListArray as ListArray, ListScalar as ListScalar, ListType as ListType, ListViewArray as ListViewArray, ListViewScalar as ListViewScalar, ListViewType as ListViewType, LoggingMemoryPool as LoggingMemoryPool, MapArray as MapArray, MapScalar as MapScalar, MapType as MapType, MemoryManager as MemoryManager, MemoryMappedFile as MemoryMappedFile, MemoryPool as MemoryPool, MockOutputStream as MockOutputStream, MonthDayNano as MonthDayNano, MonthDayNanoIntervalArray as MonthDayNanoIntervalArray, MonthDayNanoIntervalScalar as MonthDayNanoIntervalScalar, NA as NA, NativeFile as NativeFile, NullArray as NullArray, NullScalar as NullScalar, NumericArray as NumericArray, OSFile as OSFile, OpaqueArray as OpaqueArray, OpaqueScalar as OpaqueScalar, OpaqueType as OpaqueType, ProxyMemoryPool as ProxyMemoryPool, PythonFile as PythonFile, RecordBatch as RecordBatch, RecordBatchReader as RecordBatchReader, ResizableBuffer as ResizableBuffer, RunEndEncodedArray as RunEndEncodedArray, RunEndEncodedScalar as RunEndEncodedScalar, RunEndEncodedType as RunEndEncodedType, RuntimeInfo as RuntimeInfo, Scalar as Scalar, Schema as Schema, SparseCOOTensor as SparseCOOTensor, SparseCSCMatrix as SparseCSCMatrix, SparseCSFTensor as SparseCSFTensor, SparseCSRMatrix as SparseCSRMatrix, SparseUnionType as SparseUnionType, StringArray as StringArray, StringScalar as StringScalar, StringViewArray as StringViewArray, StringViewScalar as StringViewScalar, StructArray as StructArray, StructScalar as StructScalar, StructType as StructType, Table as Table, TableGroupBy as TableGroupBy, Tensor as Tensor, Time32Array as Time32Array, Time32Scalar as Time32Scalar, Time32Type as Time32Type, Time64Array as Time64Array, Time64Scalar as Time64Scalar, Time64Type as Time64Type, TimestampArray as TimestampArray, TimestampScalar as TimestampScalar, TimestampType as TimestampType, TransformInputStream as TransformInputStream, UInt16Array as UInt16Array, UInt16Scalar as UInt16Scalar, UInt32Array as UInt32Array, UInt32Scalar as UInt32Scalar, UInt64Array as UInt64Array, UInt64Scalar as UInt64Scalar, UInt8Array as UInt8Array, UInt8Scalar as UInt8Scalar, UnionArray as UnionArray, UnionScalar as UnionScalar, UnionType as UnionType, UnknownExtensionType as UnknownExtensionType, UuidArray as UuidArray, UuidScalar as UuidScalar, UuidType as UuidType, VersionInfo as VersionInfo, allocate_buffer as allocate_buffer, arange as arange, array as array, binary as binary, binary_view as binary_view, bool8 as bool8, bool_ as bool_, build_info as build_info, chunked_array as chunked_array, compress as compress, concat_arrays as concat_arrays, concat_batches as concat_batches, concat_tables as concat_tables, cpp_build_info as cpp_build_info, cpp_version as cpp_version, cpp_version_info as cpp_version_info, cpu_count as cpu_count, create_memory_map as create_memory_map, date32 as date32, date64 as date64, decimal128 as decimal128, decimal256 as decimal256, decimal32 as decimal32, decimal64 as decimal64, decompress as decompress, default_cpu_memory_manager as default_cpu_memory_manager, default_memory_pool as default_memory_pool, dense_union as dense_union, dictionary as dictionary, duration as duration, enable_signal_handlers as enable_signal_handlers, field as field, fixed_shape_tensor as fixed_shape_tensor, float16 as float16, float32 as float32, float64 as float64, foreign_buffer as foreign_buffer, from_numpy_dtype as from_numpy_dtype, infer_type as infer_type, input_stream as input_stream, int16 as int16, int32 as int32, int64 as int64, int8 as int8, io_thread_count as io_thread_count, is_opentelemetry_enabled as is_opentelemetry_enabled, jemalloc_memory_pool as jemalloc_memory_pool, jemalloc_set_decay_ms as jemalloc_set_decay_ms, json_ as json_, large_binary as large_binary, large_list as large_list, large_list_view as large_list_view, large_string as large_string, large_utf8 as large_utf8, list_ as list_, list_view as list_view, log_memory_allocations as log_memory_allocations, logging_memory_pool as logging_memory_pool, map_ as map_, memory_map as memory_map, mimalloc_memory_pool as mimalloc_memory_pool, month_day_nano_interval as month_day_nano_interval, null as null, nulls as nulls, opaque as opaque, output_stream as output_stream, proxy_memory_pool as proxy_memory_pool, py_buffer as py_buffer, record_batch as record_batch, register_extension_type as register_extension_type, repeat as repeat, run_end_encoded as run_end_encoded, runtime_info as runtime_info, scalar as scalar, schema as schema, set_cpu_count as set_cpu_count, set_io_thread_count as set_io_thread_count, set_memory_pool as set_memory_pool, set_timezone_db_path as set_timezone_db_path, sparse_union as sparse_union, string as string, string_view as string_view, struct as struct, supported_memory_backends as supported_memory_backends, system_memory_pool as system_memory_pool, table as table, time32 as time32, time64 as time64, timestamp as timestamp, total_allocated_bytes as total_allocated_bytes, transcoding_input_stream as transcoding_input_stream, type_for_alias as type_for_alias, uint16 as uint16, uint32 as uint32, uint64 as uint64, uint8 as uint8, unify_schemas as unify_schemas, union as union, unregister_extension_type as unregister_extension_type, utf8 as utf8, uuid as uuid -from pyarrow.util import _deprecate_api as _deprecate_api, _deprecate_class as _deprecate_class -from pyarrow.lib import _NULL as NULL +from pyarrow.ipc import ( + Message as Message, + MessageReader as MessageReader, + MetadataVersion as MetadataVersion, + RecordBatchFileReader as RecordBatchFileReader, + RecordBatchFileWriter as RecordBatchFileWriter, + RecordBatchStreamReader as RecordBatchStreamReader, + RecordBatchStreamWriter as RecordBatchStreamWriter, + deserialize_pandas as deserialize_pandas, + serialize_pandas as serialize_pandas, +) +from pyarrow.lib import ( + Array as Array, + ArrowCancelled as ArrowCancelled, + ArrowCapacityError as ArrowCapacityError, + ArrowException as ArrowException, + ArrowIOError as ArrowIOError, + ArrowIndexError as ArrowIndexError, + ArrowInvalid as ArrowInvalid, + ArrowKeyError as ArrowKeyError, + ArrowMemoryError as ArrowMemoryError, + ArrowNotImplementedError as ArrowNotImplementedError, + ArrowSerializationError as ArrowSerializationError, + ArrowTypeError as ArrowTypeError, + BaseExtensionType as BaseExtensionType, + BinaryArray as BinaryArray, + BinaryScalar as BinaryScalar, + BinaryViewArray as BinaryViewArray, + BinaryViewScalar as BinaryViewScalar, + Bool8Array as Bool8Array, + Bool8Scalar as Bool8Scalar, + Bool8Type as Bool8Type, + BooleanArray as BooleanArray, + BooleanScalar as BooleanScalar, + Buffer as Buffer, + BufferOutputStream as BufferOutputStream, + BufferReader as BufferReader, + BufferedInputStream as BufferedInputStream, + BufferedOutputStream as BufferedOutputStream, + BuildInfo as BuildInfo, + CacheOptions as CacheOptions, + ChunkedArray as ChunkedArray, + Codec as Codec, + CompressedInputStream as CompressedInputStream, + CompressedOutputStream as CompressedOutputStream, + CppBuildInfo as CppBuildInfo, + DataType as DataType, + Date32Array as Date32Array, + Date32Scalar as Date32Scalar, + Date64Array as Date64Array, + Date64Scalar as Date64Scalar, + Decimal128Array as Decimal128Array, + Decimal128Scalar as Decimal128Scalar, + Decimal128Type as Decimal128Type, + Decimal256Array as Decimal256Array, + Decimal256Scalar as Decimal256Scalar, + Decimal256Type as Decimal256Type, + Decimal32Array as Decimal32Array, + Decimal32Scalar as Decimal32Scalar, + Decimal32Type as Decimal32Type, + Decimal64Array as Decimal64Array, + Decimal64Scalar as Decimal64Scalar, + Decimal64Type as Decimal64Type, + DenseUnionType as DenseUnionType, + Device as Device, + DeviceAllocationType as DeviceAllocationType, + DictionaryArray as DictionaryArray, + DictionaryMemo as DictionaryMemo, + DictionaryScalar as DictionaryScalar, + DictionaryType as DictionaryType, + DoubleArray as DoubleArray, + DoubleScalar as DoubleScalar, + DurationArray as DurationArray, + DurationScalar as DurationScalar, + DurationType as DurationType, + ExtensionArray as ExtensionArray, + ExtensionScalar as ExtensionScalar, + ExtensionType as ExtensionType, + Field as Field, + FixedShapeTensorArray as FixedShapeTensorArray, + FixedShapeTensorScalar as FixedShapeTensorScalar, + FixedShapeTensorType as FixedShapeTensorType, + FixedSizeBinaryArray as FixedSizeBinaryArray, + FixedSizeBinaryScalar as FixedSizeBinaryScalar, + FixedSizeBinaryType as FixedSizeBinaryType, + FixedSizeBufferWriter as FixedSizeBufferWriter, + FixedSizeListArray as FixedSizeListArray, + FixedSizeListScalar as FixedSizeListScalar, + FixedSizeListType as FixedSizeListType, + FloatArray as FloatArray, + FloatScalar as FloatScalar, + FloatingPointArray as FloatingPointArray, + HalfFloatArray as HalfFloatArray, + HalfFloatScalar as HalfFloatScalar, + Int16Array as Int16Array, + Int16Scalar as Int16Scalar, + Int32Array as Int32Array, + Int32Scalar as Int32Scalar, + Int64Array as Int64Array, + Int64Scalar as Int64Scalar, + Int8Array as Int8Array, + Int8Scalar as Int8Scalar, + IntegerArray as IntegerArray, + JsonArray as JsonArray, + JsonScalar as JsonScalar, + JsonType as JsonType, + KeyValueMetadata as KeyValueMetadata, + LargeBinaryArray as LargeBinaryArray, + LargeBinaryScalar as LargeBinaryScalar, + LargeListArray as LargeListArray, + LargeListScalar as LargeListScalar, + LargeListType as LargeListType, + LargeListViewArray as LargeListViewArray, + LargeListViewScalar as LargeListViewScalar, + LargeListViewType as LargeListViewType, + LargeStringArray as LargeStringArray, + LargeStringScalar as LargeStringScalar, + ListArray as ListArray, + ListScalar as ListScalar, + ListType as ListType, + ListViewArray as ListViewArray, + ListViewScalar as ListViewScalar, + ListViewType as ListViewType, + LoggingMemoryPool as LoggingMemoryPool, + MapArray as MapArray, + MapScalar as MapScalar, + MapType as MapType, + MemoryManager as MemoryManager, + MemoryMappedFile as MemoryMappedFile, + MemoryPool as MemoryPool, + MockOutputStream as MockOutputStream, + MonthDayNano as MonthDayNano, + MonthDayNanoIntervalArray as MonthDayNanoIntervalArray, + MonthDayNanoIntervalScalar as MonthDayNanoIntervalScalar, + NA as NA, + NativeFile as NativeFile, + NullArray as NullArray, + NullScalar as NullScalar, + NumericArray as NumericArray, + OSFile as OSFile, + OpaqueArray as OpaqueArray, + OpaqueScalar as OpaqueScalar, + OpaqueType as OpaqueType, + ProxyMemoryPool as ProxyMemoryPool, + PythonFile as PythonFile, + RecordBatch as RecordBatch, + RecordBatchReader as RecordBatchReader, + ResizableBuffer as ResizableBuffer, + RunEndEncodedArray as RunEndEncodedArray, + RunEndEncodedScalar as RunEndEncodedScalar, + RunEndEncodedType as RunEndEncodedType, + RuntimeInfo as RuntimeInfo, + Scalar as Scalar, + Schema as Schema, + SparseCOOTensor as SparseCOOTensor, + SparseCSCMatrix as SparseCSCMatrix, + SparseCSFTensor as SparseCSFTensor, + SparseCSRMatrix as SparseCSRMatrix, + SparseUnionType as SparseUnionType, + StringArray as StringArray, + StringScalar as StringScalar, + StringViewArray as StringViewArray, + StringViewScalar as StringViewScalar, + StructArray as StructArray, + StructScalar as StructScalar, + StructType as StructType, + Table as Table, + TableGroupBy as TableGroupBy, + Tensor as Tensor, + Time32Array as Time32Array, + Time32Scalar as Time32Scalar, + Time32Type as Time32Type, + Time64Array as Time64Array, + Time64Scalar as Time64Scalar, + Time64Type as Time64Type, + TimestampArray as TimestampArray, + TimestampScalar as TimestampScalar, + TimestampType as TimestampType, + TransformInputStream as TransformInputStream, + UInt16Array as UInt16Array, + UInt16Scalar as UInt16Scalar, + UInt32Array as UInt32Array, + UInt32Scalar as UInt32Scalar, + UInt64Array as UInt64Array, + UInt64Scalar as UInt64Scalar, + UInt8Array as UInt8Array, + UInt8Scalar as UInt8Scalar, + UnionArray as UnionArray, + UnionScalar as UnionScalar, + UnionType as UnionType, + UnknownExtensionType as UnknownExtensionType, + UuidArray as UuidArray, + UuidScalar as UuidScalar, + UuidType as UuidType, + VersionInfo as VersionInfo, + allocate_buffer as allocate_buffer, + arange as arange, + array as array, + binary as binary, + binary_view as binary_view, + bool8 as bool8, + bool_ as bool_, + build_info as build_info, + chunked_array as chunked_array, + compress as compress, + concat_arrays as concat_arrays, + concat_batches as concat_batches, + concat_tables as concat_tables, + cpp_build_info as cpp_build_info, + cpp_version as cpp_version, + cpp_version_info as cpp_version_info, + cpu_count as cpu_count, + create_memory_map as create_memory_map, + date32 as date32, + date64 as date64, + decimal128 as decimal128, + decimal256 as decimal256, + decimal32 as decimal32, + decimal64 as decimal64, + decompress as decompress, + default_cpu_memory_manager as default_cpu_memory_manager, + default_memory_pool as default_memory_pool, + dense_union as dense_union, + dictionary as dictionary, + duration as duration, + enable_signal_handlers as enable_signal_handlers, + field as field, + fixed_shape_tensor as fixed_shape_tensor, + float16 as float16, + float32 as float32, + float64 as float64, + foreign_buffer as foreign_buffer, + from_numpy_dtype as from_numpy_dtype, + infer_type as infer_type, + input_stream as input_stream, + int16 as int16, + int32 as int32, + int64 as int64, + int8 as int8, + io_thread_count as io_thread_count, + is_opentelemetry_enabled as is_opentelemetry_enabled, + jemalloc_memory_pool as jemalloc_memory_pool, + jemalloc_set_decay_ms as jemalloc_set_decay_ms, + json_ as json_, + large_binary as large_binary, + large_list as large_list, + large_list_view as large_list_view, + large_string as large_string, + large_utf8 as large_utf8, + list_ as list_, + list_view as list_view, + log_memory_allocations as log_memory_allocations, + logging_memory_pool as logging_memory_pool, + map_ as map_, + memory_map as memory_map, + mimalloc_memory_pool as mimalloc_memory_pool, + month_day_nano_interval as month_day_nano_interval, + null as null, + nulls as nulls, + opaque as opaque, + output_stream as output_stream, + proxy_memory_pool as proxy_memory_pool, + py_buffer as py_buffer, + record_batch as record_batch, + register_extension_type as register_extension_type, + repeat as repeat, + run_end_encoded as run_end_encoded, + runtime_info as runtime_info, + scalar as scalar, + schema as schema, + set_cpu_count as set_cpu_count, + set_io_thread_count as set_io_thread_count, + set_memory_pool as set_memory_pool, + set_timezone_db_path as set_timezone_db_path, + sparse_union as sparse_union, + string as string, + string_view as string_view, + struct as struct, + supported_memory_backends as supported_memory_backends, + system_memory_pool as system_memory_pool, + table as table, + time32 as time32, + time64 as time64, + timestamp as timestamp, + total_allocated_bytes as total_allocated_bytes, + transcoding_input_stream as transcoding_input_stream, + type_for_alias as type_for_alias, + uint16 as uint16, + uint32 as uint32, + uint64 as uint64, + uint8 as uint8, + unify_schemas as unify_schemas, + union as union, + unregister_extension_type as unregister_extension_type, + utf8 as utf8, + uuid as uuid, +) +from pyarrow.util import ( + _deprecate_api as _deprecate_api, + _deprecate_class as _deprecate_class, +) -def parse_git(root, **kwargs): - ... - -def show_versions() -> None: - ... - -def _module_is_available(module): - ... - -def _filesystem_is_available(fs): - ... - -def have_libhdfs() -> bool: - ... - -def show_info() -> None: - ... - -def get_include() -> str: - ... - -def _get_pkg_config_executable(): - ... - -def _has_pkg_config(pkgname): - ... - -def _read_pkg_config_variable(pkgname, cli_args): - ... - -def get_libraries(): - ... - -def create_library_symlinks(): - ... - -def get_library_dirs(): - ... +def parse_git(root, **kwargs): ... +def show_versions() -> None: ... +def _module_is_available(module): ... +def _filesystem_is_available(fs): ... +def have_libhdfs() -> bool: ... +def show_info() -> None: ... +def get_include() -> str: ... +def _get_pkg_config_executable(): ... +def _has_pkg_config(pkgname): ... +def _read_pkg_config_variable(pkgname, cli_args): ... +def get_libraries(): ... +def create_library_symlinks(): ... +def get_library_dirs(): ... diff --git a/python/pyarrow-stubs/pyarrow/_acero.pyi b/python/pyarrow-stubs/pyarrow/_acero.pyi index 5f3901e2cdf3..fb2fdb7f3b2c 100644 --- a/python/pyarrow-stubs/pyarrow/_acero.pyi +++ b/python/pyarrow-stubs/pyarrow/_acero.pyi @@ -16,12 +16,8 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_acero.pyx -import os -import re import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence from pyarrow.lib import _Weakrefable -from _typeshed import Incomplete from pyarrow._compute import Expression from pyarrow.lib import * from pyarrow.lib import RecordBatchReader, Table @@ -30,102 +26,76 @@ class ExecNodeOptions(_Weakrefable): __slots__ = () class _TableSourceNodeOptions(ExecNodeOptions): - - def _set_options(self, table: Table) -> None: - ... + def _set_options(self, table: Table) -> None: ... class TableSourceNodeOptions(_TableSourceNodeOptions): - - def __init__(self, table: Table): - ... + def __init__(self, table: Table): ... class _RecordBatchReaderSourceNodeOptions(ExecNodeOptions): - - def _set_options(self, reader: RecordBatchReader) -> None: - ... + def _set_options(self, reader: RecordBatchReader) -> None: ... class RecordBatchReaderSourceNodeOptions(_RecordBatchReaderSourceNodeOptions): - - def __init__(self, reader: RecordBatchReader): - ... + def __init__(self, reader: RecordBatchReader): ... class _FilterNodeOptions(ExecNodeOptions): - - def _set_options(self, filter_expression: Expression) -> None: - ... + def _set_options(self, filter_expression: Expression) -> None: ... class FilterNodeOptions(_FilterNodeOptions): - - def __init__(self, filter_expression: Expression): - ... + def __init__(self, filter_expression: Expression): ... class _ProjectNodeOptions(ExecNodeOptions): - - def _set_options(self, expressions, names=None) -> None: - ... + def _set_options(self, expressions, names=None) -> None: ... class ProjectNodeOptions(_ProjectNodeOptions): - - def __init__(self, expressions, names=None): - ... + def __init__(self, expressions, names=None): ... class _AggregateNodeOptions(ExecNodeOptions): - - def _set_options(self, aggregates, keys=None) -> None: - ... + def _set_options(self, aggregates, keys=None) -> None: ... class AggregateNodeOptions(_AggregateNodeOptions): - - def __init__(self, aggregates, keys=None): - ... + def __init__(self, aggregates, keys=None): ... class _OrderByNodeOptions(ExecNodeOptions): - - def _set_options(self, sort_keys, null_placement) -> None: - ... + def _set_options(self, sort_keys, null_placement) -> None: ... class OrderByNodeOptions(_OrderByNodeOptions): - - def __init__(self, sort_keys=(), *, null_placement=None): - ... + def __init__(self, sort_keys=(), *, null_placement=None): ... class _HashJoinNodeOptions(ExecNodeOptions): - - def _set_options(self, join_type, left_keys, right_keys, left_output=None, right_output=None, output_suffix_for_left: str='', output_suffix_for_right: str='', filter_expression: Expression | None=None) -> None: - ... + def _set_options( + self, + join_type, + left_keys, + right_keys, + left_output=None, + right_output=None, + output_suffix_for_left: str = "", + output_suffix_for_right: str = "", + filter_expression: Expression | None = None, + ) -> None: ... class HashJoinNodeOptions(_HashJoinNodeOptions): - - def __init__(self, join_type, left_keys, right_keys, left_output=None, right_output=None, output_suffix_for_left: str='', output_suffix_for_right: str='', filter_expression=None): - ... + def __init__( + self, + join_type, + left_keys, + right_keys, + left_output=None, + right_output=None, + output_suffix_for_left: str = "", + output_suffix_for_right: str = "", + filter_expression=None, + ): ... class _AsofJoinNodeOptions(ExecNodeOptions): - - def _set_options(self, left_on, left_by, right_on, right_by, tolerance) -> None: - ... + def _set_options(self, left_on, left_by, right_on, right_by, tolerance) -> None: ... class AsofJoinNodeOptions(_AsofJoinNodeOptions): - - def __init__(self, left_on, left_by, right_on, right_by, tolerance): - ... + def __init__(self, left_on, left_by, right_on, right_by, tolerance): ... class Declaration(_Weakrefable): - - def __init__(self, factory_name, options: ExecNodeOptions, inputs=None): - ... - + def __init__(self, factory_name, options: ExecNodeOptions, inputs=None): ... @staticmethod - def from_sequence(decls: list[Declaration]) -> Declaration: - ... - - def __str__(self) -> str: - ... - - def __repr__(self) -> str: - ... - - def to_table(self, use_threads: bool=True) -> pyarrow.Table: - ... - - def to_reader(self, use_threads: bool=True) -> pyarrow.RecordBatchReader: - ... + def from_sequence(decls: list[Declaration]) -> Declaration: ... + def to_table(self, use_threads: bool = True) -> pyarrow.Table: ... + def to_reader(self, use_threads: bool = True) -> pyarrow.RecordBatchReader: ... diff --git a/python/pyarrow-stubs/pyarrow/_azurefs.pyi b/python/pyarrow-stubs/pyarrow/_azurefs.pyi index e9da70ac4d20..b0a8ac096fd2 100644 --- a/python/pyarrow-stubs/pyarrow/_azurefs.pyi +++ b/python/pyarrow-stubs/pyarrow/_azurefs.pyi @@ -16,21 +16,23 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_azurefs.pyx -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete from pyarrow._fs import FileSystem class AzureFileSystem(FileSystem): - - def __init__(self, account_name, *, account_key=None, blob_storage_authority=None, blob_storage_scheme=None, client_id=None, client_secret=None, dfs_storage_authority=None, dfs_storage_scheme=None, sas_token=None, tenant_id=None): - ... - + def __init__( + self, + account_name, + *, + account_key=None, + blob_storage_authority=None, + blob_storage_scheme=None, + client_id=None, + client_secret=None, + dfs_storage_authority=None, + dfs_storage_scheme=None, + sas_token=None, + tenant_id=None, + ): ... @staticmethod - def _reconstruct(kwargs) -> AzureFileSystem: - ... - - def __reduce__(self) -> tuple: - ... + def _reconstruct(kwargs) -> AzureFileSystem: ... + def __reduce__(self) -> tuple: ... diff --git a/python/pyarrow-stubs/pyarrow/_compute.pyi b/python/pyarrow-stubs/pyarrow/_compute.pyi index 665644920841..f74ab5c045a3 100644 --- a/python/pyarrow-stubs/pyarrow/_compute.pyi +++ b/python/pyarrow-stubs/pyarrow/_compute.pyi @@ -16,69 +16,82 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_compute.pyx -import os -import re import pyarrow -from typing import IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from typing import NamedTuple +from collections.abc import Iterable from pyarrow.lib import _Weakrefable from _typeshed import Incomplete -from collections import namedtuple -from typing import Any, Callable -import pyarrow.lib as lib +from typing import Any +from collections.abc import Callable from pyarrow.lib import * -from typing_extensions import TypeAlias + __pas: Incomplete _substrait_msg: str SUPPORTED_INPUT_ARR_TYPES: tuple[Incomplete, ...] _deserialize: Incomplete -def _pas(): - ... - -def _forbid_instantiation(klass, subclasses_instead: bool=True): - ... - -def function_registry(): - ... - -def get_function(name: str): - ... - -def list_functions(): - ... - -def call_function(name: str, args: list, options=None, memory_pool: pyarrow.MemoryPool | None=None, length: int | None=None): - ... - -def _raise_invalid_function_option(value, description, *, exception_class=ValueError): - ... - -def _skip_nulls_doc() -> str: - ... - -def _min_count_doc(*, default) -> str: - ... - -def _get_udf_context(memory_pool, batch_length): - ... - -def register_scalar_function(func: Callable[..., Any], function_name: str, function_doc: dict, in_types, out_type: pyarrow.DataType, func_registry: FunctionRegistry | None=None): - ... - -def register_vector_function(func: Callable[..., Any], function_name: str, function_doc: dict, in_types, out_type: pyarrow.DataType, func_registry: FunctionRegistry | None=None): - ... - -def register_aggregate_function(func: Callable[..., Any], function_name: str, function_doc: dict, in_types, out_type: pyarrow.DataType, func_registry: FunctionRegistry | None=None): - ... - -def register_tabular_function(func: Callable[..., Any], function_name: str, function_doc: dict, in_types, out_type, func_registry: FunctionRegistry | None=None): - ... - -def _register_user_defined_function(register_func, func: Callable[..., Any], function_name: str, function_doc: dict, in_types, out_type: pyarrow.DataType, func_registry: FunctionRegistry | None=None) -> None: - ... - -def call_tabular_function(function_name: str, args: Iterable[Any] | None=None, func_registry: FunctionRegistry | None=None) -> pyarrow.RecordBatchReader: - ... +def _pas(): ... +def _forbid_instantiation(klass, subclasses_instead: bool = True): ... +def function_registry(): ... +def get_function(name: str): ... +def list_functions(): ... +def call_function( + name: str, + args: list, + options=None, + memory_pool: pyarrow.MemoryPool | None = None, + length: int | None = None, +): ... +def _raise_invalid_function_option(value, description, *, exception_class=...): ... +def _skip_nulls_doc() -> str: ... +def _min_count_doc(*, default) -> str: ... +def _get_udf_context(memory_pool, batch_length): ... +def register_scalar_function( + func: Callable[..., Any], + function_name: str, + function_doc: dict, + in_types, + out_type: pyarrow.DataType, + func_registry: FunctionRegistry | None = None, +): ... +def register_vector_function( + func: Callable[..., Any], + function_name: str, + function_doc: dict, + in_types, + out_type: pyarrow.DataType, + func_registry: FunctionRegistry | None = None, +): ... +def register_aggregate_function( + func: Callable[..., Any], + function_name: str, + function_doc: dict, + in_types, + out_type: pyarrow.DataType, + func_registry: FunctionRegistry | None = None, +): ... +def register_tabular_function( + func: Callable[..., Any], + function_name: str, + function_doc: dict, + in_types, + out_type, + func_registry: FunctionRegistry | None = None, +): ... +def _register_user_defined_function( + register_func, + func: Callable[..., Any], + function_name: str, + function_doc: dict, + in_types, + out_type: pyarrow.DataType, + func_registry: FunctionRegistry | None = None, +) -> None: ... +def call_tabular_function( + function_name: str, + args: Iterable[Any] | None = None, + func_registry: FunctionRegistry | None = None, +) -> pyarrow.RecordBatchReader: ... class FunctionDoc(NamedTuple): summary: Incomplete @@ -88,889 +101,599 @@ class FunctionDoc(NamedTuple): options_required: Incomplete class Kernel(_Weakrefable): + def __init__(self): ... - def __init__(self): - ... - -class ScalarKernel(Kernel): - - def __repr__(self) -> str: - ... - -class VectorKernel(Kernel): - - def __repr__(self) -> str: - ... - -class ScalarAggregateKernel(Kernel): - - def __repr__(self) -> str: - ... - -class HashAggregateKernel(Kernel): - - def __repr__(self) -> str: - ... +class ScalarKernel(Kernel): ... +class VectorKernel(Kernel): ... +class ScalarAggregateKernel(Kernel): ... +class HashAggregateKernel(Kernel): ... class Function(_Weakrefable): _kind_map: Incomplete - def __init__(self): - ... - - def __repr__(self) -> str: - ... - - def __reduce__(self) -> tuple: - ... - + def __init__(self): ... + def __reduce__(self) -> tuple: ... @property - def name(self) -> str: - ... - + def name(self) -> str: ... @property - def arity(self): - ... - + def arity(self): ... @property - def kind(self): - ... - + def kind(self): ... @property - def _doc(self): - ... - + def _doc(self): ... @property - def num_kernels(self): - ... - - def call(self, args: Iterable[Any], options: FunctionOptions | None=None, memory_pool: MemoryPool | None=None, length: int | None=None): - ... + def num_kernels(self): ... + def call( + self, + args: Iterable[Any], + options: FunctionOptions | None = None, + memory_pool: MemoryPool | None = None, + length: int | None = None, + ): ... class ScalarFunction(Function): - @property - def kernels(self) -> list: - ... + def kernels(self) -> list: ... class VectorFunction(Function): - @property - def kernels(self) -> list: - ... + def kernels(self) -> list: ... class ScalarAggregateFunction(Function): - @property - def kernels(self) -> list: - ... + def kernels(self) -> list: ... class HashAggregateFunction(Function): - @property - def kernels(self) -> list: - ... + def kernels(self) -> list: ... class MetaFunction(Function): - @property - def kernels(self) -> list: - ... + def kernels(self) -> list: ... class FunctionRegistry(_Weakrefable): - - def __init__(self): - ... - - def list_functions(self) -> list: - ... - - def get_function(self, name: str): - ... + def __init__(self): ... + def list_functions(self) -> list: ... + def get_function(self, name: str): ... class FunctionOptions(_Weakrefable): __slots__ = () - def serialize(self) -> pyarrow.Buffer: - ... - + def serialize(self) -> pyarrow.Buffer: ... @staticmethod - def deserialize(buf: Buffer): - ... - - def __repr__(self) -> str: - ... - - def __eq__(self, other: FunctionOptions): - ... + def deserialize(buf: Buffer): ... + def __eq__(self, other: FunctionOptions): ... class _CastOptions(FunctionOptions): - - def _set_options(self, target_type: DataType, allow_int_overflow, allow_time_truncate, allow_time_overflow, allow_decimal_truncate, allow_float_truncate, allow_invalid_utf8) -> None: - ... - - def _set_type(self, target_type=None) -> None: - ... - - def _set_safe(self) -> None: - ... - - def _set_unsafe(self) -> None: - ... - - def is_safe(self) -> bool: - ... - + def _set_options( + self, + target_type: DataType, + allow_int_overflow, + allow_time_truncate, + allow_time_overflow, + allow_decimal_truncate, + allow_float_truncate, + allow_invalid_utf8, + ) -> None: ... + def _set_type(self, target_type=None) -> None: ... + def _set_safe(self) -> None: ... + def _set_unsafe(self) -> None: ... + def is_safe(self) -> bool: ... @property - def allow_int_overflow(self) -> None: - ... - + def allow_int_overflow(self) -> None: ... @allow_int_overflow.setter - def allow_int_overflow(self, flag: bool) -> None: - ... - + def allow_int_overflow(self, flag: bool) -> None: ... @property - def allow_time_truncate(self) -> None: - ... - + def allow_time_truncate(self) -> None: ... @allow_time_truncate.setter - def allow_time_truncate(self, flag: bool) -> None: - ... - + def allow_time_truncate(self, flag: bool) -> None: ... @property - def allow_time_overflow(self) -> None: - ... - + def allow_time_overflow(self) -> None: ... @allow_time_overflow.setter - def allow_time_overflow(self, flag: bool) -> None: - ... - + def allow_time_overflow(self, flag: bool) -> None: ... @property - def allow_decimal_truncate(self) -> None: - ... - + def allow_decimal_truncate(self) -> None: ... @allow_decimal_truncate.setter - def allow_decimal_truncate(self, flag: bool) -> None: - ... - + def allow_decimal_truncate(self, flag: bool) -> None: ... @property - def allow_float_truncate(self) -> None: - ... - + def allow_float_truncate(self) -> None: ... @allow_float_truncate.setter - def allow_float_truncate(self, flag: bool) -> None: - ... - + def allow_float_truncate(self, flag: bool) -> None: ... @property - def allow_invalid_utf8(self) -> None: - ... - + def allow_invalid_utf8(self) -> None: ... @allow_invalid_utf8.setter - def allow_invalid_utf8(self, flag: bool) -> None: - ... + def allow_invalid_utf8(self, flag: bool) -> None: ... class CastOptions(_CastOptions): - - def __init__(self, target_type=None, *, allow_int_overflow=None, allow_time_truncate=None, allow_time_overflow=None, allow_decimal_truncate=None, allow_float_truncate=None, allow_invalid_utf8=None): - ... - + def __init__( + self, + target_type=None, + *, + allow_int_overflow=None, + allow_time_truncate=None, + allow_time_overflow=None, + allow_decimal_truncate=None, + allow_float_truncate=None, + allow_invalid_utf8=None, + ): ... @staticmethod - def safe(target_type=None) -> CastOptions: - ... - + def safe(target_type=None) -> CastOptions: ... @staticmethod - def unsafe(target_type=None) -> CastOptions: - ... + def unsafe(target_type=None) -> CastOptions: ... class _ElementWiseAggregateOptions(FunctionOptions): - - def _set_options(self, skip_nulls) -> None: - ... + def _set_options(self, skip_nulls) -> None: ... class ElementWiseAggregateOptions(_ElementWiseAggregateOptions): - - def __init__(self, *, skip_nulls: bool=True): - ... + def __init__(self, *, skip_nulls: bool = True): ... class _RoundOptions(FunctionOptions): - - def _set_options(self, ndigits, round_mode) -> None: - ... + def _set_options(self, ndigits, round_mode) -> None: ... class RoundOptions(_RoundOptions): - - def __init__(self, ndigits: int=0, round_mode: str='half_to_even'): - ... + def __init__(self, ndigits: int = 0, round_mode: str = "half_to_even"): ... class _RoundBinaryOptions(FunctionOptions): - - def _set_options(self, round_mode) -> None: - ... + def _set_options(self, round_mode) -> None: ... class RoundBinaryOptions(_RoundBinaryOptions): - - def __init__(self, round_mode: str='half_to_even'): - ... + def __init__(self, round_mode: str = "half_to_even"): ... class _RoundTemporalOptions(FunctionOptions): - - def _set_options(self, multiple, unit, week_starts_monday, ceil_is_strictly_greater, calendar_based_origin) -> None: - ... + def _set_options( + self, + multiple, + unit, + week_starts_monday, + ceil_is_strictly_greater, + calendar_based_origin, + ) -> None: ... class RoundTemporalOptions(_RoundTemporalOptions): - - def __init__(self, multiple: int=1, unit: str='day', *, week_starts_monday: bool=True, ceil_is_strictly_greater: bool=False, calendar_based_origin: bool=False): - ... + def __init__( + self, + multiple: int = 1, + unit: str = "day", + *, + week_starts_monday: bool = True, + ceil_is_strictly_greater: bool = False, + calendar_based_origin: bool = False, + ): ... class _RoundToMultipleOptions(FunctionOptions): - - def _set_options(self, multiple, round_mode) -> None: - ... + def _set_options(self, multiple, round_mode) -> None: ... class RoundToMultipleOptions(_RoundToMultipleOptions): - - def __init__(self, multiple: float=1.0, round_mode: str='half_to_even'): - ... + def __init__(self, multiple: float = 1.0, round_mode: str = "half_to_even"): ... class _JoinOptions(FunctionOptions): _null_handling_map: Incomplete - def _set_options(self, null_handling, null_replacement) -> None: - ... + def _set_options(self, null_handling, null_replacement) -> None: ... class JoinOptions(_JoinOptions): - - def __init__(self, null_handling: str='emit_null', null_replacement: str=''): - ... + def __init__( + self, null_handling: str = "emit_null", null_replacement: str = "" + ): ... class _MatchSubstringOptions(FunctionOptions): - - def _set_options(self, pattern, ignore_case) -> None: - ... + def _set_options(self, pattern, ignore_case) -> None: ... class MatchSubstringOptions(_MatchSubstringOptions): - - def __init__(self, pattern, *, ignore_case: bool=False): - ... + def __init__(self, pattern, *, ignore_case: bool = False): ... class _PadOptions(FunctionOptions): - - def _set_options(self, width, padding, lean_left_on_odd_padding) -> None: - ... + def _set_options(self, width, padding, lean_left_on_odd_padding) -> None: ... class PadOptions(_PadOptions): - - def __init__(self, width, padding: str=' ', lean_left_on_odd_padding: bool=True): - ... + def __init__( + self, width, padding: str = " ", lean_left_on_odd_padding: bool = True + ): ... class _ZeroFillOptions(FunctionOptions): - - def _set_options(self, width, padding) -> None: - ... + def _set_options(self, width, padding) -> None: ... class ZeroFillOptions(_ZeroFillOptions): - - def __init__(self, width, padding: str='0'): - ... + def __init__(self, width, padding: str = "0"): ... class _TrimOptions(FunctionOptions): - - def _set_options(self, characters) -> None: - ... + def _set_options(self, characters) -> None: ... class TrimOptions(_TrimOptions): - - def __init__(self, characters): - ... + def __init__(self, characters): ... class _ReplaceSubstringOptions(FunctionOptions): - - def _set_options(self, pattern, replacement, max_replacements) -> None: - ... + def _set_options(self, pattern, replacement, max_replacements) -> None: ... class ReplaceSubstringOptions(_ReplaceSubstringOptions): - - def __init__(self, pattern, replacement, *, max_replacements=None): - ... + def __init__(self, pattern, replacement, *, max_replacements=None): ... class _ExtractRegexOptions(FunctionOptions): - - def _set_options(self, pattern) -> None: - ... + def _set_options(self, pattern) -> None: ... class ExtractRegexOptions(_ExtractRegexOptions): - - def __init__(self, pattern): - ... + def __init__(self, pattern): ... class _ExtractRegexSpanOptions(FunctionOptions): - - def _set_options(self, pattern) -> None: - ... + def _set_options(self, pattern) -> None: ... class ExtractRegexSpanOptions(_ExtractRegexSpanOptions): - - def __init__(self, pattern): - ... + def __init__(self, pattern): ... class _SliceOptions(FunctionOptions): - - def _set_options(self, start, stop, step) -> None: - ... + def _set_options(self, start, stop, step) -> None: ... class SliceOptions(_SliceOptions): - - def __init__(self, start, stop=None, step: int=1): - ... + def __init__(self, start, stop=None, step: int = 1): ... class _ListSliceOptions(FunctionOptions): - - def _set_options(self, start, stop=None, step: int=1, return_fixed_size_list=None): - ... + def _set_options( + self, start, stop=None, step: int = 1, return_fixed_size_list=None + ): ... class ListSliceOptions(_ListSliceOptions): - - def __init__(self, start, stop=None, step: int=1, return_fixed_size_list=None): - ... + def __init__( + self, start, stop=None, step: int = 1, return_fixed_size_list=None + ): ... class _ReplaceSliceOptions(FunctionOptions): - - def _set_options(self, start, stop, replacement) -> None: - ... + def _set_options(self, start, stop, replacement) -> None: ... class ReplaceSliceOptions(_ReplaceSliceOptions): - - def __init__(self, start, stop, replacement): - ... + def __init__(self, start, stop, replacement): ... class _FilterOptions(FunctionOptions): _null_selection_map: Incomplete - def _set_options(self, null_selection_behavior) -> None: - ... + def _set_options(self, null_selection_behavior) -> None: ... class FilterOptions(_FilterOptions): - - def __init__(self, null_selection_behavior: str='drop'): - ... + def __init__(self, null_selection_behavior: str = "drop"): ... class _DictionaryEncodeOptions(FunctionOptions): _null_encoding_map: Incomplete - def _set_options(self, null_encoding) -> None: - ... + def _set_options(self, null_encoding) -> None: ... class DictionaryEncodeOptions(_DictionaryEncodeOptions): - - def __init__(self, null_encoding: str='mask'): - ... + def __init__(self, null_encoding: str = "mask"): ... class _RunEndEncodeOptions(FunctionOptions): - - def _set_options(self, run_end_type) -> None: - ... + def _set_options(self, run_end_type) -> None: ... class RunEndEncodeOptions(_RunEndEncodeOptions): - - def __init__(self, run_end_type=lib.int32()): - ... + def __init__(self, run_end_type=...): ... class _InversePermutationOptions(FunctionOptions): - - def _set_options(self, max_index=-1, output_type=None) -> None: - ... + def _set_options(self, max_index=-1, output_type=None) -> None: ... class InversePermutationOptions(_InversePermutationOptions): - - def __init__(self, max_index=-1, output_type=None): - ... + def __init__(self, max_index=-1, output_type=None): ... class _ScatterOptions(FunctionOptions): - - def _set_options(self, max_index) -> None: - ... + def _set_options(self, max_index) -> None: ... class ScatterOptions(_ScatterOptions): - - def __init__(self, max_index=-1): - ... + def __init__(self, max_index=-1): ... class _TakeOptions(FunctionOptions): - - def _set_options(self, boundscheck) -> None: - ... + def _set_options(self, boundscheck) -> None: ... class TakeOptions(_TakeOptions): - - def __init__(self, *, boundscheck: bool=True): - ... + def __init__(self, *, boundscheck: bool = True): ... class _MakeStructOptions(FunctionOptions): - - def _set_options(self, field_names, field_nullability, field_metadata) -> None: - ... + def _set_options(self, field_names, field_nullability, field_metadata) -> None: ... class MakeStructOptions(_MakeStructOptions): - - def __init__(self, field_names=(), *, field_nullability=None, field_metadata=None): - ... + def __init__( + self, field_names=(), *, field_nullability=None, field_metadata=None + ): ... class _StructFieldOptions(FunctionOptions): - - def _set_options(self, indices) -> None: - ... + def _set_options(self, indices) -> None: ... class StructFieldOptions(_StructFieldOptions): - - def __init__(self, indices): - ... + def __init__(self, indices): ... class _ScalarAggregateOptions(FunctionOptions): - - def _set_options(self, skip_nulls, min_count) -> None: - ... + def _set_options(self, skip_nulls, min_count) -> None: ... class ScalarAggregateOptions(_ScalarAggregateOptions): - - def __init__(self, *, skip_nulls: bool=True, min_count: int=1): - ... + def __init__(self, *, skip_nulls: bool = True, min_count: int = 1): ... class _CountOptions(FunctionOptions): _mode_map: Incomplete - def _set_options(self, mode) -> None: - ... + def _set_options(self, mode) -> None: ... class CountOptions(_CountOptions): - - def __init__(self, mode: str='only_valid'): - ... + def __init__(self, mode: str = "only_valid"): ... class _IndexOptions(FunctionOptions): - - def _set_options(self, scalar) -> None: - ... + def _set_options(self, scalar) -> None: ... class IndexOptions(_IndexOptions): - - def __init__(self, value): - ... + def __init__(self, value): ... class _MapLookupOptions(FunctionOptions): _occurrence_map: Incomplete - def _set_options(self, query_key, occurrence) -> None: - ... + def _set_options(self, query_key, occurrence) -> None: ... class MapLookupOptions(_MapLookupOptions): - - def __init__(self, query_key, occurrence): - ... + def __init__(self, query_key, occurrence): ... class _ModeOptions(FunctionOptions): - - def _set_options(self, n, skip_nulls, min_count) -> None: - ... + def _set_options(self, n, skip_nulls, min_count) -> None: ... class ModeOptions(_ModeOptions): - - def __init__(self, n: int=1, *, skip_nulls: bool=True, min_count: int=0): - ... + def __init__(self, n: int = 1, *, skip_nulls: bool = True, min_count: int = 0): ... class _SetLookupOptions(FunctionOptions): - - def _set_options(self, value_set, skip_nulls: bool) -> None: - ... + def _set_options(self, value_set, skip_nulls: bool) -> None: ... class SetLookupOptions(_SetLookupOptions): - - def __init__(self, value_set, *, skip_nulls: bool=False): - ... + def __init__(self, value_set, *, skip_nulls: bool = False): ... class _StrptimeOptions(FunctionOptions): _unit_map: Incomplete - def _set_options(self, format, unit, error_is_null) -> None: - ... + def _set_options(self, format, unit, error_is_null) -> None: ... class StrptimeOptions(_StrptimeOptions): - - def __init__(self, format, unit, error_is_null: bool=False): - ... + def __init__(self, format, unit, error_is_null: bool = False): ... class _StrftimeOptions(FunctionOptions): - - def _set_options(self, format, locale) -> None: - ... + def _set_options(self, format, locale) -> None: ... class StrftimeOptions(_StrftimeOptions): - - def __init__(self, format: str='%Y-%m-%dT%H:%M:%S', locale: str='C'): - ... + def __init__(self, format: str = "%Y-%m-%dT%H:%M:%S", locale: str = "C"): ... class _DayOfWeekOptions(FunctionOptions): - - def _set_options(self, count_from_zero, week_start) -> None: - ... + def _set_options(self, count_from_zero, week_start) -> None: ... class DayOfWeekOptions(_DayOfWeekOptions): - - def __init__(self, *, count_from_zero: bool=True, week_start: int=1): - ... + def __init__(self, *, count_from_zero: bool = True, week_start: int = 1): ... class _WeekOptions(FunctionOptions): - - def _set_options(self, week_starts_monday, count_from_zero, first_week_is_fully_in_year) -> None: - ... + def _set_options( + self, week_starts_monday, count_from_zero, first_week_is_fully_in_year + ) -> None: ... class WeekOptions(_WeekOptions): - - def __init__(self, *, week_starts_monday: bool=True, count_from_zero: bool=False, first_week_is_fully_in_year: bool=False): - ... + def __init__( + self, + *, + week_starts_monday: bool = True, + count_from_zero: bool = False, + first_week_is_fully_in_year: bool = False, + ): ... class _AssumeTimezoneOptions(FunctionOptions): _ambiguous_map: Incomplete _nonexistent_map: Incomplete - def _set_options(self, timezone, ambiguous, nonexistent) -> None: - ... + def _set_options(self, timezone, ambiguous, nonexistent) -> None: ... class AssumeTimezoneOptions(_AssumeTimezoneOptions): - - def __init__(self, timezone, *, ambiguous: str='raise', nonexistent: str='raise'): - ... + def __init__( + self, timezone, *, ambiguous: str = "raise", nonexistent: str = "raise" + ): ... class _NullOptions(FunctionOptions): - - def _set_options(self, nan_is_null) -> None: - ... + def _set_options(self, nan_is_null) -> None: ... class NullOptions(_NullOptions): - - def __init__(self, *, nan_is_null: bool=False): - ... + def __init__(self, *, nan_is_null: bool = False): ... class _VarianceOptions(FunctionOptions): - - def _set_options(self, ddof, skip_nulls, min_count) -> None: - ... + def _set_options(self, ddof, skip_nulls, min_count) -> None: ... class VarianceOptions(_VarianceOptions): - - def __init__(self, *, ddof: int=0, skip_nulls: bool=True, min_count: int=0): - ... + def __init__( + self, *, ddof: int = 0, skip_nulls: bool = True, min_count: int = 0 + ): ... class _SkewOptions(FunctionOptions): - - def _set_options(self, skip_nulls, biased, min_count) -> None: - ... + def _set_options(self, skip_nulls, biased, min_count) -> None: ... class SkewOptions(_SkewOptions): - - def __init__(self, *, skip_nulls: bool=True, biased: bool=True, min_count: int=0): - ... + def __init__( + self, *, skip_nulls: bool = True, biased: bool = True, min_count: int = 0 + ): ... class _SplitOptions(FunctionOptions): - - def _set_options(self, max_splits, reverse) -> None: - ... + def _set_options(self, max_splits, reverse) -> None: ... class SplitOptions(_SplitOptions): - - def __init__(self, *, max_splits=None, reverse: bool=False): - ... + def __init__(self, *, max_splits=None, reverse: bool = False): ... class _SplitPatternOptions(FunctionOptions): - - def _set_options(self, pattern, max_splits, reverse) -> None: - ... + def _set_options(self, pattern, max_splits, reverse) -> None: ... class SplitPatternOptions(_SplitPatternOptions): - - def __init__(self, pattern, *, max_splits=None, reverse: bool=False): - ... + def __init__(self, pattern, *, max_splits=None, reverse: bool = False): ... class _PartitionNthOptions(FunctionOptions): - - def _set_options(self, pivot, null_placement) -> None: - ... + def _set_options(self, pivot, null_placement) -> None: ... class PartitionNthOptions(_PartitionNthOptions): - - def __init__(self, pivot, *, null_placement: str='at_end'): - ... + def __init__(self, pivot, *, null_placement: str = "at_end"): ... class _WinsorizeOptions(FunctionOptions): - - def _set_options(self, lower_limit, upper_limit) -> None: - ... + def _set_options(self, lower_limit, upper_limit) -> None: ... class WinsorizeOptions(_WinsorizeOptions): - - def __init__(self, lower_limit, upper_limit): - ... + def __init__(self, lower_limit, upper_limit): ... class _CumulativeOptions(FunctionOptions): - - def _set_options(self, start, skip_nulls) -> None: - ... + def _set_options(self, start, skip_nulls) -> None: ... class CumulativeOptions(_CumulativeOptions): - - def __init__(self, start=None, *, skip_nulls: bool=False): - ... + def __init__(self, start=None, *, skip_nulls: bool = False): ... class CumulativeSumOptions(_CumulativeOptions): - - def __init__(self, start=None, *, skip_nulls: bool=False): - ... + def __init__(self, start=None, *, skip_nulls: bool = False): ... class _PairwiseOptions(FunctionOptions): - - def _set_options(self, period) -> None: - ... + def _set_options(self, period) -> None: ... class PairwiseOptions(_PairwiseOptions): - - def __init__(self, period: int=1): - ... + def __init__(self, period: int = 1): ... class _ListFlattenOptions(FunctionOptions): - - def _set_options(self, recursive) -> None: - ... + def _set_options(self, recursive) -> None: ... class ListFlattenOptions(_ListFlattenOptions): - - def __init__(self, recursive: bool=False): - ... + def __init__(self, recursive: bool = False): ... class _ArraySortOptions(FunctionOptions): - - def _set_options(self, order, null_placement) -> None: - ... + def _set_options(self, order, null_placement) -> None: ... class ArraySortOptions(_ArraySortOptions): - - def __init__(self, order: str='ascending', *, null_placement: str='at_end'): - ... + def __init__(self, order: str = "ascending", *, null_placement: str = "at_end"): ... class _SearchSortedOptions(FunctionOptions): - - def _set_options(self, side) -> None: - ... + def _set_options(self, side) -> None: ... class SearchSortedOptions(_SearchSortedOptions): - - def __init__(self, side: str='left'): - ... + def __init__(self, side: str = "left"): ... class _SortOptions(FunctionOptions): - - def _set_options(self, sort_keys, null_placement) -> None: - ... + def _set_options(self, sort_keys, null_placement) -> None: ... class SortOptions(_SortOptions): - - def __init__(self, sort_keys=(), *, null_placement=None): - ... + def __init__(self, sort_keys=(), *, null_placement=None): ... class _SelectKOptions(FunctionOptions): - - def _set_options(self, k, sort_keys) -> None: - ... + def _set_options(self, k, sort_keys) -> None: ... class SelectKOptions(_SelectKOptions): - - def __init__(self, k, sort_keys): - ... + def __init__(self, k, sort_keys): ... class _QuantileOptions(FunctionOptions): _interp_map: Incomplete - def _set_options(self, quantiles, interp, skip_nulls, min_count) -> None: - ... + def _set_options(self, quantiles, interp, skip_nulls, min_count) -> None: ... class QuantileOptions(_QuantileOptions): - - def __init__(self, q: float=0.5, *, interpolation: str='linear', skip_nulls: bool=True, min_count: int=0): - ... + def __init__( + self, + q: float = 0.5, + *, + interpolation: str = "linear", + skip_nulls: bool = True, + min_count: int = 0, + ): ... class _TDigestOptions(FunctionOptions): - - def _set_options(self, quantiles, delta, buffer_size, skip_nulls, min_count) -> None: - ... + def _set_options( + self, quantiles, delta, buffer_size, skip_nulls, min_count + ) -> None: ... class TDigestOptions(_TDigestOptions): - - def __init__(self, q: float=0.5, *, delta: int=100, buffer_size: int=500, skip_nulls: bool=True, min_count: int=0): - ... + def __init__( + self, + q: float = 0.5, + *, + delta: int = 100, + buffer_size: int = 500, + skip_nulls: bool = True, + min_count: int = 0, + ): ... class _Utf8NormalizeOptions(FunctionOptions): _form_map: Incomplete - def _set_options(self, form) -> None: - ... + def _set_options(self, form) -> None: ... class Utf8NormalizeOptions(_Utf8NormalizeOptions): - - def __init__(self, form): - ... + def __init__(self, form): ... class _RandomOptions(FunctionOptions): - - def _set_options(self, initializer) -> None: - ... + def _set_options(self, initializer) -> None: ... class RandomOptions(_RandomOptions): - - def __init__(self, *, initializer: str='system'): - ... + def __init__(self, *, initializer: str = "system"): ... class _RankOptions(FunctionOptions): _tiebreaker_map: Incomplete - def _set_options(self, sort_keys, null_placement, tiebreaker) -> None: - ... + def _set_options(self, sort_keys, null_placement, tiebreaker) -> None: ... class RankOptions(_RankOptions): - - def __init__(self, sort_keys: str='ascending', *, null_placement=None, tiebreaker: str='first'): - ... + def __init__( + self, + sort_keys: str = "ascending", + *, + null_placement=None, + tiebreaker: str = "first", + ): ... class _RankQuantileOptions(FunctionOptions): - - def _set_options(self, sort_keys, null_placement) -> None: - ... + def _set_options(self, sort_keys, null_placement) -> None: ... class RankQuantileOptions(_RankQuantileOptions): - - def __init__(self, sort_keys: str='ascending', *, null_placement=None): - ... + def __init__(self, sort_keys: str = "ascending", *, null_placement=None): ... class _PivotWiderOptions(FunctionOptions): - - def _set_options(self, key_names, unexpected_key_behavior) -> None: - ... + def _set_options(self, key_names, unexpected_key_behavior) -> None: ... class PivotWiderOptions(_PivotWiderOptions): - - def __init__(self, key_names, *, unexpected_key_behavior: str='ignore'): - ... + def __init__(self, key_names, *, unexpected_key_behavior: str = "ignore"): ... class Expression(_Weakrefable): - - def __init__(self): - ... - - def equals(self, other: Expression) -> bool: - ... - - def __str__(self) -> str: - ... - - def __repr__(self) -> str: - ... - + def __init__(self): ... + def equals(self, other: Expression) -> bool: ... @staticmethod - def from_substrait(message: object) -> Expression: - ... - - def to_substrait(self, schema: Schema, allow_arrow_extensions: bool=False) -> pyarrow.Buffer: - ... - + def from_substrait(message: object) -> Expression: ... + def to_substrait( + self, schema: Schema, allow_arrow_extensions: bool = False + ) -> pyarrow.Buffer: ... @staticmethod - def _deserialize(buffer: Buffer) -> Expression: - ... - - def __reduce__(self) -> tuple: - ... - + def _deserialize(buffer: Buffer) -> Expression: ... + def __reduce__(self) -> tuple: ... @staticmethod - def _call(function_name: str, arguments: list[Any], options: FunctionOptions | None=None) -> Expression: - ... - - def __bool__(self): - ... - - def __invert__(self): - ... - - def __and__(self: Expression, other): - ... - - def __rand__(self: Expression, other): - ... - - def __or__(self: Expression, other): - ... - - def __ror__(self: Expression, other): - ... - - def __add__(self: Expression, other): - ... - - def __radd__(self: Expression, other): - ... - - def __mul__(self: Expression, other): - ... - - def __rmul__(self: Expression, other): - ... - - def __sub__(self: Expression, other): - ... - - def __rsub__(self: Expression, other): - ... - - def __truediv__(self: Expression, other): - ... - - def __rtruediv__(self: Expression, other): - ... - - def is_valid(self) -> Expression: - ... - - def is_null(self, nan_is_null: bool=False) -> Expression: - ... - - def is_nan(self) -> Expression: - ... - - def cast(self, type: pyarrow.DataType | None=None, safe: bool | None=None, options: CastOptions | None=None) -> Expression: - ... - - def isin(self, values: pyarrow.Array | Iterable[Any]) -> Expression: - ... - + def _call( + function_name: str, arguments: list[Any], options: FunctionOptions | None = None + ) -> Expression: ... + def __bool__(self): ... + def __invert__(self): ... + def __and__(self: Expression, other): ... + def __rand__(self: Expression, other): ... + def __or__(self: Expression, other): ... + def __ror__(self: Expression, other): ... + def __add__(self: Expression, other): ... + def __radd__(self: Expression, other): ... + def __mul__(self: Expression, other): ... + def __rmul__(self: Expression, other): ... + def __sub__(self: Expression, other): ... + def __rsub__(self: Expression, other): ... + def __truediv__(self: Expression, other): ... + def __rtruediv__(self: Expression, other): ... + def is_valid(self) -> Expression: ... + def is_null(self, nan_is_null: bool = False) -> Expression: ... + def is_nan(self) -> Expression: ... + def cast( + self, + type: pyarrow.DataType | None = None, + safe: bool | None = None, + options: CastOptions | None = None, + ) -> Expression: ... + def isin(self, values: pyarrow.Array | Iterable[Any]) -> Expression: ... @staticmethod - def _field(name_or_idx) -> Expression: - ... - + def _field(name_or_idx) -> Expression: ... @staticmethod - def _nested_field(names: tuple[Any, ...]) -> Expression: - ... - + def _nested_field(names: tuple[Any, ...]) -> Expression: ... @staticmethod - def _scalar(value) -> Expression: - ... + def _scalar(value) -> Expression: ... class UdfContext(_Weakrefable): - - def __init__(self): - ... - + def __init__(self): ... @property - def batch_length(self): - ... - + def batch_length(self): ... @property - def memory_pool(self): - ... + def memory_pool(self): ... -class RegisterUdf(_Weakrefable): - ... +class RegisterUdf(_Weakrefable): ... diff --git a/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi b/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi index 3038ba72b538..a83cc5dc1e77 100644 --- a/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi +++ b/python/pyarrow-stubs/pyarrow/_compute_docstrings.pyi @@ -15,9 +15,6 @@ # specific language governing permissions and limitations # under the License. -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence from _typeshed import Incomplete + function_doc_additions: Incomplete diff --git a/python/pyarrow-stubs/pyarrow/_csv.pyi b/python/pyarrow-stubs/pyarrow/_csv.pyi index 6c6c8dfb1ff9..84b3fd8bf37d 100644 --- a/python/pyarrow-stubs/pyarrow/_csv.pyi +++ b/python/pyarrow-stubs/pyarrow/_csv.pyi @@ -16,28 +16,42 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_csv.pyx -import os -import re import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from typing import NamedTuple import pyarrow.csv from pyarrow.lib import __name__ as __name__ from _typeshed import Incomplete -from collections import namedtuple -from pyarrow.lib import MemoryPool, RecordBatchReader, Schema, _CRecordBatchWriter, _Weakrefable -ISO8601: _ISO8601 - -def read_csv(input_file, read_options: pyarrow.csv.ReadOptions | None=None, parse_options: pyarrow.csv.ParseOptions | None=None, convert_options: pyarrow.csv.ConvertOptions | None=None, memory_pool: MemoryPool | None=None) -> pyarrow.Table: - ... +from pyarrow.lib import ( + MemoryPool, + RecordBatchReader, + Schema, + _CRecordBatchWriter, + _Weakrefable, +) -def open_csv(input_file, read_options: pyarrow.csv.ReadOptions | None=None, parse_options: pyarrow.csv.ParseOptions | None=None, convert_options: pyarrow.csv.ConvertOptions | None=None, memory_pool: MemoryPool | None=None) -> CSVStreamingReader: - ... - -def _raise_invalid_function_option(value, description, *, exception_class=ValueError): - ... +ISO8601: _ISO8601 -def write_csv(data: pyarrow.RecordBatch | pyarrow.Table, output_file, write_options: pyarrow.csv.WriteOptions | None=None, memory_pool: MemoryPool | None=None) -> None: - ... +def read_csv( + input_file, + read_options: pyarrow.csv.ReadOptions | None = None, + parse_options: pyarrow.csv.ParseOptions | None = None, + convert_options: pyarrow.csv.ConvertOptions | None = None, + memory_pool: MemoryPool | None = None, +) -> pyarrow.Table: ... +def open_csv( + input_file, + read_options: pyarrow.csv.ReadOptions | None = None, + parse_options: pyarrow.csv.ParseOptions | None = None, + convert_options: pyarrow.csv.ConvertOptions | None = None, + memory_pool: MemoryPool | None = None, +) -> CSVStreamingReader: ... +def _raise_invalid_function_option(value, description, *, exception_class=...): ... +def write_csv( + data: pyarrow.RecordBatch | pyarrow.Table, + output_file, + write_options: pyarrow.csv.WriteOptions | None = None, + memory_pool: MemoryPool | None = None, +) -> None: ... class _InvalidRow(NamedTuple): expected_columns: Incomplete @@ -52,399 +66,242 @@ class ReadOptions(_Weakrefable): __slots__ = () encoding: object - def __init__(self, *, use_threads=None, block_size=None, skip_rows=None, skip_rows_after_names=None, column_names=None, autogenerate_column_names=None, encoding: str='utf8'): - ... - - @property - def use_threads(self) -> None: - ... - + def __init__( + self, + *, + use_threads=None, + block_size=None, + skip_rows=None, + skip_rows_after_names=None, + column_names=None, + autogenerate_column_names=None, + encoding: str = "utf8", + ): ... + @property + def use_threads(self) -> None: ... @use_threads.setter - def use_threads(self, value) -> None: - ... - + def use_threads(self, value) -> None: ... @property - def block_size(self) -> None: - ... - + def block_size(self) -> None: ... @block_size.setter - def block_size(self, value) -> None: - ... - + def block_size(self, value) -> None: ... @property - def skip_rows(self) -> None: - ... - + def skip_rows(self) -> None: ... @skip_rows.setter - def skip_rows(self, value) -> None: - ... - + def skip_rows(self, value) -> None: ... @property - def skip_rows_after_names(self) -> None: - ... - + def skip_rows_after_names(self) -> None: ... @skip_rows_after_names.setter - def skip_rows_after_names(self, value) -> None: - ... - + def skip_rows_after_names(self, value) -> None: ... @property - def column_names(self) -> None: - ... - + def column_names(self) -> None: ... @column_names.setter - def column_names(self, value) -> None: - ... - + def column_names(self, value) -> None: ... @property - def autogenerate_column_names(self) -> None: - ... - + def autogenerate_column_names(self) -> None: ... @autogenerate_column_names.setter - def autogenerate_column_names(self, value) -> None: - ... - - def validate(self) -> None: - ... - - def equals(self, other: ReadOptions) -> bool: - ... - - def __getstate__(self) -> tuple: - ... - - def __setstate__(self, state) -> None: - ... - - def __eq__(self, other): - ... - - def _repr_base(self) -> str: - ... - - def __repr__(self) -> str: - ... - - def __str__(self) -> str: - ... + def autogenerate_column_names(self, value) -> None: ... + def validate(self) -> None: ... + def equals(self, other: ReadOptions) -> bool: ... + def __getstate__(self) -> tuple: ... + def __setstate__(self, state) -> None: ... + def __eq__(self, other): ... + def _repr_base(self) -> str: ... class ParseOptions(_Weakrefable): __slots__ = () - def __init__(self, *, delimiter=None, quote_char=None, double_quote=None, escape_char=None, newlines_in_values=None, ignore_empty_lines=None, invalid_row_handler=None): - ... - - @property - def delimiter(self) -> None: - ... - + def __init__( + self, + *, + delimiter=None, + quote_char=None, + double_quote=None, + escape_char=None, + newlines_in_values=None, + ignore_empty_lines=None, + invalid_row_handler=None, + ): ... + @property + def delimiter(self) -> None: ... @delimiter.setter - def delimiter(self, value) -> None: - ... - + def delimiter(self, value) -> None: ... @property - def quote_char(self) -> None: - ... - + def quote_char(self) -> None: ... @quote_char.setter - def quote_char(self, value) -> None: - ... - + def quote_char(self, value) -> None: ... @property - def double_quote(self) -> None: - ... - + def double_quote(self) -> None: ... @double_quote.setter - def double_quote(self, value) -> None: - ... - + def double_quote(self, value) -> None: ... @property - def escape_char(self) -> None: - ... - + def escape_char(self) -> None: ... @escape_char.setter - def escape_char(self, value) -> None: - ... - + def escape_char(self, value) -> None: ... @property - def newlines_in_values(self) -> None: - ... - + def newlines_in_values(self) -> None: ... @newlines_in_values.setter - def newlines_in_values(self, value) -> None: - ... - + def newlines_in_values(self, value) -> None: ... @property - def ignore_empty_lines(self) -> None: - ... - + def ignore_empty_lines(self) -> None: ... @property - def invalid_row_handler(self) -> None: - ... - + def invalid_row_handler(self) -> None: ... @invalid_row_handler.setter - def invalid_row_handler(self, value) -> None: - ... - + def invalid_row_handler(self, value) -> None: ... @ignore_empty_lines.setter - def ignore_empty_lines(self, value) -> None: - ... - - def validate(self) -> None: - ... - - def equals(self, other: ParseOptions) -> bool: - ... - - def __getstate__(self) -> tuple: - ... - - def __setstate__(self, state) -> None: - ... - - def __eq__(self, other): - ... - - def _repr_base(self) -> str: - ... - - def __repr__(self) -> str: - ... - - def __str__(self) -> str: - ... + def ignore_empty_lines(self, value) -> None: ... + def validate(self) -> None: ... + def equals(self, other: ParseOptions) -> bool: ... + def __getstate__(self) -> tuple: ... + def __setstate__(self, state) -> None: ... + def __eq__(self, other): ... + def _repr_base(self) -> str: ... class _ISO8601(_Weakrefable): __slots__ = () - def __str__(self) -> str: - ... - - def __eq__(self, other) -> bool: - ... + def __eq__(self, other) -> bool: ... class ConvertOptions(_Weakrefable): __slots__ = () - def __init__(self, *, check_utf8=None, column_types=None, default_column_type=None, null_values=None, true_values=None, false_values=None, decimal_point=None, strings_can_be_null=None, quoted_strings_can_be_null=None, include_columns=None, include_missing_columns=None, auto_dict_encode=None, auto_dict_max_cardinality=None, timestamp_parsers=None): - ... - - @property - def check_utf8(self) -> None: - ... - + def __init__( + self, + *, + check_utf8=None, + column_types=None, + default_column_type=None, + null_values=None, + true_values=None, + false_values=None, + decimal_point=None, + strings_can_be_null=None, + quoted_strings_can_be_null=None, + include_columns=None, + include_missing_columns=None, + auto_dict_encode=None, + auto_dict_max_cardinality=None, + timestamp_parsers=None, + ): ... + @property + def check_utf8(self) -> None: ... @check_utf8.setter - def check_utf8(self, value) -> None: - ... - + def check_utf8(self, value) -> None: ... @property - def strings_can_be_null(self) -> None: - ... - + def strings_can_be_null(self) -> None: ... @strings_can_be_null.setter - def strings_can_be_null(self, value) -> None: - ... - + def strings_can_be_null(self, value) -> None: ... @property - def quoted_strings_can_be_null(self) -> None: - ... - + def quoted_strings_can_be_null(self) -> None: ... @quoted_strings_can_be_null.setter - def quoted_strings_can_be_null(self, value) -> None: - ... - + def quoted_strings_can_be_null(self, value) -> None: ... @property - def column_types(self) -> None: - ... - + def column_types(self) -> None: ... @column_types.setter - def column_types(self, value) -> None: - ... - + def column_types(self, value) -> None: ... @property - def default_column_type(self) -> None: - ... - + def default_column_type(self) -> None: ... @default_column_type.setter - def default_column_type(self, value) -> None: - ... - + def default_column_type(self, value) -> None: ... @property - def null_values(self) -> None: - ... - + def null_values(self) -> None: ... @null_values.setter - def null_values(self, value) -> None: - ... - + def null_values(self, value) -> None: ... @property - def true_values(self) -> None: - ... - + def true_values(self) -> None: ... @true_values.setter - def true_values(self, value) -> None: - ... - + def true_values(self, value) -> None: ... @property - def false_values(self) -> None: - ... - + def false_values(self) -> None: ... @false_values.setter - def false_values(self, value) -> None: - ... - + def false_values(self, value) -> None: ... @property - def decimal_point(self) -> None: - ... - + def decimal_point(self) -> None: ... @decimal_point.setter - def decimal_point(self, value) -> None: - ... - + def decimal_point(self, value) -> None: ... @property - def auto_dict_encode(self) -> None: - ... - + def auto_dict_encode(self) -> None: ... @auto_dict_encode.setter - def auto_dict_encode(self, value) -> None: - ... - + def auto_dict_encode(self, value) -> None: ... @property - def auto_dict_max_cardinality(self) -> None: - ... - + def auto_dict_max_cardinality(self) -> None: ... @auto_dict_max_cardinality.setter - def auto_dict_max_cardinality(self, value) -> None: - ... - + def auto_dict_max_cardinality(self, value) -> None: ... @property - def include_columns(self) -> None: - ... - + def include_columns(self) -> None: ... @include_columns.setter - def include_columns(self, value) -> None: - ... - + def include_columns(self, value) -> None: ... @property - def include_missing_columns(self) -> None: - ... - + def include_missing_columns(self) -> None: ... @include_missing_columns.setter - def include_missing_columns(self, value) -> None: - ... - + def include_missing_columns(self, value) -> None: ... @property - def timestamp_parsers(self) -> None: - ... - + def timestamp_parsers(self) -> None: ... @timestamp_parsers.setter - def timestamp_parsers(self, value) -> None: - ... - - def validate(self) -> None: - ... - - def equals(self, other: ConvertOptions) -> bool: - ... - - def __getstate__(self) -> tuple: - ... - - def __setstate__(self, state) -> None: - ... - - def __eq__(self, other): - ... - - def _repr_base(self) -> str: - ... - - def __repr__(self) -> str: - ... - - def __str__(self) -> str: - ... + def timestamp_parsers(self, value) -> None: ... + def validate(self) -> None: ... + def equals(self, other: ConvertOptions) -> bool: ... + def __getstate__(self) -> tuple: ... + def __setstate__(self, state) -> None: ... + def __eq__(self, other): ... + def _repr_base(self) -> str: ... class CSVStreamingReader(RecordBatchReader): schema: Schema - def __init__(self): - ... + def __init__(self): ... class WriteOptions(_Weakrefable): __slots__ = () - def __init__(self, *, include_header=None, batch_size=None, delimiter=None, eol=None, null_string=None, quoting_style=None, quoting_header=None): - ... - - @property - def include_header(self) -> None: - ... - + def __init__( + self, + *, + include_header=None, + batch_size=None, + delimiter=None, + eol=None, + null_string=None, + quoting_style=None, + quoting_header=None, + ): ... + @property + def include_header(self) -> None: ... @include_header.setter - def include_header(self, value) -> None: - ... - + def include_header(self, value) -> None: ... @property - def batch_size(self) -> None: - ... - + def batch_size(self) -> None: ... @batch_size.setter - def batch_size(self, value) -> None: - ... - + def batch_size(self, value) -> None: ... @property - def delimiter(self) -> None: - ... - + def delimiter(self) -> None: ... @delimiter.setter - def delimiter(self, value) -> None: - ... - + def delimiter(self, value) -> None: ... @property - def eol(self) -> None: - ... - + def eol(self) -> None: ... @eol.setter - def eol(self, value) -> None: - ... - + def eol(self, value) -> None: ... @property - def null_string(self) -> None: - ... - + def null_string(self) -> None: ... @null_string.setter - def null_string(self, value) -> None: - ... - + def null_string(self, value) -> None: ... @property - def quoting_style(self) -> None: - ... - + def quoting_style(self) -> None: ... @quoting_style.setter - def quoting_style(self, value) -> None: - ... - + def quoting_style(self, value) -> None: ... @property - def quoting_header(self) -> None: - ... - + def quoting_header(self) -> None: ... @quoting_header.setter - def quoting_header(self, value) -> None: - ... - - def validate(self) -> None: - ... - - def _repr_base(self) -> str: - ... - - def __repr__(self) -> str: - ... - - def __str__(self) -> str: - ... + def quoting_header(self, value) -> None: ... + def validate(self) -> None: ... + def _repr_base(self) -> str: ... class CSVWriter(_CRecordBatchWriter): - - def __init__(self, sink, schema: Schema, *, write_options: WriteOptions | None=None, memory_pool: MemoryPool | None=None): - ... + def __init__( + self, + sink, + schema: Schema, + *, + write_options: WriteOptions | None = None, + memory_pool: MemoryPool | None = None, + ): ... diff --git a/python/pyarrow-stubs/pyarrow/_cuda.pyi b/python/pyarrow-stubs/pyarrow/_cuda.pyi index cd5f02af7b2e..a0b978c0084e 100644 --- a/python/pyarrow-stubs/pyarrow/_cuda.pyi +++ b/python/pyarrow-stubs/pyarrow/_cuda.pyi @@ -16,14 +16,12 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_cuda.pyx -from _typeshed import Incomplete from pyarrow.includes.common import * from pyarrow.includes.libarrow import * from pyarrow.includes.libarrow_cuda import * from pyarrow.lib import * - class Context(_Weakrefable): def __init__(self, *args, **kwargs): ... @staticmethod @@ -46,7 +44,7 @@ class Context(_Weakrefable): def device(self): ... def foreign_buffer(self, address, size, base=None): ... def open_ipc_buffer(self, ipc_handle): ... - def buffer_from_data(self, data: object, offset: int=0, size: int=-1): ... + def buffer_from_data(self, data: object, offset: int = 0, size: int = -1): ... def buffer_from_object(self, obj): ... class IpcMemHandle(_Weakrefable): @@ -61,9 +59,16 @@ class CudaBuffer(Buffer): @staticmethod def from_numba(mem): ... def to_numba(self): ... - def copy_to_host(self, position: int=0, nbytes: int=-1, buf: Buffer | None=None, memory_pool: MemoryPool | None=None, resizable: c_bool=False): ... - def copy_from_host(self, data, position: int=0, nbytes: int=-1): ... - def copy_from_device(self, buf, position: int=0, nbytes: int=-1): ... + def copy_to_host( + self, + position: int = 0, + nbytes: int = -1, + buf: Buffer | None = None, + memory_pool: MemoryPool | None = None, + resizable: c_bool = False, + ): ... + def copy_from_host(self, data, position: int = 0, nbytes: int = -1): ... + def copy_from_device(self, buf, position: int = 0, nbytes: int = -1): ... def export_for_ipc(self): ... @property def context(self): ... @@ -83,7 +88,7 @@ class BufferWriter(NativeFile): def __init__(self, buffer: CudaBuffer): ... def writeat(self, position: int, data: object): ... def flush(self): ... - def seek(self, position: int, whence: int=0): ... + def seek(self, position: int, whence: int = 0): ... @property def buffer_size(self): ... @buffer_size.setter @@ -91,7 +96,13 @@ class BufferWriter(NativeFile): @property def num_bytes_buffered(self): ... -def new_host_buffer(size: int, device: int=0): ... +def new_host_buffer(size: int, device: int = 0): ... def serialize_record_batch(batch: object, ctx: object): ... def read_message(source: object, pool=None): ... -def read_record_batch(buffer: object, schema: object, *, dictionary_memo: DictionaryMemo | None=None, pool=None): ... +def read_record_batch( + buffer: object, + schema: object, + *, + dictionary_memo: DictionaryMemo | None = None, + pool=None, +): ... diff --git a/python/pyarrow-stubs/pyarrow/_dataset.pyi b/python/pyarrow-stubs/pyarrow/_dataset.pyi index 73831c6c9c94..86100b465553 100644 --- a/python/pyarrow-stubs/pyarrow/_dataset.pyi +++ b/python/pyarrow-stubs/pyarrow/_dataset.pyi @@ -17,15 +17,14 @@ # Generated by stubgen-pyx from python/pyarrow/_dataset.pyx import os -import re import pyarrow -from typing import Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from typing import IO, NamedTuple, Self +from collections.abc import Iterator import pyarrow.compute import pyarrow.dataset import pyarrow.fs from pyarrow.lib import _Weakrefable from _typeshed import Incomplete -import collections from typing import Any from pyarrow._acero import ExecNodeOptions from pyarrow._compute import Expression @@ -34,6 +33,7 @@ from pyarrow._fs import FileSystem from pyarrow._json import ParseOptions as JsonParseOptions from pyarrow._json import ReadOptions as JsonReadOptions from pyarrow.lib import * + _DEFAULT_BATCH_SIZE: int _DEFAULT_BATCH_READAHEAD: int _DEFAULT_FRAGMENT_READAHEAD: int @@ -41,624 +41,599 @@ _orc_fileformat: Incomplete _orc_imported: bool _dataset_pq: bool -def _get_orc_fileformat() -> None: - ... - -def _get_parquet_classes() -> None: - ... - -def _get_parquet_symbol(name): - ... - -def _constructor_directory_partitioning_factory(*args): - ... - -def _constructor_hive_partitioning_factory(*args): - ... - -def _constructor_filename_partitioning_factory(*args): - ... - -def get_partition_keys(partition_expression: Expression) -> dict: - ... - -def _filesystemdataset_write(data: Scanner, base_dir: object, basename_template: str, filesystem: FileSystem, partitioning: Partitioning, preserve_order: bool, file_options: FileWriteOptions, max_partitions: int, file_visitor: object, existing_data_behavior: str, max_open_files: int, max_rows_per_file: int, min_rows_per_group: int, max_rows_per_group: int, create_dir: bool) -> None: - ... +def _get_orc_fileformat() -> None: ... +def _get_parquet_classes() -> None: ... +def _get_parquet_symbol(name): ... +def _constructor_directory_partitioning_factory(*args): ... +def _constructor_hive_partitioning_factory(*args): ... +def _constructor_filename_partitioning_factory(*args): ... +def get_partition_keys(partition_expression: Expression) -> dict: ... +def _filesystemdataset_write( + data: Scanner, + base_dir: object, + basename_template: str, + filesystem: FileSystem, + partitioning: Partitioning, + preserve_order: bool, + file_options: FileWriteOptions, + max_partitions: int, + file_visitor: object, + existing_data_behavior: str, + max_open_files: int, + max_rows_per_file: int, + min_rows_per_group: int, + max_rows_per_group: int, + create_dir: bool, +) -> None: ... class Dataset(_Weakrefable): _scan_options: dict[Any, Any] - def __init__(self): - ... - - @property - def partition_expression(self) -> pyarrow.compute.Expression: - ... - - def replace_schema(self, schema: Schema) -> Dataset: - ... - - def get_fragments(self, filter: Expression | None=None) -> Iterator[Fragment]: - ... - - def _get_fragments(self, filter: Expression) -> Iterator[Fragment]: - ... - - def _scanner_options(self, options): - ... - - def scanner(self, columns: object | None=None, filter: object | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> Scanner: - ... - - def to_batches(self, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> Iterator[pyarrow.RecordBatch]: - ... - - def to_table(self, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> pyarrow.Table: - ... - - def take(self, indices: object, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> pyarrow.Table: - ... - - def head(self, num_rows: int, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> pyarrow.Table: - ... - - def count_rows(self, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> int: - ... - - @property - def schema(self) -> pyarrow.Schema: - ... - - def filter(self, expression: pyarrow.compute.Expression) -> Dataset: - ... - - def sort_by(self, sorting, **kwargs) -> InMemoryDataset: - ... - - def join(self, right_dataset, keys: str | list[str], right_keys: str | list[str] | None=None, join_type: str='left outer', left_suffix: str | None=None, right_suffix: str | None=None, coalesce_keys: bool=True, use_threads: bool=True) -> InMemoryDataset: - ... - - def join_asof(self, right_dataset, on: str, by: str | list[str], tolerance: int, right_on: str | list[str] | None=None, right_by: str | list[str] | None=None) -> InMemoryDataset: - ... + def __init__(self): ... + @property + def partition_expression(self) -> pyarrow.compute.Expression: ... + def replace_schema(self, schema: Schema) -> Dataset: ... + def get_fragments(self, filter: Expression | None = None) -> Iterator[Fragment]: ... + def _get_fragments(self, filter: Expression) -> Iterator[Fragment]: ... + def _scanner_options(self, options): ... + def scanner( + self, + columns: object | None = None, + filter: object | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Scanner: ... + def to_batches( + self, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Iterator[pyarrow.RecordBatch]: ... + def to_table( + self, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> pyarrow.Table: ... + def take( + self, + indices: object, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> pyarrow.Table: ... + def head( + self, + num_rows: int, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> pyarrow.Table: ... + def count_rows( + self, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> int: ... + @property + def schema(self) -> pyarrow.Schema: ... + def filter(self, expression: pyarrow.compute.Expression) -> Dataset: ... + def sort_by(self, sorting, **kwargs) -> InMemoryDataset: ... + def join( + self, + right_dataset, + keys: str | list[str], + right_keys: str | list[str] | None = None, + join_type: str = "left outer", + left_suffix: str | None = None, + right_suffix: str | None = None, + coalesce_keys: bool = True, + use_threads: bool = True, + ) -> InMemoryDataset: ... + def join_asof( + self, + right_dataset, + on: str, + by: str | list[str], + tolerance: int, + right_on: str | list[str] | None = None, + right_by: str | list[str] | None = None, + ) -> InMemoryDataset: ... class InMemoryDataset(Dataset): - - def __init__(self, source, schema: Schema | None=None): - ... + def __init__(self, source, schema: Schema | None = None): ... class UnionDataset(Dataset): - - def __init__(self, schema: Schema, children): - ... - - def __reduce__(self) -> tuple: - ... - + def __init__(self, schema: Schema, children): ... + def __reduce__(self) -> tuple: ... @property - def children(self) -> list: - ... + def children(self) -> list: ... class FileSystemDataset(Dataset): - - def __init__(self, fragments, schema: Schema, format: FileFormat, filesystem: FileSystem | None=None, root_partition=None): - ... - - @property - def filesystem(self) -> pyarrow.fs.FileSystem: - ... - - @property - def partitioning(self) -> None | Partitioning: - ... - - def __reduce__(self) -> tuple: - ... - + def __init__( + self, + fragments, + schema: Schema, + format: FileFormat, + filesystem: FileSystem | None = None, + root_partition=None, + ): ... + @property + def filesystem(self) -> pyarrow.fs.FileSystem: ... + @property + def partitioning(self) -> None | Partitioning: ... + def __reduce__(self) -> tuple: ... @classmethod - def from_paths(cls, paths: list[str], schema: pyarrow.Schema | None=None, format: FileFormat | None=None, filesystem: pyarrow.fs.FileSystem | None=None, partitions: list[pyarrow.compute.Expression] | None=None, root_partition: pyarrow.compute.Expression | None=None) -> FileSystemDataset: - ... - + def from_paths( + cls, + paths: list[str], + schema: pyarrow.Schema | None = None, + format: FileFormat | None = None, + filesystem: pyarrow.fs.FileSystem | None = None, + partitions: list[pyarrow.compute.Expression] | None = None, + root_partition: pyarrow.compute.Expression | None = None, + ) -> FileSystemDataset: ... @property - def files(self) -> list: - ... - + def files(self) -> list: ... @property - def format(self) -> FileFormat: - ... + def format(self) -> FileFormat: ... class FileWriteOptions(_Weakrefable): - - def __init__(self): - ... - + def __init__(self): ... @property - def format(self) -> FileFormat: - ... + def format(self) -> FileFormat: ... class FileFormat(_Weakrefable): - - def __init__(self): - ... - - def inspect(self, file: IO[Any] | os.PathLike[str] | str, filesystem=None) -> pyarrow.Schema: - ... - - def make_fragment(self, file: IO[Any] | os.PathLike[str] | str, filesystem=None, partition_expression: Expression | None=None, *, file_size: int | None=None) -> Fragment: - ... - - def make_write_options(self) -> FileWriteOptions: - ... - - @property - def default_extname(self) -> str: - ... - - @property - def default_fragment_scan_options(self) -> None: - ... - + def __init__(self): ... + def inspect( + self, file: IO[Any] | os.PathLike[str] | str, filesystem=None + ) -> pyarrow.Schema: ... + def make_fragment( + self, + file: IO[Any] | os.PathLike[str] | str, + filesystem=None, + partition_expression: Expression | None = None, + *, + file_size: int | None = None, + ) -> Fragment: ... + def make_write_options(self) -> FileWriteOptions: ... + @property + def default_extname(self) -> str: ... + @property + def default_fragment_scan_options(self) -> None: ... @default_fragment_scan_options.setter - def default_fragment_scan_options(self, options: FragmentScanOptions) -> None: - ... - - def __eq__(self, other): - ... + def default_fragment_scan_options(self, options: FragmentScanOptions) -> None: ... + def __eq__(self, other): ... class Fragment(_Weakrefable): - - def __init__(self): - ... - - @property - def physical_schema(self) -> pyarrow.Schema: - ... - - @property - def partition_expression(self) -> pyarrow.compute.Expression: - ... - - def scanner(self, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> Scanner: - ... - - def to_batches(self, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> Iterator[pyarrow.RecordBatch]: - ... - - def to_table(self, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> pyarrow.Table: - ... - - def take(self, indices: object, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> pyarrow.Table: - ... - - def head(self, num_rows: int, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> pyarrow.Table: - ... - - def count_rows(self, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> int: - ... + def __init__(self): ... + @property + def physical_schema(self) -> pyarrow.Schema: ... + @property + def partition_expression(self) -> pyarrow.compute.Expression: ... + def scanner( + self, + schema: Schema | None = None, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Scanner: ... + def to_batches( + self, + schema: Schema | None = None, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Iterator[pyarrow.RecordBatch]: ... + def to_table( + self, + schema: Schema | None = None, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> pyarrow.Table: ... + def take( + self, + indices: object, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> pyarrow.Table: ... + def head( + self, + num_rows: int, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> pyarrow.Table: ... + def count_rows( + self, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> int: ... class FileFragment(Fragment): - - def __repr__(self) -> str: - ... - - def __reduce__(self) -> tuple: - ... - - def open(self): - ... - + def __reduce__(self) -> tuple: ... + def open(self): ... @property - def path(self) -> str: - ... - + def path(self) -> str: ... @property - def filesystem(self) -> None | pyarrow.fs.FileSystem: - ... - + def filesystem(self) -> None | pyarrow.fs.FileSystem: ... @property - def buffer(self) -> None | pyarrow.Buffer: - ... - + def buffer(self) -> None | pyarrow.Buffer: ... @property - def format(self) -> FileFormat: - ... + def format(self) -> FileFormat: ... class FragmentScanOptions(_Weakrefable): - - def __init__(self): - ... - + def __init__(self): ... @property - def type_name(self) -> str: - ... - - def __eq__(self, other): - ... + def type_name(self) -> str: ... + def __eq__(self, other): ... class IpcFileWriteOptions(FileWriteOptions): - - def __init__(self): - ... - + def __init__(self): ... @property - def write_options(self) -> None: - ... - + def write_options(self) -> None: ... @write_options.setter - def write_options(self, write_options: IpcWriteOptions) -> None: - ... + def write_options(self, write_options: IpcWriteOptions) -> None: ... class IpcFileFormat(FileFormat): - - def __init__(self): - ... - - def equals(self, other: IpcFileFormat) -> bool: - ... - - def make_write_options(self, **kwargs) -> IpcFileWriteOptions: - ... - + def __init__(self): ... + def equals(self, other: IpcFileFormat) -> bool: ... + def make_write_options(self, **kwargs) -> IpcFileWriteOptions: ... @property - def default_extname(self) -> str: - ... - - def __reduce__(self) -> tuple: - ... + def default_extname(self) -> str: ... + def __reduce__(self) -> tuple: ... class FeatherFileFormat(IpcFileFormat): - @property - def default_extname(self) -> str: - ... + def default_extname(self) -> str: ... class CsvFileFormat(FileFormat): __slots__ = () _read_options_py: ReadOptions - def __init__(self, parse_options: ParseOptions | None=None, default_fragment_scan_options=None, convert_options: ConvertOptions | None=None, read_options: ReadOptions | None=None): - ... - - def make_write_options(self, **kwargs) -> CsvFileWriteOptions: - ... - + def __init__( + self, + parse_options: ParseOptions | None = None, + default_fragment_scan_options=None, + convert_options: ConvertOptions | None = None, + read_options: ReadOptions | None = None, + ): ... + def make_write_options(self, **kwargs) -> CsvFileWriteOptions: ... @property - def parse_options(self) -> None: - ... - + def parse_options(self) -> None: ... @parse_options.setter - def parse_options(self, parse_options: ParseOptions) -> None: - ... - - def equals(self, other: CsvFileFormat) -> bool: - ... - - def __reduce__(self) -> tuple: - ... - - def __repr__(self) -> str: - ... + def parse_options(self, parse_options: ParseOptions) -> None: ... + def equals(self, other: CsvFileFormat) -> bool: ... + def __reduce__(self) -> tuple: ... class CsvFragmentScanOptions(FragmentScanOptions): __slots__ = () - def __init__(self, convert_options: ConvertOptions | None=None, read_options: ReadOptions | None=None): - ... - + def __init__( + self, + convert_options: ConvertOptions | None = None, + read_options: ReadOptions | None = None, + ): ... @property - def convert_options(self) -> None: - ... - + def convert_options(self) -> None: ... @convert_options.setter - def convert_options(self, convert_options: ConvertOptions) -> None: - ... - + def convert_options(self, convert_options: ConvertOptions) -> None: ... @property - def read_options(self) -> None: - ... - + def read_options(self) -> None: ... @read_options.setter - def read_options(self, read_options: ReadOptions) -> None: - ... - - def equals(self, other: CsvFragmentScanOptions) -> bool: - ... - - def __reduce__(self) -> tuple: - ... + def read_options(self, read_options: ReadOptions) -> None: ... + def equals(self, other: CsvFragmentScanOptions) -> bool: ... + def __reduce__(self) -> tuple: ... class CsvFileWriteOptions(FileWriteOptions): - - def __init__(self): - ... - + def __init__(self): ... @property - def write_options(self) -> None: - ... - + def write_options(self) -> None: ... @write_options.setter - def write_options(self, write_options: WriteOptions) -> None: - ... + def write_options(self, write_options: WriteOptions) -> None: ... class JsonFileFormat(FileFormat): __slots__ = () - def __init__(self, default_fragment_scan_options=None, parse_options: JsonParseOptions | None=None, read_options: JsonReadOptions | None=None): - ... - - def equals(self, other: JsonFileFormat) -> bool: - ... - - def __reduce__(self) -> tuple: - ... - - def __repr__(self) -> str: - ... + def __init__( + self, + default_fragment_scan_options=None, + parse_options: JsonParseOptions | None = None, + read_options: JsonReadOptions | None = None, + ): ... + def equals(self, other: JsonFileFormat) -> bool: ... + def __reduce__(self) -> tuple: ... class JsonFragmentScanOptions(FragmentScanOptions): __slots__ = () - def __init__(self, parse_options: JsonParseOptions | None=None, read_options: JsonReadOptions | None=None): - ... - + def __init__( + self, + parse_options: JsonParseOptions | None = None, + read_options: JsonReadOptions | None = None, + ): ... @property - def parse_options(self) -> None: - ... - + def parse_options(self) -> None: ... @parse_options.setter - def parse_options(self, parse_options: JsonParseOptions) -> None: - ... - + def parse_options(self, parse_options: JsonParseOptions) -> None: ... @property - def read_options(self) -> None: - ... - + def read_options(self) -> None: ... @read_options.setter - def read_options(self, read_options: JsonReadOptions) -> None: - ... - - def equals(self, other: JsonFragmentScanOptions) -> bool: - ... - - def __reduce__(self) -> tuple: - ... + def read_options(self, read_options: JsonReadOptions) -> None: ... + def equals(self, other: JsonFragmentScanOptions) -> bool: ... + def __reduce__(self) -> tuple: ... class Partitioning(_Weakrefable): - - def __init__(self): - ... - - def __eq__(self, other): - ... - - def parse(self, path: str) -> pyarrow.compute.Expression: - ... - - def format(self, expr: pyarrow.dataset.Expression) -> tuple: - ... - + def __init__(self): ... + def __eq__(self, other): ... + def parse(self, path: str) -> pyarrow.compute.Expression: ... + def format(self, expr: pyarrow.dataset.Expression) -> tuple: ... @property - def schema(self) -> pyarrow.Schema: - ... + def schema(self) -> pyarrow.Schema: ... class PartitioningFactory(_Weakrefable): - - def __init__(self): - ... - - def __reduce__(self) -> tuple: - ... - + def __init__(self): ... + def __reduce__(self) -> tuple: ... @property - def type_name(self) -> str: - ... + def type_name(self) -> str: ... class KeyValuePartitioning(Partitioning): - - def __init__(self): - ... - - def __reduce__(self) -> tuple: - ... - + def __init__(self): ... + def __reduce__(self) -> tuple: ... @property - def dictionaries(self) -> list: - ... + def dictionaries(self) -> list: ... class DirectoryPartitioning(KeyValuePartitioning): - - def __init__(self, schema: Schema, dictionaries=None, segment_encoding: str='uri'): - ... - + def __init__( + self, schema: Schema, dictionaries=None, segment_encoding: str = "uri" + ): ... @staticmethod - def discover(field_names: list[str] | None=None, infer_dictionary: bool=False, max_partition_dictionary_size: int=0, schema: pyarrow.Schema | None=None, segment_encoding: str='uri') -> PartitioningFactory: - ... + def discover( + field_names: list[str] | None = None, + infer_dictionary: bool = False, + max_partition_dictionary_size: int = 0, + schema: pyarrow.Schema | None = None, + segment_encoding: str = "uri", + ) -> PartitioningFactory: ... class HivePartitioning(KeyValuePartitioning): - - def __init__(self, schema: Schema, dictionaries=None, null_fallback: str='__HIVE_DEFAULT_PARTITION__', segment_encoding: str='uri'): - ... - - def __reduce__(self) -> tuple: - ... - + def __init__( + self, + schema: Schema, + dictionaries=None, + null_fallback: str = "__HIVE_DEFAULT_PARTITION__", + segment_encoding: str = "uri", + ): ... + def __reduce__(self) -> tuple: ... @staticmethod - def discover(infer_dictionary: bool=False, max_partition_dictionary_size: int=0, null_fallback: str='__HIVE_DEFAULT_PARTITION__', schema: pyarrow.Schema | None=None, segment_encoding: str='uri') -> PartitioningFactory: - ... + def discover( + infer_dictionary: bool = False, + max_partition_dictionary_size: int = 0, + null_fallback: str = "__HIVE_DEFAULT_PARTITION__", + schema: pyarrow.Schema | None = None, + segment_encoding: str = "uri", + ) -> PartitioningFactory: ... class FilenamePartitioning(KeyValuePartitioning): - - def __init__(self, schema: Schema, dictionaries=None, segment_encoding: str='uri'): - ... - + def __init__( + self, schema: Schema, dictionaries=None, segment_encoding: str = "uri" + ): ... @staticmethod - def discover(field_names: list[str] | None=None, infer_dictionary: bool=False, schema: pyarrow.Schema | None=None, segment_encoding: str='uri') -> PartitioningFactory: - ... + def discover( + field_names: list[str] | None = None, + infer_dictionary: bool = False, + schema: pyarrow.Schema | None = None, + segment_encoding: str = "uri", + ) -> PartitioningFactory: ... class DatasetFactory(_Weakrefable): - - def __init__(self): - ... - + def __init__(self): ... @property - def root_partition(self) -> None: - ... - + def root_partition(self) -> None: ... @root_partition.setter - def root_partition(self, expr: Expression) -> None: - ... - - def inspect_schemas(self) -> list: - ... - - def inspect(self, *, promote_options: str='default', fragments: int | None=None) -> pyarrow.Schema: - ... - - def finish(self, schema: Schema | None=None) -> Dataset: - ... + def root_partition(self, expr: Expression) -> None: ... + def inspect_schemas(self) -> list: ... + def inspect( + self, *, promote_options: str = "default", fragments: int | None = None + ) -> pyarrow.Schema: ... + def finish(self, schema: Schema | None = None) -> Dataset: ... class FileSystemFactoryOptions(_Weakrefable): __slots__ = () - def __init__(self, partition_base_dir=None, partitioning=None, exclude_invalid_files=None, selector_ignore_prefixes: list[Any] | None=None): - ... - + def __init__( + self, + partition_base_dir=None, + partitioning=None, + exclude_invalid_files=None, + selector_ignore_prefixes: list[Any] | None = None, + ): ... @property - def partitioning(self) -> None: - ... - + def partitioning(self) -> None: ... @partitioning.setter - def partitioning(self, value: Partitioning) -> None: - ... - + def partitioning(self, value: Partitioning) -> None: ... @property - def partitioning_factory(self) -> None: - ... - + def partitioning_factory(self) -> None: ... @partitioning_factory.setter - def partitioning_factory(self, value: PartitioningFactory) -> None: - ... - + def partitioning_factory(self, value: PartitioningFactory) -> None: ... @property - def partition_base_dir(self) -> None: - ... - + def partition_base_dir(self) -> None: ... @partition_base_dir.setter - def partition_base_dir(self, value) -> None: - ... - + def partition_base_dir(self, value) -> None: ... @property - def exclude_invalid_files(self) -> None: - ... - + def exclude_invalid_files(self) -> None: ... @exclude_invalid_files.setter - def exclude_invalid_files(self, value: bool) -> None: - ... - + def exclude_invalid_files(self, value: bool) -> None: ... @property - def selector_ignore_prefixes(self) -> None: - ... - + def selector_ignore_prefixes(self) -> None: ... @selector_ignore_prefixes.setter - def selector_ignore_prefixes(self, values) -> None: - ... + def selector_ignore_prefixes(self, values) -> None: ... class FileSystemDatasetFactory(DatasetFactory): - - def __init__(self, filesystem: FileSystem, paths_or_selector, format: FileFormat, options: FileSystemFactoryOptions | None=None): - ... + def __init__( + self, + filesystem: FileSystem, + paths_or_selector, + format: FileFormat, + options: FileSystemFactoryOptions | None = None, + ): ... class UnionDatasetFactory(DatasetFactory): - - def __init__(self, factories: list[Any]): - ... + def __init__(self, factories: list[Any]): ... class RecordBatchIterator(_Weakrefable): - - def __init__(self): - ... - - def __iter__(self) -> Self: - ... - - def __next__(self) -> pyarrow.RecordBatch: - ... + def __init__(self): ... + def __iter__(self) -> Self: ... + def __next__(self) -> pyarrow.RecordBatch: ... class TaggedRecordBatch(NamedTuple): record_batch: Incomplete fragment: Incomplete - ... class TaggedRecordBatchIterator(_Weakrefable): - - def __init__(self): - ... - - def __iter__(self) -> Self: - ... - - def __next__(self) -> TaggedRecordBatch: - ... + def __init__(self): ... + def __iter__(self) -> Self: ... + def __next__(self) -> TaggedRecordBatch: ... class Scanner(_Weakrefable): - - def __init__(self): - ... - + def __init__(self): ... @staticmethod - def from_dataset(dataset: Dataset, *, columns: object | None=None, filter: object | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> Scanner: - ... - + def from_dataset( + dataset: Dataset, + *, + columns: object | None = None, + filter: object | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Scanner: ... @staticmethod - def from_fragment(fragment: Fragment, *, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> Scanner: - ... - + def from_fragment( + fragment: Fragment, + *, + schema: Schema | None = None, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Scanner: ... @staticmethod - def from_batches(source, *, schema: Schema | None=None, columns: object | None=None, filter: Expression | None=None, batch_size: int=_DEFAULT_BATCH_SIZE, batch_readahead: int=_DEFAULT_BATCH_READAHEAD, fragment_readahead: int=_DEFAULT_FRAGMENT_READAHEAD, fragment_scan_options: FragmentScanOptions | None=None, use_threads: bool=True, cache_metadata: bool=True, memory_pool: MemoryPool | None=None) -> Scanner: - ... - - @property - def dataset_schema(self) -> pyarrow.Schema: - ... - - @property - def projected_schema(self) -> pyarrow.Schema: - ... - - def to_batches(self) -> Iterator[pyarrow.RecordBatch]: - ... - - def scan_batches(self) -> TaggedRecordBatchIterator: - ... - - def to_table(self) -> pyarrow.Table: - ... - - def take(self, indices: object) -> pyarrow.Table: - ... - - def head(self, num_rows: int) -> pyarrow.Table: - ... - - def count_rows(self) -> int: - ... - - def to_reader(self) -> pyarrow.RecordBatchReader: - ... + def from_batches( + source, + *, + schema: Schema | None = None, + columns: object | None = None, + filter: Expression | None = None, + batch_size: int = ..., + batch_readahead: int = ..., + fragment_readahead: int = ..., + fragment_scan_options: FragmentScanOptions | None = None, + use_threads: bool = True, + cache_metadata: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Scanner: ... + @property + def dataset_schema(self) -> pyarrow.Schema: ... + @property + def projected_schema(self) -> pyarrow.Schema: ... + def to_batches(self) -> Iterator[pyarrow.RecordBatch]: ... + def scan_batches(self) -> TaggedRecordBatchIterator: ... + def to_table(self) -> pyarrow.Table: ... + def take(self, indices: object) -> pyarrow.Table: ... + def head(self, num_rows: int) -> pyarrow.Table: ... + def count_rows(self) -> int: ... + def to_reader(self) -> pyarrow.RecordBatchReader: ... class WrittenFile(_Weakrefable): path: str metadata: object size: int - def __init__(self, path, metadata, size): - ... + def __init__(self, path, metadata, size): ... class _ScanNodeOptions(ExecNodeOptions): - - def _set_options(self, dataset: Dataset, scan_options: dict[Any, Any]) -> None: - ... + def _set_options(self, dataset: Dataset, scan_options: dict[Any, Any]) -> None: ... class ScanNodeOptions(_ScanNodeOptions): - - def __init__(self, dataset: Dataset, **kwargs): - ... + def __init__(self, dataset: Dataset, **kwargs): ... diff --git a/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi b/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi index 04b47805112a..72242b17ec99 100644 --- a/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi +++ b/python/pyarrow-stubs/pyarrow/_dataset_orc.pyi @@ -16,25 +16,12 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_dataset_orc.pyx -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete from pyarrow._dataset import FileFormat from pyarrow.lib import * class OrcFileFormat(FileFormat): - - def __init__(self): - ... - - def equals(self, other: OrcFileFormat) -> bool: - ... - + def __init__(self): ... + def equals(self, other: OrcFileFormat) -> bool: ... @property - def default_extname(self) -> str: - ... - - def __reduce__(self) -> tuple: - ... + def default_extname(self) -> str: ... + def __reduce__(self) -> tuple: ... diff --git a/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi b/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi index bb433ddff5db..ec62a32f7643 100644 --- a/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi +++ b/python/pyarrow-stubs/pyarrow/_dataset_parquet.pyi @@ -17,97 +17,78 @@ # Generated by stubgen-pyx from python/pyarrow/_dataset_parquet.pyx import os -import re import pyarrow -from typing import Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from typing import IO +from collections.abc import Iterable import pyarrow.dataset import pyarrow.parquet from pyarrow.lib import _Weakrefable -from _typeshed import Incomplete from typing import Any from pyarrow._compute import Expression -from pyarrow._dataset import CacheOptions, DatasetFactory, FileFormat, FileFragment, FileWriteOptions, FragmentScanOptions, Partitioning, PartitioningFactory +from pyarrow._dataset import ( + CacheOptions, + DatasetFactory, + FileFormat, + FileFragment, + FileWriteOptions, + FragmentScanOptions, + Partitioning, + PartitioningFactory, +) from pyarrow._fs import FileSystem from pyarrow.lib import * -from typing_extensions import TypeAlias + parquet_encryption_enabled: bool class ParquetFileFormat(FileFormat): - - def __init__(self, read_options=None, default_fragment_scan_options=None, **kwargs): - ... - - @property - def read_options(self) -> ParquetReadOptions: - ... - - def make_write_options(self, **kwargs) -> pyarrow.dataset.FileWriteOptions: - ... - - def equals(self, other: ParquetFileFormat) -> bool: - ... - - @property - def default_extname(self) -> str: - ... - - def __reduce__(self) -> tuple: - ... - - def __repr__(self) -> str: - ... - - def make_fragment(self, file: IO[Any] | os.PathLike[str] | str, filesystem=None, partition_expression: Expression | None=None, row_groups: Iterable[Any] | None=None, *, file_size: int | None=None) -> pyarrow.dataset.Fragment: - ... + def __init__( + self, read_options=None, default_fragment_scan_options=None, **kwargs + ): ... + @property + def read_options(self) -> ParquetReadOptions: ... + def make_write_options(self, **kwargs) -> pyarrow.dataset.FileWriteOptions: ... + def equals(self, other: ParquetFileFormat) -> bool: ... + @property + def default_extname(self) -> str: ... + def __reduce__(self) -> tuple: ... + def make_fragment( + self, + file: IO[Any] | os.PathLike[str] | str, + filesystem=None, + partition_expression: Expression | None = None, + row_groups: Iterable[Any] | None = None, + *, + file_size: int | None = None, + ) -> pyarrow.dataset.Fragment: ... class RowGroupInfo: - - def __init__(self, id, metadata, schema): - ... - + def __init__(self, id, metadata, schema): ... @property - def num_rows(self): - ... - + def num_rows(self): ... @property - def total_byte_size(self): - ... - + def total_byte_size(self): ... @property - def statistics(self) -> dict: - ... - - def __repr__(self) -> str: - ... - - def __eq__(self, other) -> bool: - ... + def statistics(self) -> dict: ... + def __eq__(self, other) -> bool: ... class ParquetFileFragment(FileFragment): - - def __reduce__(self) -> tuple: - ... - - def ensure_complete_metadata(self) -> None: - ... - + def __reduce__(self) -> tuple: ... + def ensure_complete_metadata(self) -> None: ... @property - def row_groups(self) -> list: - ... - + def row_groups(self) -> list: ... @property - def metadata(self) -> pyarrow.parquet.FileMetaData: - ... - + def metadata(self) -> pyarrow.parquet.FileMetaData: ... @property - def num_row_groups(self): - ... - - def split_by_row_group(self, filter: Expression | None=None, schema: Schema | None=None) -> list: - ... - - def subset(self, filter: Expression | None=None, schema: Schema | None=None, row_group_ids: object | None=None) -> pyarrow.dataset.Fragment: - ... + def num_row_groups(self): ... + def split_by_row_group( + self, filter: Expression | None = None, schema: Schema | None = None + ) -> list: ... + def subset( + self, + filter: Expression | None = None, + schema: Schema | None = None, + row_group_ids: object | None = None, + ) -> pyarrow.dataset.Fragment: ... class ParquetReadOptions(_Weakrefable): dictionary_columns: set[Any] @@ -115,202 +96,132 @@ class ParquetReadOptions(_Weakrefable): _binary_type: int _list_type: int - def __init__(self, dictionary_columns=None, coerce_int96_timestamp_unit=None, binary_type=None, list_type=None): - ... - + def __init__( + self, + dictionary_columns=None, + coerce_int96_timestamp_unit=None, + binary_type=None, + list_type=None, + ): ... @property - def binary_type(self) -> None: - ... - + def binary_type(self) -> None: ... @binary_type.setter - def binary_type(self, ty) -> None: - ... - + def binary_type(self, ty) -> None: ... @property - def list_type(self) -> None: - ... - + def list_type(self) -> None: ... @list_type.setter - def list_type(self, ty) -> None: - ... - + def list_type(self, ty) -> None: ... @property - def coerce_int96_timestamp_unit(self) -> None: - ... - + def coerce_int96_timestamp_unit(self) -> None: ... @coerce_int96_timestamp_unit.setter - def coerce_int96_timestamp_unit(self, unit) -> None: - ... - - def equals(self, other: ParquetReadOptions) -> bool: - ... - - def __eq__(self, other): - ... - - def __repr__(self) -> str: - ... + def coerce_int96_timestamp_unit(self, unit) -> None: ... + def equals(self, other: ParquetReadOptions) -> bool: ... + def __eq__(self, other): ... class ParquetFileWriteOptions(FileWriteOptions): - - def update(self, **kwargs) -> None: - ... - - def _set_properties(self) -> None: - ... - - def _set_arrow_properties(self) -> None: - ... - - def _set_encryption_config(self) -> None: - ... - - def __repr__(self) -> str: - ... + def update(self, **kwargs) -> None: ... + def _set_properties(self) -> None: ... + def _set_arrow_properties(self) -> None: ... + def _set_encryption_config(self) -> None: ... class ParquetFragmentScanOptions(FragmentScanOptions): __slots__ = () - def __init__(self, *, use_buffered_stream: bool=False, buffer_size: int=8192, pre_buffer: bool=True, cache_options=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, decryption_config=None, decryption_properties=None, page_checksum_verification: bool=False, arrow_extensions_enabled: bool=False): - ... - - @property - def use_buffered_stream(self) -> None: - ... - + def __init__( + self, + *, + use_buffered_stream: bool = False, + buffer_size: int = 8192, + pre_buffer: bool = True, + cache_options=None, + thrift_string_size_limit=None, + thrift_container_size_limit=None, + schema_depth_limit=None, + decryption_config=None, + decryption_properties=None, + page_checksum_verification: bool = False, + arrow_extensions_enabled: bool = False, + ): ... + @property + def use_buffered_stream(self) -> None: ... @use_buffered_stream.setter - def use_buffered_stream(self, use_buffered_stream: bool) -> None: - ... - + def use_buffered_stream(self, use_buffered_stream: bool) -> None: ... @property - def buffer_size(self) -> None: - ... - + def buffer_size(self) -> None: ... @buffer_size.setter - def buffer_size(self, buffer_size) -> None: - ... - + def buffer_size(self, buffer_size) -> None: ... @property - def pre_buffer(self) -> None: - ... - + def pre_buffer(self) -> None: ... @pre_buffer.setter - def pre_buffer(self, pre_buffer: bool) -> None: - ... - + def pre_buffer(self, pre_buffer: bool) -> None: ... @property - def cache_options(self) -> None: - ... - + def cache_options(self) -> None: ... @cache_options.setter - def cache_options(self, options: CacheOptions) -> None: - ... - + def cache_options(self, options: CacheOptions) -> None: ... @property - def thrift_string_size_limit(self) -> None: - ... - + def thrift_string_size_limit(self) -> None: ... @thrift_string_size_limit.setter - def thrift_string_size_limit(self, size) -> None: - ... - + def thrift_string_size_limit(self, size) -> None: ... @property - def thrift_container_size_limit(self) -> None: - ... - + def thrift_container_size_limit(self) -> None: ... @thrift_container_size_limit.setter - def thrift_container_size_limit(self, size) -> None: - ... - + def thrift_container_size_limit(self, size) -> None: ... @property - def schema_depth_limit(self) -> None: - ... - + def schema_depth_limit(self) -> None: ... @schema_depth_limit.setter - def schema_depth_limit(self, limit) -> None: - ... - + def schema_depth_limit(self, limit) -> None: ... @property - def decryption_properties(self) -> None: - ... - + def decryption_properties(self) -> None: ... @decryption_properties.setter - def decryption_properties(self, config) -> None: - ... - + def decryption_properties(self, config) -> None: ... @property - def parquet_decryption_config(self) -> None: - ... - + def parquet_decryption_config(self) -> None: ... @parquet_decryption_config.setter - def parquet_decryption_config(self, config) -> None: - ... - + def parquet_decryption_config(self, config) -> None: ... @property - def page_checksum_verification(self) -> None: - ... - + def page_checksum_verification(self) -> None: ... @page_checksum_verification.setter - def page_checksum_verification(self, page_checksum_verification: bool) -> None: - ... - + def page_checksum_verification(self, page_checksum_verification: bool) -> None: ... @property - def arrow_extensions_enabled(self) -> None: - ... - + def arrow_extensions_enabled(self) -> None: ... @arrow_extensions_enabled.setter - def arrow_extensions_enabled(self, arrow_extensions_enabled: bool) -> None: - ... - - def equals(self, other: ParquetFragmentScanOptions) -> bool: - ... - + def arrow_extensions_enabled(self, arrow_extensions_enabled: bool) -> None: ... + def equals(self, other: ParquetFragmentScanOptions) -> bool: ... @staticmethod - def _reconstruct(kwargs) -> ParquetFragmentScanOptions: - ... - - def __reduce__(self) -> tuple: - ... + def _reconstruct(kwargs) -> ParquetFragmentScanOptions: ... + def __reduce__(self) -> tuple: ... class ParquetFactoryOptions(_Weakrefable): __slots__ = () - def __init__(self, partition_base_dir=None, partitioning=None, validate_column_chunk_paths: bool=False): - ... - + def __init__( + self, + partition_base_dir=None, + partitioning=None, + validate_column_chunk_paths: bool = False, + ): ... @property - def partitioning(self) -> None: - ... - + def partitioning(self) -> None: ... @partitioning.setter - def partitioning(self, value: Partitioning) -> None: - ... - + def partitioning(self, value: Partitioning) -> None: ... @property - def partitioning_factory(self) -> None: - ... - + def partitioning_factory(self) -> None: ... @partitioning_factory.setter - def partitioning_factory(self, value: PartitioningFactory) -> None: - ... - + def partitioning_factory(self, value: PartitioningFactory) -> None: ... @property - def partition_base_dir(self) -> None: - ... - + def partition_base_dir(self) -> None: ... @partition_base_dir.setter - def partition_base_dir(self, value) -> None: - ... - + def partition_base_dir(self, value) -> None: ... @property - def validate_column_chunk_paths(self) -> None: - ... - + def validate_column_chunk_paths(self) -> None: ... @validate_column_chunk_paths.setter - def validate_column_chunk_paths(self, value) -> None: - ... + def validate_column_chunk_paths(self, value) -> None: ... class ParquetDatasetFactory(DatasetFactory): - - def __init__(self, metadata_path, filesystem: FileSystem, format: FileFormat, options: ParquetFactoryOptions | None=None): - ... + def __init__( + self, + metadata_path, + filesystem: FileSystem, + format: FileFormat, + options: ParquetFactoryOptions | None = None, + ): ... diff --git a/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi b/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi index ad4c42b7492c..315a86b82c15 100644 --- a/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi +++ b/python/pyarrow-stubs/pyarrow/_dataset_parquet_encryption.pyi @@ -17,32 +17,37 @@ # Generated by stubgen-pyx from python/pyarrow/_dataset_parquet_encryption.pyx import pyarrow.parquet -import os -import re import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence from pyarrow.lib import _Weakrefable -from _typeshed import Incomplete from pyarrow._dataset_parquet import ParquetFileWriteOptions, ParquetFragmentScanOptions from pyarrow._parquet_encryption import * -def set_encryption_config(opts: ParquetFileWriteOptions, config: ParquetEncryptionConfig) -> None: - ... - -def set_decryption_properties(opts: ParquetFragmentScanOptions, config: pyarrow.parquet.FileDecryptionProperties) -> None: - ... - -def set_decryption_config(opts: ParquetFragmentScanOptions, config: ParquetDecryptionConfig) -> None: - ... +def set_encryption_config( + opts: ParquetFileWriteOptions, config: ParquetEncryptionConfig +) -> None: ... +def set_decryption_properties( + opts: ParquetFragmentScanOptions, config: pyarrow.parquet.FileDecryptionProperties +) -> None: ... +def set_decryption_config( + opts: ParquetFragmentScanOptions, config: ParquetDecryptionConfig +) -> None: ... class ParquetEncryptionConfig(_Weakrefable): __slots__ = () - def __init__(self, crypto_factory: CryptoFactory, kms_connection_config: KmsConnectionConfig, encryption_config: EncryptionConfiguration): - ... + def __init__( + self, + crypto_factory: CryptoFactory, + kms_connection_config: KmsConnectionConfig, + encryption_config: EncryptionConfiguration, + ): ... class ParquetDecryptionConfig(_Weakrefable): __slots__ = () - def __init__(self, crypto_factory: CryptoFactory, kms_connection_config: KmsConnectionConfig, decryption_config: DecryptionConfiguration): - ... + def __init__( + self, + crypto_factory: CryptoFactory, + kms_connection_config: KmsConnectionConfig, + decryption_config: DecryptionConfiguration, + ): ... diff --git a/python/pyarrow-stubs/pyarrow/_feather.pyi b/python/pyarrow-stubs/pyarrow/_feather.pyi index 9fd838070f61..8b2785320ae6 100644 --- a/python/pyarrow-stubs/pyarrow/_feather.pyi +++ b/python/pyarrow-stubs/pyarrow/_feather.pyi @@ -16,33 +16,24 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_feather.pyx -import os -import re import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete from pyarrow.lib import Table, _Weakrefable -def write_feather(table: Table, dest: object, compression=None, compression_level=None, chunksize=None, version: int=2) -> None: - ... +def write_feather( + table: Table, + dest: object, + compression=None, + compression_level=None, + chunksize=None, + version: int = 2, +) -> None: ... -class FeatherError(Exception): - ... +class FeatherError(Exception): ... class FeatherReader(_Weakrefable): - - def __init__(self, source, use_memory_map: bool, use_threads: bool): - ... - + def __init__(self, source, use_memory_map: bool, use_threads: bool): ... @property - def version(self): - ... - - def read(self) -> pyarrow.Table: - ... - - def read_indices(self, indices) -> pyarrow.Table: - ... - - def read_names(self, names) -> pyarrow.Table: - ... + def version(self): ... + def read(self) -> pyarrow.Table: ... + def read_indices(self, indices) -> pyarrow.Table: ... + def read_names(self, names) -> pyarrow.Table: ... diff --git a/python/pyarrow-stubs/pyarrow/_flight.pyi b/python/pyarrow-stubs/pyarrow/_flight.pyi index e4bc8a887400..a1cc899886de 100644 --- a/python/pyarrow-stubs/pyarrow/_flight.pyi +++ b/python/pyarrow-stubs/pyarrow/_flight.pyi @@ -16,26 +16,27 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_flight.pyx -import os import pyarrow -from typing import Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from typing import NamedTuple, Self +from collections.abc import Iterable, Iterator import pyarrow.ipc from pyarrow.lib import _CRecordBatchWriter, _Weakrefable from _typeshed import Incomplete -import collections import enum import re from typing import Any from pyarrow.ipc import _ReadPandasMixin as _ReadPandasMixin from pyarrow.lib import * -from pyarrow.lib import ArrowCancelled as ArrowCancelled, ArrowException as ArrowException, ArrowInvalid as ArrowInvalid -_FLIGHT_SERVER_ERROR_REGEX: re.Pattern[str] +from pyarrow.lib import ( + ArrowCancelled as ArrowCancelled, + ArrowException as ArrowException, + ArrowInvalid as ArrowInvalid, +) -def _munge_grpc_python_error(message): - ... +_FLIGHT_SERVER_ERROR_REGEX: re.Pattern[str] -def connect(location: str | tuple | Location, **kwargs) -> FlightClient: - ... +def _munge_grpc_python_error(message): ... +def connect(location: str | tuple | Location, **kwargs) -> FlightClient: ... class _CertKeyPair(NamedTuple): cert: Incomplete @@ -49,141 +50,71 @@ class _CallInfo(NamedTuple): method: Incomplete class FlightCallOptions(_Weakrefable): - - def __init__(self, timeout: float | None=None, write_options: pyarrow.ipc.IpcWriteOptions | None=None, headers=None, read_options: IpcReadOptions | None=None): - ... - + def __init__( + self, + timeout: float | None = None, + write_options: pyarrow.ipc.IpcWriteOptions | None = None, + headers=None, + read_options: IpcReadOptions | None = None, + ): ... @property - def timeout(self): - ... - + def timeout(self): ... @property - def headers(self): - ... - + def headers(self): ... @property - def read_options(self): - ... - + def read_options(self): ... @property - def write_options(self): - ... - - def __repr__(self) -> str: - ... + def write_options(self): ... -class CertKeyPair(_CertKeyPair): - ... +class CertKeyPair(_CertKeyPair): ... class FlightError(Exception): + def __init__(self, message: str = "", extra_info: bytes = b""): ... - def __init__(self, message: str='', extra_info: bytes=b''): - ... - -class FlightInternalError(FlightError, ArrowException): - ... - -class FlightTimedOutError(FlightError, ArrowException): - ... - -class FlightCancelledError(FlightError, ArrowCancelled): - ... - -class FlightServerError(FlightError, ArrowException): - ... - -class FlightUnauthenticatedError(FlightError, ArrowException): - ... - -class FlightUnauthorizedError(FlightError, ArrowException): - ... - -class FlightUnavailableError(FlightError, ArrowException): - ... +class FlightInternalError(FlightError, ArrowException): ... +class FlightTimedOutError(FlightError, ArrowException): ... +class FlightCancelledError(FlightError, ArrowCancelled): ... +class FlightServerError(FlightError, ArrowException): ... +class FlightUnauthenticatedError(FlightError, ArrowException): ... +class FlightUnauthorizedError(FlightError, ArrowException): ... +class FlightUnavailableError(FlightError, ArrowException): ... class FlightWriteSizeExceededError(ArrowInvalid): - - def __init__(self, message, limit, actual): - ... + def __init__(self, message, limit, actual): ... class Action(_Weakrefable): - - def __init__(self, action_type: bytes | str, buf): - ... - + def __init__(self, action_type: bytes | str, buf): ... @property - def type(self) -> str: - ... - + def type(self) -> str: ... @property - def body(self) -> pyarrow.Buffer: - ... - - def serialize(self): - ... - + def body(self) -> pyarrow.Buffer: ... + def serialize(self): ... @classmethod - def deserialize(cls, serialized) -> Action: - ... - - def __eq__(self, other: Action) -> bool: - ... - - def __repr__(self) -> str: - ... + def deserialize(cls, serialized) -> Action: ... + def __eq__(self, other: Action) -> bool: ... class ActionType(_ActionType): - - def make_action(self, buf) -> Action: - ... + def make_action(self, buf) -> Action: ... class Result(_Weakrefable): - - def __init__(self, buf): - ... - + def __init__(self, buf): ... @property - def body(self) -> pyarrow.Buffer: - ... - - def serialize(self): - ... - + def body(self) -> pyarrow.Buffer: ... + def serialize(self): ... @classmethod - def deserialize(cls, serialized) -> Result: - ... - - def __eq__(self, other: Result) -> bool: - ... - - def __repr__(self) -> str: - ... + def deserialize(cls, serialized) -> Result: ... + def __eq__(self, other: Result) -> bool: ... class BasicAuth(_Weakrefable): - - def __init__(self, username: str | None=None, password: str | None=None): - ... - + def __init__(self, username: str | None = None, password: str | None = None): ... @property - def username(self): - ... - + def username(self): ... @property - def password(self): - ... - + def password(self): ... @staticmethod - def deserialize(serialized) -> BasicAuth: - ... - - def serialize(self): - ... - - def __eq__(self, other: BasicAuth) -> bool: - ... - - def __repr__(self) -> str: - ... + def deserialize(serialized) -> BasicAuth: ... + def serialize(self): ... + def __eq__(self, other: BasicAuth) -> bool: ... class DescriptorType(enum.Enum): UNKNOWN = 0 @@ -203,546 +134,351 @@ class FlightMethod(enum.Enum): DO_EXCHANGE = 9 class FlightDescriptor(_Weakrefable): - - def __init__(self): - ... - + def __init__(self): ... @staticmethod - def for_path(*path) -> FlightDescriptor: - ... - + def for_path(*path) -> FlightDescriptor: ... @staticmethod - def for_command(command) -> FlightDescriptor: - ... - + def for_command(command) -> FlightDescriptor: ... @property - def descriptor_type(self): - ... - + def descriptor_type(self): ... @property - def command(self): - ... - + def command(self): ... @property - def path(self): - ... - - def __repr__(self) -> str: - ... - - def serialize(self): - ... - + def path(self): ... + def serialize(self): ... @classmethod - def deserialize(cls, serialized) -> FlightDescriptor: - ... - - def __eq__(self, other: FlightDescriptor) -> bool: - ... + def deserialize(cls, serialized) -> FlightDescriptor: ... + def __eq__(self, other: FlightDescriptor) -> bool: ... class Ticket(_Weakrefable): - - def __init__(self, ticket): - ... - + def __init__(self, ticket): ... @property - def ticket(self): - ... - - def serialize(self): - ... - + def ticket(self): ... + def serialize(self): ... @classmethod - def deserialize(cls, serialized) -> Ticket: - ... - - def __eq__(self, other: Ticket) -> bool: - ... - - def __repr__(self) -> str: - ... + def deserialize(cls, serialized) -> Ticket: ... + def __eq__(self, other: Ticket) -> bool: ... class Location(_Weakrefable): - - def __init__(self, uri): - ... - - def __repr__(self) -> str: - ... - + def __init__(self, uri): ... @property - def uri(self): - ... - - def equals(self, other: Location) -> bool: - ... - - def __eq__(self, other): - ... - + def uri(self): ... + def equals(self, other: Location) -> bool: ... + def __eq__(self, other): ... @staticmethod - def for_grpc_tcp(host, port) -> Location: - ... - + def for_grpc_tcp(host, port) -> Location: ... @staticmethod - def for_grpc_tls(host, port) -> Location: - ... - + def for_grpc_tls(host, port) -> Location: ... @staticmethod - def for_grpc_unix(path) -> Location: - ... + def for_grpc_unix(path) -> Location: ... class FlightEndpoint(_Weakrefable): - - def __init__(self, ticket: Ticket | bytes, locations, expiration_time: pyarrow.TimestampScalar | None=None, app_metadata: bytes | str=''): - ... - + def __init__( + self, + ticket: Ticket | bytes, + locations, + expiration_time: pyarrow.TimestampScalar | None = None, + app_metadata: bytes | str = "", + ): ... @property - def ticket(self) -> Ticket: - ... - + def ticket(self) -> Ticket: ... @property - def locations(self) -> list: - ... - + def locations(self) -> list: ... @property - def expiration_time(self): - ... - + def expiration_time(self): ... @property - def app_metadata(self): - ... - - def serialize(self): - ... - + def app_metadata(self): ... + def serialize(self): ... @classmethod - def deserialize(cls, serialized) -> FlightEndpoint: - ... - - def __repr__(self) -> str: - ... - - def __eq__(self, other: FlightEndpoint) -> bool: - ... + def deserialize(cls, serialized) -> FlightEndpoint: ... + def __eq__(self, other: FlightEndpoint) -> bool: ... class SchemaResult(_Weakrefable): - - def __init__(self, schema: Schema): - ... - + def __init__(self, schema: Schema): ... @property - def schema(self) -> pyarrow.Schema: - ... - - def serialize(self): - ... - + def schema(self) -> pyarrow.Schema: ... + def serialize(self): ... @classmethod - def deserialize(cls, serialized) -> SchemaResult: - ... - - def __eq__(self, other: SchemaResult) -> bool: - ... - - def __repr__(self) -> str: - ... + def deserialize(cls, serialized) -> SchemaResult: ... + def __eq__(self, other: SchemaResult) -> bool: ... class FlightInfo(_Weakrefable): - - def __init__(self, schema: Schema, descriptor: FlightDescriptor, endpoints: list[FlightEndpoint], total_records: int | None=None, total_bytes: int | None=None, ordered: bool=False, app_metadata: bytes | str=''): - ... - + def __init__( + self, + schema: Schema, + descriptor: FlightDescriptor, + endpoints: list[FlightEndpoint], + total_records: int | None = None, + total_bytes: int | None = None, + ordered: bool = False, + app_metadata: bytes | str = "", + ): ... @property - def total_records(self): - ... - + def total_records(self): ... @property - def total_bytes(self): - ... - + def total_bytes(self): ... @property - def ordered(self): - ... - + def ordered(self): ... @property - def app_metadata(self): - ... - + def app_metadata(self): ... @property - def schema(self) -> None | pyarrow.Schema: - ... - + def schema(self) -> None | pyarrow.Schema: ... @property - def descriptor(self) -> FlightDescriptor: - ... - + def descriptor(self) -> FlightDescriptor: ... @property - def endpoints(self) -> list: - ... - - def serialize(self): - ... - + def endpoints(self) -> list: ... + def serialize(self): ... @classmethod - def deserialize(cls, serialized) -> FlightInfo: - ... - - def __eq__(self, other: FlightInfo) -> bool: - ... - - def __repr__(self) -> str: - ... + def deserialize(cls, serialized) -> FlightInfo: ... + def __eq__(self, other: FlightInfo) -> bool: ... class FlightStreamChunk(_Weakrefable): - @property - def data(self) -> None | pyarrow.RecordBatch: - ... - + def data(self) -> None | pyarrow.RecordBatch: ... @property - def app_metadata(self) -> None | pyarrow.Buffer: - ... - - def __iter__(self): - ... - - def __repr__(self) -> str: - ... + def app_metadata(self) -> None | pyarrow.Buffer: ... + def __iter__(self): ... class _MetadataRecordBatchReader(_Weakrefable, _ReadPandasMixin): - - def __iter__(self) -> Self: - ... - - def __next__(self): - ... - + def __iter__(self) -> Self: ... + def __next__(self): ... @property - def schema(self) -> pyarrow.Schema: - ... - - def read_all(self) -> pyarrow.Table: - ... - - def read_chunk(self) -> FlightStreamChunk: - ... - - def to_reader(self) -> pyarrow.RecordBatchReader: - ... - + def schema(self) -> pyarrow.Schema: ... + def read_all(self) -> pyarrow.Table: ... + def read_chunk(self) -> FlightStreamChunk: ... + def to_reader(self) -> pyarrow.RecordBatchReader: ... @property - def stats(self) -> pyarrow.ipc.ReadStats: - ... + def stats(self) -> pyarrow.ipc.ReadStats: ... -class MetadataRecordBatchReader(_MetadataRecordBatchReader): - ... +class MetadataRecordBatchReader(_MetadataRecordBatchReader): ... class FlightStreamReader(MetadataRecordBatchReader): - - def cancel(self) -> None: - ... - - def read_all(self) -> pyarrow.Table: - ... + def cancel(self) -> None: ... + def read_all(self) -> pyarrow.Table: ... class MetadataRecordBatchWriter(_CRecordBatchWriter): - - def begin(self, schema: Schema, options=None) -> None: - ... - - def write_metadata(self, buf) -> None: - ... - - def write_batch(self, batch: RecordBatch) -> None: - ... - - def write_table(self, table: Table, max_chunksize: int | None=None, **kwargs) -> None: - ... - - def close(self) -> None: - ... - - def write_with_metadata(self, batch: RecordBatch, buf: pyarrow.Buffer) -> None: - ... + def begin(self, schema: Schema, options=None) -> None: ... + def write_metadata(self, buf) -> None: ... + def write_batch(self, batch: RecordBatch) -> None: ... + def write_table( + self, table: Table, max_chunksize: int | None = None, **kwargs + ) -> None: ... + def close(self) -> None: ... + def write_with_metadata(self, batch: RecordBatch, buf: pyarrow.Buffer) -> None: ... class FlightStreamWriter(MetadataRecordBatchWriter): - - def done_writing(self) -> None: - ... + def done_writing(self) -> None: ... class FlightMetadataReader(_Weakrefable): - - def read(self) -> None | pyarrow.Buffer: - ... + def read(self) -> None | pyarrow.Buffer: ... class FlightMetadataWriter(_Weakrefable): - - def write(self, message: pyarrow.Buffer) -> None: - ... + def write(self, message: pyarrow.Buffer) -> None: ... class AsyncioCall: - - def __init__(self) -> None: - ... - - def as_awaitable(self) -> object: - ... - - def wakeup(self, result_or_exception) -> None: - ... + def __init__(self) -> None: ... + def as_awaitable(self) -> object: ... + def wakeup(self, result_or_exception) -> None: ... class AsyncioFlightClient: - - def __init__(self, client: FlightClient) -> None: - ... - - async def get_flight_info(self, descriptor: FlightDescriptor, *, options: FlightCallOptions | None=None): - ... + def __init__(self, client: FlightClient) -> None: ... + async def get_flight_info( + self, descriptor: FlightDescriptor, *, options: FlightCallOptions | None = None + ): ... class FlightClient(_Weakrefable): - - def __init__(self, location, *, tls_root_certs=None, cert_chain=None, private_key=None, override_hostname=None, middleware=None, write_size_limit_bytes=None, disable_server_verification=None, generic_options=None): - ... - - @property - def supports_async(self): - ... - - def as_async(self) -> None: - ... - - def wait_for_available(self, timeout: int=5) -> None: - ... - + def __init__( + self, + location, + *, + tls_root_certs=None, + cert_chain=None, + private_key=None, + override_hostname=None, + middleware=None, + write_size_limit_bytes=None, + disable_server_verification=None, + generic_options=None, + ): ... + @property + def supports_async(self): ... + def as_async(self) -> None: ... + def wait_for_available(self, timeout: int = 5) -> None: ... @classmethod - def connect(cls, location, tls_root_certs=None, cert_chain=None, private_key=None, override_hostname=None, disable_server_verification=None) -> FlightClient: - ... - - def authenticate(self, auth_handler: ClientAuthHandler, options: FlightCallOptions | None=None) -> None: - ... - - def authenticate_basic_token(self, username: str, password: str, options: FlightCallOptions | None=None): - ... - - def list_actions(self, options: FlightCallOptions | None=None) -> list: - ... - - def do_action(self, action: str | tuple | Action, options: FlightCallOptions | None=None): - ... - - def list_flights(self, criteria: bytes | None=None, options: FlightCallOptions | None=None) -> Iterator[FlightInfo]: - ... - - def get_flight_info(self, descriptor: FlightDescriptor, options: FlightCallOptions | None=None) -> FlightInfo: - ... - - def get_schema(self, descriptor: FlightDescriptor, options: FlightCallOptions | None=None) -> SchemaResult: - ... - - def do_get(self, ticket: Ticket, options: FlightCallOptions | None=None) -> FlightStreamReader: - ... - - def do_put(self, descriptor: FlightDescriptor, schema: Schema, options: FlightCallOptions | None=None) -> tuple: - ... - - def do_exchange(self, descriptor: FlightDescriptor, options: FlightCallOptions | None=None) -> tuple: - ... - - def close(self) -> None: - ... - - def __del__(self) -> None: - ... - - def __enter__(self) -> Self: - ... - - def __exit__(self, exc_type, exc_value, traceback) -> None: - ... - -class FlightDataStream(_Weakrefable): - ... + def connect( + cls, + location, + tls_root_certs=None, + cert_chain=None, + private_key=None, + override_hostname=None, + disable_server_verification=None, + ) -> FlightClient: ... + def authenticate( + self, auth_handler: ClientAuthHandler, options: FlightCallOptions | None = None + ) -> None: ... + def authenticate_basic_token( + self, username: str, password: str, options: FlightCallOptions | None = None + ): ... + def list_actions(self, options: FlightCallOptions | None = None) -> list: ... + def do_action( + self, action: str | tuple | Action, options: FlightCallOptions | None = None + ): ... + def list_flights( + self, criteria: bytes | None = None, options: FlightCallOptions | None = None + ) -> Iterator[FlightInfo]: ... + def get_flight_info( + self, descriptor: FlightDescriptor, options: FlightCallOptions | None = None + ) -> FlightInfo: ... + def get_schema( + self, descriptor: FlightDescriptor, options: FlightCallOptions | None = None + ) -> SchemaResult: ... + def do_get( + self, ticket: Ticket, options: FlightCallOptions | None = None + ) -> FlightStreamReader: ... + def do_put( + self, + descriptor: FlightDescriptor, + schema: Schema, + options: FlightCallOptions | None = None, + ) -> tuple: ... + def do_exchange( + self, descriptor: FlightDescriptor, options: FlightCallOptions | None = None + ) -> tuple: ... + def close(self) -> None: ... + def __del__(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type, exc_value, traceback) -> None: ... + +class FlightDataStream(_Weakrefable): ... class RecordBatchStream(FlightDataStream): - - def __init__(self, data_source: pyarrow.RecordBatchReader | pyarrow.Table, options: pyarrow.ipc.IpcWriteOptions | None=None): - ... + def __init__( + self, + data_source: pyarrow.RecordBatchReader | pyarrow.Table, + options: pyarrow.ipc.IpcWriteOptions | None = None, + ): ... class GeneratorStream(FlightDataStream): - - def __init__(self, schema: pyarrow.Schema, generator: Iterator[Any] | Iterable[Any], options: pyarrow.ipc.IpcWriteOptions | None=None): - ... + def __init__( + self, + schema: pyarrow.Schema, + generator: Iterator[Any] | Iterable[Any], + options: pyarrow.ipc.IpcWriteOptions | None = None, + ): ... class ServerCallContext(_Weakrefable): - - def peer_identity(self) -> bytes: - ... - - def peer(self) -> str: - ... - - def is_cancelled(self): - ... - - def add_header(self, key, value) -> None: - ... - - def add_trailer(self, key, value) -> None: - ... - - def get_middleware(self, key): - ... + def peer_identity(self) -> bytes: ... + def peer(self) -> str: ... + def is_cancelled(self): ... + def add_header(self, key, value) -> None: ... + def add_trailer(self, key, value) -> None: ... + def get_middleware(self, key): ... class ServerAuthReader(_Weakrefable): - - def read(self) -> bytes: - ... + def read(self) -> bytes: ... class ServerAuthSender(_Weakrefable): - - def write(self, message) -> None: - ... + def write(self, message) -> None: ... class ClientAuthReader(_Weakrefable): - - def read(self) -> bytes: - ... + def read(self) -> bytes: ... class ClientAuthSender(_Weakrefable): - - def write(self, message) -> None: - ... + def write(self, message) -> None: ... class ServerAuthHandler(_Weakrefable): - - def authenticate(self, outgoing: ServerAuthSender, incoming: ServerAuthReader): - ... - - def is_valid(self, token: bytes): - ... + def authenticate(self, outgoing: ServerAuthSender, incoming: ServerAuthReader): ... + def is_valid(self, token: bytes): ... class ClientAuthHandler(_Weakrefable): + def authenticate(self, outgoing: ClientAuthSender, incoming: ClientAuthReader): ... + def get_token(self): ... - def authenticate(self, outgoing: ClientAuthSender, incoming: ClientAuthReader): - ... - - def get_token(self): - ... - -class CallInfo(_CallInfo): - ... +class CallInfo(_CallInfo): ... class ClientMiddlewareFactory(_Weakrefable): - - def start_call(self, info: CallInfo) -> None: - ... + def start_call(self, info: CallInfo) -> None: ... class ClientMiddleware(_Weakrefable): - - def sending_headers(self) -> None: - ... - - def received_headers(self, headers: dict) -> None: - ... - - def call_completed(self, exception: ArrowException) -> None: - ... + def sending_headers(self) -> None: ... + def received_headers(self, headers: dict) -> None: ... + def call_completed(self, exception: ArrowException) -> None: ... class ServerMiddlewareFactory(_Weakrefable): + def start_call(self, info: CallInfo, headers: dict) -> None: ... - def start_call(self, info: CallInfo, headers: dict) -> None: - ... - -class TracingServerMiddlewareFactory(ServerMiddlewareFactory): - ... +class TracingServerMiddlewareFactory(ServerMiddlewareFactory): ... class ServerMiddleware(_Weakrefable): - - def sending_headers(self) -> None: - ... - - def call_completed(self, exception: ArrowException) -> None: - ... + def sending_headers(self) -> None: ... + def call_completed(self, exception: ArrowException) -> None: ... class TracingServerMiddleware(ServerMiddleware): trace_context: Incomplete - __slots__ = ['trace_context'] + __slots__ = ["trace_context"] - def __init__(self, trace_context): - ... + def __init__(self, trace_context): ... class _ServerMiddlewareFactoryWrapper(ServerMiddlewareFactory): - - def __init__(self, factories: dict[Any, Any]): - ... - - def start_call(self, info, headers) -> _ServerMiddlewareWrapper | None: - ... + def __init__(self, factories: dict[Any, Any]): ... + def start_call(self, info, headers) -> _ServerMiddlewareWrapper | None: ... class _ServerMiddlewareWrapper(ServerMiddleware): - - def __init__(self, middleware: dict[Any, Any]): - ... - - def sending_headers(self): - ... - - def call_completed(self, exception) -> None: - ... + def __init__(self, middleware: dict[Any, Any]): ... + def sending_headers(self): ... + def call_completed(self, exception) -> None: ... class _FlightServerFinalizer(_Weakrefable): - - def finalize(self) -> None: - ... + def finalize(self) -> None: ... class FlightServerBase(_Weakrefable): - - def __init__(self, location=None, auth_handler=None, tls_certificates=None, verify_client=None, root_certificates=None, middleware=None): - ... - - @property - def port(self): - ... - - def list_flights(self, context: ServerCallContext, criteria: bytes) -> Iterator[FlightInfo]: - ... - - def get_flight_info(self, context: ServerCallContext, descriptor: FlightDescriptor) -> FlightInfo: - ... - - def get_schema(self, context: ServerCallContext, descriptor: FlightDescriptor) -> pyarrow.Schema: - ... - - def do_put(self, context: ServerCallContext, descriptor: FlightDescriptor, reader: MetadataRecordBatchReader, writer: FlightMetadataWriter): - ... - - def do_get(self, context: ServerCallContext, ticket: Ticket) -> FlightDataStream: - ... - - def do_exchange(self, context: ServerCallContext, descriptor: FlightDescriptor, reader: MetadataRecordBatchReader, writer: MetadataRecordBatchWriter): - ... - - def list_actions(self, context: ServerCallContext) -> Iterator[ActionType] | tuple: - ... - - def do_action(self, context: ServerCallContext, action: Action) -> Iterator[bytes]: - ... - - def serve(self) -> None: - ... - - def run(self) -> None: - ... - - def shutdown(self) -> None: - ... - - def wait(self) -> None: - ... - - def __enter__(self) -> Self: - ... - - def __exit__(self, exc_type, exc_value, traceback) -> None: - ... + def __init__( + self, + location=None, + auth_handler=None, + tls_certificates=None, + verify_client=None, + root_certificates=None, + middleware=None, + ): ... + @property + def port(self): ... + def list_flights( + self, context: ServerCallContext, criteria: bytes + ) -> Iterator[FlightInfo]: ... + def get_flight_info( + self, context: ServerCallContext, descriptor: FlightDescriptor + ) -> FlightInfo: ... + def get_schema( + self, context: ServerCallContext, descriptor: FlightDescriptor + ) -> pyarrow.Schema: ... + def do_put( + self, + context: ServerCallContext, + descriptor: FlightDescriptor, + reader: MetadataRecordBatchReader, + writer: FlightMetadataWriter, + ): ... + def do_get( + self, context: ServerCallContext, ticket: Ticket + ) -> FlightDataStream: ... + def do_exchange( + self, + context: ServerCallContext, + descriptor: FlightDescriptor, + reader: MetadataRecordBatchReader, + writer: MetadataRecordBatchWriter, + ): ... + def list_actions( + self, context: ServerCallContext + ) -> Iterator[ActionType] | tuple: ... + def do_action( + self, context: ServerCallContext, action: Action + ) -> Iterator[bytes]: ... + def serve(self) -> None: ... + def run(self) -> None: ... + def shutdown(self) -> None: ... + def wait(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type, exc_value, traceback) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/_fs.pyi b/python/pyarrow-stubs/pyarrow/_fs.pyi index eb9b2f68b99c..be7557e82fdf 100644 --- a/python/pyarrow-stubs/pyarrow/_fs.pyi +++ b/python/pyarrow-stubs/pyarrow/_fs.pyi @@ -16,25 +16,32 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_fs.pyx -import os -import re import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from typing import Any +from collections.abc import Mapping from pyarrow.lib import _Weakrefable -from _typeshed import Incomplete from abc import ABC, abstractmethod from datetime import datetime from enum import IntEnum from pyarrow.lib import * -def _file_type_to_string(ty) -> str: - ... - -def _copy_files(source_fs: FileSystem, source_path: str, destination_fs: FileSystem, destination_path: str, chunk_size: int, use_threads: bool) -> None: - ... - -def _copy_files_selector(source_fs: FileSystem, source_sel: FileSelector, destination_fs: FileSystem, destination_base_dir: str, chunk_size: int, use_threads: bool) -> None: - ... +def _file_type_to_string(ty) -> str: ... +def _copy_files( + source_fs: FileSystem, + source_path: str, + destination_fs: FileSystem, + destination_path: str, + chunk_size: int, + use_threads: bool, +) -> None: ... +def _copy_files_selector( + source_fs: FileSystem, + source_sel: FileSelector, + destination_fs: FileSystem, + destination_base_dir: str, + chunk_size: int, + use_threads: bool, +) -> None: ... class FileType(IntEnum): NotFound = ... @@ -43,252 +50,144 @@ class FileType(IntEnum): Directory = ... class FileInfo(_Weakrefable): - - def __init__(self, path, type: FileType=FileType.Unknown, *, mtime=None, mtime_ns=None, size=None): - ... - - def __repr__(self) -> str: - ... - + def __init__( + self, path, type: FileType = ..., *, mtime=None, mtime_ns=None, size=None + ): ... @property - def type(self) -> FileType: - ... - + def type(self) -> FileType: ... @property - def is_file(self) -> bool: - ... - + def is_file(self) -> bool: ... @property - def path(self) -> str: - ... - + def path(self) -> str: ... @property - def base_name(self) -> str: - ... - + def base_name(self) -> str: ... @property - def size(self) -> int | None: - ... - + def size(self) -> int | None: ... @property - def extension(self) -> str: - ... - + def extension(self) -> str: ... @property - def mtime(self): - ... - + def mtime(self): ... @property - def mtime_ns(self) -> int | None: - ... + def mtime_ns(self) -> int | None: ... class FileSelector(_Weakrefable): - - def __init__(self, base_dir, allow_not_found: bool=False, recursive: bool=False): - ... - + def __init__( + self, base_dir, allow_not_found: bool = False, recursive: bool = False + ): ... @property - def base_dir(self) -> None: - ... - + def base_dir(self) -> None: ... @base_dir.setter - def base_dir(self, base_dir) -> None: - ... - + def base_dir(self, base_dir) -> None: ... @property - def allow_not_found(self) -> None: - ... - + def allow_not_found(self) -> None: ... @allow_not_found.setter - def allow_not_found(self, allow_not_found: bool) -> None: - ... - + def allow_not_found(self, allow_not_found: bool) -> None: ... @property - def recursive(self) -> None: - ... - + def recursive(self) -> None: ... @recursive.setter - def recursive(self, recursive: bool) -> None: - ... - - def __repr__(self) -> str: - ... + def recursive(self, recursive: bool) -> None: ... class FileSystem(_Weakrefable): - - def __init__(self): - ... - + def __init__(self): ... @staticmethod - def _from_uri(uri): - ... - + def _from_uri(uri): ... @staticmethod - def _fsspec_from_uri(uri) -> tuple: - ... - + def _fsspec_from_uri(uri) -> tuple: ... @staticmethod - def _native_from_uri(uri) -> tuple: - ... - + def _native_from_uri(uri) -> tuple: ... @staticmethod - def from_uri(uri: str): - ... - - def equals(self, other: FileSystem) -> bool: - ... - - def __eq__(self, other): - ... - + def from_uri(uri: str): ... + def equals(self, other: FileSystem) -> bool: ... + def __eq__(self, other): ... @property - def type_name(self) -> str: - ... - - def get_file_info(self, paths_or_selector) -> FileInfo | list: - ... - - def create_dir(self, path: str, *, recursive: bool=True) -> None: - ... - - def delete_dir(self, path: str) -> None: - ... - - def delete_dir_contents(self, path: str, *, accept_root_dir: bool=False, missing_dir_ok: bool=False) -> None: - ... - - def move(self, src: str, dest: str) -> None: - ... - - def copy_file(self, src: str, dest: str) -> None: - ... - - def delete_file(self, path: str) -> None: - ... - - def _wrap_input_stream(self, stream, path, compression, buffer_size) -> pyarrow.BufferedInputStream: - ... - - def _wrap_output_stream(self, stream, path, compression, buffer_size) -> pyarrow.BufferedOutputStream: - ... - - def open_input_file(self, path: str) -> pyarrow.NativeFile: - ... - - def open_input_stream(self, path: str, compression: str | None='detect', buffer_size: int | None=None) -> pyarrow.NativeFile: - ... - - def open_output_stream(self, path: str, compression: str | None='detect', buffer_size: int | None=None, metadata: dict | None=None) -> pyarrow.NativeFile: - ... - - def open_append_stream(self, path: str, compression: str | None='detect', buffer_size: int | None=None, metadata: dict | None=None) -> pyarrow.NativeFile: - ... - - def normalize_path(self, path: str) -> str: - ... + def type_name(self) -> str: ... + def get_file_info(self, paths_or_selector) -> FileInfo | list: ... + def create_dir(self, path: str, *, recursive: bool = True) -> None: ... + def delete_dir(self, path: str) -> None: ... + def delete_dir_contents( + self, path: str, *, accept_root_dir: bool = False, missing_dir_ok: bool = False + ) -> None: ... + def move(self, src: str, dest: str) -> None: ... + def copy_file(self, src: str, dest: str) -> None: ... + def delete_file(self, path: str) -> None: ... + def _wrap_input_stream( + self, stream, path, compression, buffer_size + ) -> pyarrow.BufferedInputStream: ... + def _wrap_output_stream( + self, stream, path, compression, buffer_size + ) -> pyarrow.BufferedOutputStream: ... + def open_input_file(self, path: str) -> pyarrow.NativeFile: ... + def open_input_stream( + self, + path: str, + compression: str | None = "detect", + buffer_size: int | None = None, + ) -> pyarrow.NativeFile: ... + def open_output_stream( + self, + path: str, + compression: str | None = "detect", + buffer_size: int | None = None, + metadata: dict | None = None, + ) -> pyarrow.NativeFile: ... + def open_append_stream( + self, + path: str, + compression: str | None = "detect", + buffer_size: int | None = None, + metadata: dict | None = None, + ) -> pyarrow.NativeFile: ... + def normalize_path(self, path: str) -> str: ... class LocalFileSystem(FileSystem): - - def __init__(self, *, use_mmap: bool=False): - ... - - def __reduce__(self) -> tuple: - ... + def __init__(self, *, use_mmap: bool = False): ... + def __reduce__(self) -> tuple: ... class SubTreeFileSystem(FileSystem): - - def __init__(self, base_path, base_fs: FileSystem): - ... - - def __repr__(self) -> str: - ... - - def __reduce__(self) -> tuple: - ... - + def __init__(self, base_path, base_fs: FileSystem): ... + def __reduce__(self) -> tuple: ... @property - def base_path(self) -> str: - ... - + def base_path(self) -> str: ... @property - def base_fs(self) -> FileSystem: - ... + def base_fs(self) -> FileSystem: ... class _MockFileSystem(FileSystem): - - def __init__(self, current_time: datetime | None=None): - ... + def __init__(self, current_time: datetime | None = None): ... class PyFileSystem(FileSystem): - - def __init__(self, handler): - ... - + def __init__(self, handler): ... @property - def handler(self) -> FileSystemHandler: - ... - - def __reduce__(self) -> tuple: - ... + def handler(self) -> FileSystemHandler: ... + def __reduce__(self) -> tuple: ... class FileSystemHandler(ABC): - @abstractmethod - def get_type_name(self) -> None: - ... - + def get_type_name(self) -> None: ... @abstractmethod - def get_file_info(self, paths: list[str]) -> None: - ... - + def get_file_info(self, paths: list[str]) -> None: ... @abstractmethod - def get_file_info_selector(self, selector: FileSelector) -> None: - ... - + def get_file_info_selector(self, selector: FileSelector) -> None: ... @abstractmethod - def create_dir(self, path: str, recursive: bool) -> None: - ... - + def create_dir(self, path: str, recursive: bool) -> None: ... @abstractmethod - def delete_dir(self, path: str) -> None: - ... - + def delete_dir(self, path: str) -> None: ... @abstractmethod - def delete_dir_contents(self, path: str, missing_dir_ok: bool=False) -> None: - ... - + def delete_dir_contents(self, path: str, missing_dir_ok: bool = False) -> None: ... @abstractmethod - def delete_root_dir_contents(self) -> None: - ... - + def delete_root_dir_contents(self) -> None: ... @abstractmethod - def delete_file(self, path: str) -> None: - ... - + def delete_file(self, path: str) -> None: ... @abstractmethod - def move(self, src: str, dest: str) -> None: - ... - + def move(self, src: str, dest: str) -> None: ... @abstractmethod - def copy_file(self, src: str, dest: str) -> None: - ... - + def copy_file(self, src: str, dest: str) -> None: ... @abstractmethod - def open_input_stream(self, path: str) -> None: - ... - + def open_input_stream(self, path: str) -> None: ... @abstractmethod - def open_input_file(self, path: str) -> None: - ... - + def open_input_file(self, path: str) -> None: ... @abstractmethod - def open_output_stream(self, path: str, metadata: Mapping[Any, Any]) -> None: - ... - + def open_output_stream(self, path: str, metadata: Mapping[Any, Any]) -> None: ... @abstractmethod - def open_append_stream(self, path: str, metadata: Mapping[Any, Any]) -> None: - ... - + def open_append_stream(self, path: str, metadata: Mapping[Any, Any]) -> None: ... @abstractmethod - def normalize_path(self, path: str) -> None: - ... + def normalize_path(self, path: str) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/_gcsfs.pyi b/python/pyarrow-stubs/pyarrow/_gcsfs.pyi index dfac2d2e323b..aa9c52ac5e80 100644 --- a/python/pyarrow-stubs/pyarrow/_gcsfs.pyi +++ b/python/pyarrow-stubs/pyarrow/_gcsfs.pyi @@ -16,32 +16,28 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_gcsfs.pyx -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete from pyarrow._fs import FileSystem class GcsFileSystem(FileSystem): - - def __init__(self, *, anonymous: bool=False, access_token=None, target_service_account=None, credential_token_expiration=None, default_bucket_location: str='US', scheme=None, endpoint_override=None, default_metadata=None, retry_time_limit=None, project_id=None): - ... - - def _expiration_datetime_from_options(self): - ... - + def __init__( + self, + *, + anonymous: bool = False, + access_token=None, + target_service_account=None, + credential_token_expiration=None, + default_bucket_location: str = "US", + scheme=None, + endpoint_override=None, + default_metadata=None, + retry_time_limit=None, + project_id=None, + ): ... + def _expiration_datetime_from_options(self): ... @staticmethod - def _reconstruct(kwargs) -> GcsFileSystem: - ... - - def __reduce__(self) -> tuple: - ... - + def _reconstruct(kwargs) -> GcsFileSystem: ... + def __reduce__(self) -> tuple: ... @property - def default_bucket_location(self) -> str: - ... - + def default_bucket_location(self) -> str: ... @property - def project_id(self) -> str: - ... + def project_id(self) -> str: ... diff --git a/python/pyarrow-stubs/pyarrow/_hdfs.pyi b/python/pyarrow-stubs/pyarrow/_hdfs.pyi index 213c67cfb42b..0c6b9ee65bb8 100644 --- a/python/pyarrow-stubs/pyarrow/_hdfs.pyi +++ b/python/pyarrow-stubs/pyarrow/_hdfs.pyi @@ -16,28 +16,25 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_hdfs.pyx -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete from pyarrow._fs import FileSystem -def _have_libhdfs() -> bool: - ... +def _have_libhdfs() -> bool: ... class HadoopFileSystem(FileSystem): - - def __init__(self, host: str, port: int=8020, *, user: str | None=None, replication: int=3, buffer_size: int=0, default_block_size=None, kerb_ticket=None, extra_conf=None): - ... - + def __init__( + self, + host: str, + port: int = 8020, + *, + user: str | None = None, + replication: int = 3, + buffer_size: int = 0, + default_block_size=None, + kerb_ticket=None, + extra_conf=None, + ): ... @staticmethod - def from_uri(uri: str) -> HadoopFileSystem: - ... - + def from_uri(uri: str) -> HadoopFileSystem: ... @staticmethod - def _reconstruct(kwargs) -> HadoopFileSystem: - ... - - def __reduce__(self) -> tuple: - ... + def _reconstruct(kwargs) -> HadoopFileSystem: ... + def __reduce__(self) -> tuple: ... diff --git a/python/pyarrow-stubs/pyarrow/_json.pyi b/python/pyarrow-stubs/pyarrow/_json.pyi index f2126427bb07..cb4f9db27f16 100644 --- a/python/pyarrow-stubs/pyarrow/_json.pyi +++ b/python/pyarrow-stubs/pyarrow/_json.pyi @@ -16,107 +16,66 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_json.pyx -import os -import re import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence import pyarrow.json -from _typeshed import Incomplete from pyarrow.lib import MemoryPool, RecordBatchReader, Schema, _Weakrefable -def read_json(input_file, read_options: pyarrow.json.ReadOptions | None=None, parse_options: pyarrow.json.ParseOptions | None=None, memory_pool: MemoryPool | None=None) -> pyarrow.Table: - ... - -def open_json(input_file, read_options: pyarrow.json.ReadOptions | None=None, parse_options: pyarrow.json.ParseOptions | None=None, memory_pool: MemoryPool | None=None) -> JSONStreamingReader: - ... +def read_json( + input_file, + read_options: pyarrow.json.ReadOptions | None = None, + parse_options: pyarrow.json.ParseOptions | None = None, + memory_pool: MemoryPool | None = None, +) -> pyarrow.Table: ... +def open_json( + input_file, + read_options: pyarrow.json.ReadOptions | None = None, + parse_options: pyarrow.json.ParseOptions | None = None, + memory_pool: MemoryPool | None = None, +) -> JSONStreamingReader: ... class ReadOptions(_Weakrefable): __slots__ = () - def __init__(self, use_threads=None, block_size=None): - ... - + def __init__(self, use_threads=None, block_size=None): ... @property - def use_threads(self) -> None: - ... - + def use_threads(self) -> None: ... @use_threads.setter - def use_threads(self, value) -> None: - ... - + def use_threads(self, value) -> None: ... @property - def block_size(self) -> None: - ... - + def block_size(self) -> None: ... @block_size.setter - def block_size(self, value) -> None: - ... - - def __reduce__(self) -> tuple: - ... - - def equals(self, other: ReadOptions) -> bool: - ... - - def __eq__(self, other): - ... - - def __repr__(self) -> str: - ... - - def __str__(self) -> str: - ... + def block_size(self, value) -> None: ... + def __reduce__(self) -> tuple: ... + def equals(self, other: ReadOptions) -> bool: ... + def __eq__(self, other): ... class ParseOptions(_Weakrefable): __slots__ = () - def __init__(self, explicit_schema=None, newlines_in_values=None, unexpected_field_behavior=None): - ... - - def __reduce__(self) -> tuple: - ... - + def __init__( + self, + explicit_schema=None, + newlines_in_values=None, + unexpected_field_behavior=None, + ): ... + def __reduce__(self) -> tuple: ... @property - def explicit_schema(self) -> None: - ... - + def explicit_schema(self) -> None: ... @explicit_schema.setter - def explicit_schema(self, value) -> None: - ... - + def explicit_schema(self, value) -> None: ... @property - def newlines_in_values(self) -> None: - ... - + def newlines_in_values(self) -> None: ... @newlines_in_values.setter - def newlines_in_values(self, value) -> None: - ... - + def newlines_in_values(self, value) -> None: ... @property - def unexpected_field_behavior(self) -> None: - ... - + def unexpected_field_behavior(self) -> None: ... @unexpected_field_behavior.setter - def unexpected_field_behavior(self, value) -> None: - ... - - def equals(self, other: ParseOptions) -> bool: - ... - - def __eq__(self, other): - ... - - def _repr_base(self) -> str: - ... - - def __repr__(self) -> str: - ... - - def __str__(self) -> str: - ... + def unexpected_field_behavior(self, value) -> None: ... + def equals(self, other: ParseOptions) -> bool: ... + def __eq__(self, other): ... + def _repr_base(self) -> str: ... class JSONStreamingReader(RecordBatchReader): schema: Schema - def __init__(self): - ... + def __init__(self): ... diff --git a/python/pyarrow-stubs/pyarrow/_orc.pyi b/python/pyarrow-stubs/pyarrow/_orc.pyi index 8f2979c10636..151d4c041641 100644 --- a/python/pyarrow-stubs/pyarrow/_orc.pyi +++ b/python/pyarrow-stubs/pyarrow/_orc.pyi @@ -16,93 +16,54 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_orc.pyx -import os -import re import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence from typing import TypedDict from _typeshed import Incomplete from pyarrow.lib import MemoryPool, Table, _Weakrefable -from typing_extensions import TypeAlias class ORCReader(_Weakrefable): - - def __init__(self, memory_pool: MemoryPool | None=None): - ... - - def open(self, source: object, use_memory_map: bool=True) -> None: - ... - - def metadata(self) -> pyarrow.KeyValueMetadata: - ... - - def schema(self) -> pyarrow.Schema: - ... - - def nrows(self): - ... - - def nstripes(self): - ... - - def file_version(self): - ... - - def software_version(self) -> str: - ... - - def compression(self): - ... - - def compression_size(self): - ... - - def row_index_stride(self): - ... - - def writer(self): - ... - - def writer_version(self): - ... - - def nstripe_statistics(self): - ... - - def content_length(self): - ... - - def stripe_statistics_length(self): - ... - - def file_footer_length(self): - ... - - def file_postscript_length(self): - ... - - def file_length(self): - ... - - def serialized_file_tail(self): - ... - - def read_stripe(self, n, columns=None) -> pyarrow.RecordBatch: - ... - - def read(self, columns=None) -> pyarrow.Table: - ... + def __init__(self, memory_pool: MemoryPool | None = None): ... + def open(self, source: object, use_memory_map: bool = True) -> None: ... + def metadata(self) -> pyarrow.KeyValueMetadata: ... + def schema(self) -> pyarrow.Schema: ... + def nrows(self): ... + def nstripes(self): ... + def file_version(self): ... + def software_version(self) -> str: ... + def compression(self): ... + def compression_size(self): ... + def row_index_stride(self): ... + def writer(self): ... + def writer_version(self): ... + def nstripe_statistics(self): ... + def content_length(self): ... + def stripe_statistics_length(self): ... + def file_footer_length(self): ... + def file_postscript_length(self): ... + def file_length(self): ... + def serialized_file_tail(self): ... + def read_stripe(self, n, columns=None) -> pyarrow.RecordBatch: ... + def read(self, columns=None) -> pyarrow.Table: ... class ORCWriter(_Weakrefable): - - def open(self, where: object, *, file_version=None, batch_size=None, stripe_size=None, compression=None, compression_block_size=None, compression_strategy=None, row_index_stride=None, padding_tolerance=None, dictionary_key_size_threshold=None, bloom_filter_columns=None, bloom_filter_fpp=None) -> None: - ... - - def write(self, table: Table) -> None: - ... - - def close(self) -> None: - ... + def open( + self, + where: object, + *, + file_version=None, + batch_size=None, + stripe_size=None, + compression=None, + compression_block_size=None, + compression_strategy=None, + row_index_stride=None, + padding_tolerance=None, + dictionary_key_size_threshold=None, + bloom_filter_columns=None, + bloom_filter_fpp=None, + ) -> None: ... + def write(self, table: Table) -> None: ... + def close(self) -> None: ... class WriteOptions(TypedDict): batch_size: int diff --git a/python/pyarrow-stubs/pyarrow/_parquet.pyi b/python/pyarrow-stubs/pyarrow/_parquet.pyi index ce6bb4fc20d8..4c4b78ece3d1 100644 --- a/python/pyarrow-stubs/pyarrow/_parquet.pyi +++ b/python/pyarrow-stubs/pyarrow/_parquet.pyi @@ -16,589 +16,336 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_parquet.pyx -import os -import re import pyarrow -from typing import Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete +from collections.abc import Iterator from typing import Any from pyarrow.lib import Buffer, MemoryPool, Schema, Table, _Weakrefable + _DEFAULT_ROW_GROUP_SIZE: int _MAX_ROW_GROUP_SIZE: int -def _reconstruct_filemetadata(serialized: Buffer) -> FileMetaData: - ... +def _reconstruct_filemetadata(serialized: Buffer) -> FileMetaData: ... class Statistics(_Weakrefable): - - def __init__(self): - ... - - def __repr__(self) -> str: - ... - - def to_dict(self) -> dict: - ... - - def __eq__(self, other): - ... - - def equals(self, other: Statistics) -> bool: - ... - + def __init__(self): ... + def to_dict(self) -> dict: ... + def __eq__(self, other): ... + def equals(self, other: Statistics) -> bool: ... @property - def has_min_max(self): - ... - + def has_min_max(self): ... @property - def has_null_count(self): - ... - + def has_null_count(self): ... @property - def has_distinct_count(self): - ... - + def has_distinct_count(self): ... @property - def min_raw(self): - ... - + def min_raw(self): ... @property - def max_raw(self): - ... - + def max_raw(self): ... @property - def min(self): - ... - + def min(self): ... @property - def max(self): - ... - + def max(self): ... @property - def null_count(self): - ... - + def null_count(self): ... @property - def distinct_count(self): - ... - + def distinct_count(self): ... @property - def num_values(self): - ... - + def num_values(self): ... @property - def physical_type(self): - ... - + def physical_type(self): ... @property - def logical_type(self): - ... - + def logical_type(self): ... @property - def converted_type(self): - ... + def converted_type(self): ... class ParquetLogicalType(_Weakrefable): - - def __init__(self): - ... - - def __repr__(self) -> str: - ... - - def __str__(self) -> str: - ... - - def to_json(self) -> str: - ... - + def __init__(self): ... + def to_json(self) -> str: ... @property - def type(self): - ... + def type(self): ... class GeoStatistics(_Weakrefable): - - def __init__(self): - ... - - def __repr__(self) -> str: - ... - - def to_dict(self) -> dict: - ... - + def __init__(self): ... + def to_dict(self) -> dict: ... @property - def geospatial_types(self) -> None | list: - ... - + def geospatial_types(self) -> None | list: ... @property - def xmin(self): - ... - + def xmin(self): ... @property - def xmax(self): - ... - + def xmax(self): ... @property - def ymin(self): - ... - + def ymin(self): ... @property - def ymax(self): - ... - + def ymax(self): ... @property - def zmin(self): - ... - + def zmin(self): ... @property - def zmax(self): - ... - + def zmax(self): ... @property - def mmin(self): - ... - + def mmin(self): ... @property - def mmax(self): - ... - - def _x_valid(self): - ... - - def _y_valid(self): - ... - - def _z_valid(self): - ... - - def _m_valid(self): - ... + def mmax(self): ... + def _x_valid(self): ... + def _y_valid(self): ... + def _z_valid(self): ... + def _m_valid(self): ... class ColumnChunkMetaData(_Weakrefable): - - def __init__(self): - ... - - def __repr__(self) -> str: - ... - - def to_dict(self) -> dict: - ... - - def __eq__(self, other): - ... - - def equals(self, other: ColumnChunkMetaData) -> bool: - ... - + def __init__(self): ... + def to_dict(self) -> dict: ... + def __eq__(self, other): ... + def equals(self, other: ColumnChunkMetaData) -> bool: ... @property - def file_offset(self): - ... - + def file_offset(self): ... @property - def file_path(self) -> str: - ... - + def file_path(self) -> str: ... @property - def physical_type(self): - ... - + def physical_type(self): ... @property - def num_values(self): - ... - + def num_values(self): ... @property - def path_in_schema(self) -> str: - ... - + def path_in_schema(self) -> str: ... @property - def is_stats_set(self): - ... - + def is_stats_set(self): ... @property - def statistics(self) -> None | Statistics: - ... - + def statistics(self) -> None | Statistics: ... @property - def is_geo_stats_set(self): - ... - + def is_geo_stats_set(self): ... @property - def geo_statistics(self) -> None | GeoStatistics: - ... - + def geo_statistics(self) -> None | GeoStatistics: ... @property - def compression(self): - ... - + def compression(self): ... @property - def encodings(self) -> tuple: - ... - + def encodings(self) -> tuple: ... @property - def has_dictionary_page(self) -> bool: - ... - + def has_dictionary_page(self) -> bool: ... @property - def dictionary_page_offset(self): - ... - + def dictionary_page_offset(self): ... @property - def data_page_offset(self): - ... - + def data_page_offset(self): ... @property - def has_index_page(self): - ... - + def has_index_page(self): ... @property - def index_page_offset(self): - ... - + def index_page_offset(self): ... @property - def total_compressed_size(self): - ... - + def total_compressed_size(self): ... @property - def total_uncompressed_size(self): - ... - + def total_uncompressed_size(self): ... @property - def bloom_filter_offset(self): - ... - + def bloom_filter_offset(self): ... @property - def bloom_filter_length(self): - ... - + def bloom_filter_length(self): ... @property - def has_offset_index(self): - ... - + def has_offset_index(self): ... @property - def has_column_index(self): - ... - + def has_column_index(self): ... @property - def metadata(self): - ... + def metadata(self): ... class SortingColumn: - - def __init__(self, column_index: int, descending: bool=False, nulls_first: bool=False): - ... - + def __init__( + self, column_index: int, descending: bool = False, nulls_first: bool = False + ): ... @classmethod - def from_ordering(cls, schema: Schema, sort_keys, null_placement: str='at_end') -> tuple: - ... - + def from_ordering( + cls, schema: Schema, sort_keys, null_placement: str = "at_end" + ) -> tuple: ... @staticmethod - def to_ordering(schema: Schema, sorting_columns: tuple[SortingColumn, ...]) -> tuple: - ... - - def __repr__(self) -> str: - ... - - def __eq__(self, other: SortingColumn): - ... - - def __hash__(self): - ... - + def to_ordering( + schema: Schema, sorting_columns: tuple[SortingColumn, ...] + ) -> tuple: ... + def __eq__(self, other: SortingColumn): ... + def __hash__(self): ... @property - def column_index(self): - ... - + def column_index(self): ... @property - def descending(self): - ... - + def descending(self): ... @property - def nulls_first(self): - ... - - def to_dict(self) -> dict: - ... + def nulls_first(self): ... + def to_dict(self) -> dict: ... class RowGroupMetaData(_Weakrefable): - - def __init__(self): - ... - - def __reduce__(self) -> tuple: - ... - - def __eq__(self, other): - ... - - def equals(self, other: RowGroupMetaData) -> bool: - ... - - def column(self, i: int) -> ColumnChunkMetaData: - ... - - def __repr__(self) -> str: - ... - - def to_dict(self) -> dict: - ... - + def __init__(self): ... + def __reduce__(self) -> tuple: ... + def __eq__(self, other): ... + def equals(self, other: RowGroupMetaData) -> bool: ... + def column(self, i: int) -> ColumnChunkMetaData: ... + def to_dict(self) -> dict: ... @property - def num_columns(self): - ... - + def num_columns(self): ... @property - def num_rows(self): - ... - + def num_rows(self): ... @property - def total_byte_size(self): - ... - + def total_byte_size(self): ... @property - def sorting_columns(self) -> tuple: - ... + def sorting_columns(self) -> tuple: ... class FileMetaData(_Weakrefable): - - def __init__(self): - ... - - def __reduce__(self) -> tuple: - ... - - def __hash__(self): - ... - - def __repr__(self) -> str: - ... - - def to_dict(self) -> dict: - ... - - def __eq__(self, other): - ... - - def equals(self, other: FileMetaData) -> bool: - ... - + def __init__(self): ... + def __reduce__(self) -> tuple: ... + def __hash__(self): ... + def to_dict(self) -> dict: ... + def __eq__(self, other): ... + def equals(self, other: FileMetaData) -> bool: ... @property - def schema(self): - ... - + def schema(self): ... @property - def serialized_size(self): - ... - + def serialized_size(self): ... @property - def num_columns(self): - ... - + def num_columns(self): ... @property - def num_rows(self): - ... - + def num_rows(self): ... @property - def num_row_groups(self): - ... - + def num_row_groups(self): ... @property - def format_version(self) -> str: - ... - + def format_version(self) -> str: ... @property - def created_by(self) -> str: - ... - + def created_by(self) -> str: ... @property - def metadata(self): - ... - - def row_group(self, i: int) -> RowGroupMetaData: - ... - - def set_file_path(self, path: str) -> None: - ... - - def append_row_groups(self, other: FileMetaData) -> None: - ... - - def write_metadata_file(self, where) -> None: - ... + def metadata(self): ... + def row_group(self, i: int) -> RowGroupMetaData: ... + def set_file_path(self, path: str) -> None: ... + def append_row_groups(self, other: FileMetaData) -> None: ... + def write_metadata_file(self, where) -> None: ... class ParquetSchema(_Weakrefable): - - def __init__(self, container: FileMetaData): - ... - - def __repr__(self) -> str: - ... - - def __reduce__(self) -> tuple: - ... - - def __len__(self): - ... - - def __getitem__(self, i): - ... - - def __hash__(self): - ... - - @property - def names(self) -> list: - ... - - def to_arrow_schema(self) -> pyarrow.Schema: - ... - - def __eq__(self, other): - ... - - def equals(self, other: ParquetSchema) -> bool: - ... - - def column(self, i: int) -> ColumnSchema: - ... + def __init__(self, container: FileMetaData): ... + def __reduce__(self) -> tuple: ... + def __len__(self): ... + def __getitem__(self, i): ... + def __hash__(self): ... + @property + def names(self) -> list: ... + def to_arrow_schema(self) -> pyarrow.Schema: ... + def __eq__(self, other): ... + def equals(self, other: ParquetSchema) -> bool: ... + def column(self, i: int) -> ColumnSchema: ... class ColumnSchema(_Weakrefable): - - def __init__(self, schema: ParquetSchema, index: int): - ... - - def __eq__(self, other): - ... - - def __reduce__(self) -> tuple: - ... - - def equals(self, other: ColumnSchema) -> bool: - ... - - def __repr__(self) -> str: - ... - + def __init__(self, schema: ParquetSchema, index: int): ... + def __eq__(self, other): ... + def __reduce__(self) -> tuple: ... + def equals(self, other: ColumnSchema) -> bool: ... @property - def name(self) -> str: - ... - + def name(self) -> str: ... @property - def path(self) -> str: - ... - + def path(self) -> str: ... @property - def max_definition_level(self): - ... - + def max_definition_level(self): ... @property - def max_repetition_level(self): - ... - + def max_repetition_level(self): ... @property - def physical_type(self): - ... - + def physical_type(self): ... @property - def logical_type(self): - ... - + def logical_type(self): ... @property - def converted_type(self): - ... - + def converted_type(self): ... @property - def length(self): - ... - + def length(self): ... @property - def precision(self): - ... - + def precision(self): ... @property - def scale(self): - ... + def scale(self): ... class ParquetReader(_Weakrefable): _column_idx_map: Any - def __init__(self, memory_pool: MemoryPool | None=None): - ... - - def open(self, source: object, *, use_memory_map: bool=False, read_dictionary=None, binary_type: pyarrow.DataType | None=None, list_type=None, metadata: FileMetaData | None=None, buffer_size: int=0, pre_buffer: bool=True, coerce_int96_timestamp_unit: str | None=None, decryption_properties: FileDecryptionProperties | None=None, thrift_string_size_limit: int | None=None, thrift_container_size_limit: int | None=None, schema_depth_limit: int | None=None, page_checksum_verification: bool=False, arrow_extensions_enabled: bool=False) -> None: - ... - - @property - def column_paths(self) -> list: - ... - - @property - def metadata(self): - ... - - @property - def schema_arrow(self) -> pyarrow.Schema: - ... - - @property - def num_row_groups(self): - ... - - def set_use_threads(self, use_threads: bool) -> None: - ... - - def set_batch_size(self, batch_size: int) -> None: - ... - - def iter_batches(self, batch_size: int, row_groups: list[int], column_indices: list[int] | None=None, use_threads: bool=True) -> Iterator[pyarrow.RecordBatch]: - ... - - def read_row_group(self, i: int, column_indices: list[int] | None=None, use_threads: bool=True) -> pyarrow.Table: - ... - - def read_row_groups(self, row_groups: list[int], column_indices: list[int] | None=None, use_threads: bool=True) -> pyarrow.Table: - ... - - def read_all(self, column_indices: list[int] | None=None, use_threads: bool=True) -> pyarrow.Table: - ... - - def scan_contents(self, column_indices: list[int] | None=None, batch_size: int=65536) -> int: - ... - - def column_name_idx(self, column_name: str) -> int: - ... - - def read_column(self, column_index: int) -> pyarrow.ChunkedArray: - ... - - def close(self) -> None: - ... - - @property - def closed(self): - ... + def __init__(self, memory_pool: MemoryPool | None = None): ... + def open( + self, + source: object, + *, + use_memory_map: bool = False, + read_dictionary=None, + binary_type: pyarrow.DataType | None = None, + list_type=None, + metadata: FileMetaData | None = None, + buffer_size: int = 0, + pre_buffer: bool = True, + coerce_int96_timestamp_unit: str | None = None, + decryption_properties: FileDecryptionProperties | None = None, + thrift_string_size_limit: int | None = None, + thrift_container_size_limit: int | None = None, + schema_depth_limit: int | None = None, + page_checksum_verification: bool = False, + arrow_extensions_enabled: bool = False, + ) -> None: ... + @property + def column_paths(self) -> list: ... + @property + def metadata(self): ... + @property + def schema_arrow(self) -> pyarrow.Schema: ... + @property + def num_row_groups(self): ... + def set_use_threads(self, use_threads: bool) -> None: ... + def set_batch_size(self, batch_size: int) -> None: ... + def iter_batches( + self, + batch_size: int, + row_groups: list[int], + column_indices: list[int] | None = None, + use_threads: bool = True, + ) -> Iterator[pyarrow.RecordBatch]: ... + def read_row_group( + self, i: int, column_indices: list[int] | None = None, use_threads: bool = True + ) -> pyarrow.Table: ... + def read_row_groups( + self, + row_groups: list[int], + column_indices: list[int] | None = None, + use_threads: bool = True, + ) -> pyarrow.Table: ... + def read_all( + self, column_indices: list[int] | None = None, use_threads: bool = True + ) -> pyarrow.Table: ... + def scan_contents( + self, column_indices: list[int] | None = None, batch_size: int = 65536 + ) -> int: ... + def column_name_idx(self, column_name: str) -> int: ... + def read_column(self, column_index: int) -> pyarrow.ChunkedArray: ... + def close(self) -> None: ... + @property + def closed(self): ... class ParquetWriter(_Weakrefable): - - def __init__(self, where, schema: Schema, use_dictionary=None, compression=None, version=None, write_statistics=None, memory_pool: MemoryPool | None=None, use_deprecated_int96_timestamps: bool=False, coerce_timestamps=None, data_page_size=None, max_rows_per_page=None, allow_truncated_timestamps: bool=False, compression_level=None, use_byte_stream_split: bool=False, column_encoding=None, writer_engine_version=None, data_page_version=None, use_compliant_nested_type: bool=True, encryption_properties=None, write_batch_size=None, dictionary_pagesize_limit=None, store_schema: bool=True, write_page_index: bool=False, write_page_checksum: bool=False, sorting_columns=None, store_decimal_as_integer: bool=False, use_content_defined_chunking: bool=False, write_time_adjusted_to_utc: bool=False, bloom_filter_options=None): - ... - - def close(self) -> None: - ... - - def write_table(self, table: Table, row_group_size=None) -> None: - ... - - def add_key_value_metadata(self, key_value_metadata) -> None: - ... - - @property - def metadata(self) -> FileMetaData: - ... - -class FileEncryptionProperties: - ... - -class FileDecryptionProperties: - ... + def __init__( + self, + where, + schema: Schema, + use_dictionary=None, + compression=None, + version=None, + write_statistics=None, + memory_pool: MemoryPool | None = None, + use_deprecated_int96_timestamps: bool = False, + coerce_timestamps=None, + data_page_size=None, + max_rows_per_page=None, + allow_truncated_timestamps: bool = False, + compression_level=None, + use_byte_stream_split: bool = False, + column_encoding=None, + writer_engine_version=None, + data_page_version=None, + use_compliant_nested_type: bool = True, + encryption_properties=None, + write_batch_size=None, + dictionary_pagesize_limit=None, + store_schema: bool = True, + write_page_index: bool = False, + write_page_checksum: bool = False, + sorting_columns=None, + store_decimal_as_integer: bool = False, + use_content_defined_chunking: bool = False, + write_time_adjusted_to_utc: bool = False, + bloom_filter_options=None, + ): ... + def close(self) -> None: ... + def write_table(self, table: Table, row_group_size=None) -> None: ... + def add_key_value_metadata(self, key_value_metadata) -> None: ... + @property + def metadata(self) -> FileMetaData: ... + +class FileEncryptionProperties: ... +class FileDecryptionProperties: ... diff --git a/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi b/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi index 3ef1683e171b..ee2e916b360d 100644 --- a/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi +++ b/python/pyarrow-stubs/pyarrow/_parquet_encryption.pyi @@ -16,237 +16,180 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_parquet_encryption.pyx -import os -import re import pyarrow -from typing import Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence import pyarrow.parquet -from _typeshed import Incomplete from typing import Any from pyarrow._fs import FileSystem from pyarrow.lib import _Weakrefable -def create_decryption_properties(footer_key: bytes, *, aad_prefix: bytes | None=None, check_footer_integrity: bool=True, allow_plaintext_files: bool=False) -> pyarrow.parquet.FileDecryptionProperties: - ... - -def create_encryption_properties(footer_key: bytes, *, aad_prefix: bytes | None=None, store_aad_prefix: bool=True, encryption_algorithm: str='AES_GCM_V1', plaintext_footer: bool=False) -> pyarrow.parquet.FileEncryptionProperties: - ... +def create_decryption_properties( + footer_key: bytes, + *, + aad_prefix: bytes | None = None, + check_footer_integrity: bool = True, + allow_plaintext_files: bool = False, +) -> pyarrow.parquet.FileDecryptionProperties: ... +def create_encryption_properties( + footer_key: bytes, + *, + aad_prefix: bytes | None = None, + store_aad_prefix: bool = True, + encryption_algorithm: str = "AES_GCM_V1", + plaintext_footer: bool = False, +) -> pyarrow.parquet.FileEncryptionProperties: ... class EncryptionConfiguration(_Weakrefable): __slots__ = () - def __init__(self, footer_key, *, column_keys=None, uniform_encryption=None, encryption_algorithm=None, plaintext_footer=None, double_wrapping=None, cache_lifetime=None, internal_key_material=None, data_key_length_bits=None): - ... - - @property - def footer_key(self) -> str: - ... - - @property - def column_keys(self) -> None: - ... - + def __init__( + self, + footer_key, + *, + column_keys=None, + uniform_encryption=None, + encryption_algorithm=None, + plaintext_footer=None, + double_wrapping=None, + cache_lifetime=None, + internal_key_material=None, + data_key_length_bits=None, + ): ... + @property + def footer_key(self) -> str: ... + @property + def column_keys(self) -> None: ... @column_keys.setter - def column_keys(self, value: dict[Any, Any]) -> None: - ... - + def column_keys(self, value: dict[Any, Any]) -> None: ... @property - def uniform_encryption(self) -> None: - ... - + def uniform_encryption(self) -> None: ... @uniform_encryption.setter - def uniform_encryption(self, value) -> None: - ... - + def uniform_encryption(self, value) -> None: ... @property - def encryption_algorithm(self) -> None: - ... - + def encryption_algorithm(self) -> None: ... @encryption_algorithm.setter - def encryption_algorithm(self, value) -> None: - ... - + def encryption_algorithm(self, value) -> None: ... @property - def plaintext_footer(self) -> None: - ... - + def plaintext_footer(self) -> None: ... @plaintext_footer.setter - def plaintext_footer(self, value) -> None: - ... - + def plaintext_footer(self, value) -> None: ... @property - def double_wrapping(self) -> None: - ... - + def double_wrapping(self) -> None: ... @double_wrapping.setter - def double_wrapping(self, value) -> None: - ... - + def double_wrapping(self, value) -> None: ... @property - def cache_lifetime(self) -> None: - ... - + def cache_lifetime(self) -> None: ... @cache_lifetime.setter - def cache_lifetime(self, value) -> None: - ... - + def cache_lifetime(self, value) -> None: ... @property - def internal_key_material(self) -> None: - ... - + def internal_key_material(self) -> None: ... @internal_key_material.setter - def internal_key_material(self, value) -> None: - ... - + def internal_key_material(self, value) -> None: ... @property - def data_key_length_bits(self) -> None: - ... - + def data_key_length_bits(self) -> None: ... @data_key_length_bits.setter - def data_key_length_bits(self, value) -> None: - ... + def data_key_length_bits(self, value) -> None: ... class DecryptionConfiguration(_Weakrefable): __slots__ = () - def __init__(self, *, cache_lifetime=None, read_kms_url=None): - ... - + def __init__(self, *, cache_lifetime=None, read_kms_url=None): ... @property - def cache_lifetime(self) -> None: - ... - + def cache_lifetime(self) -> None: ... @cache_lifetime.setter - def cache_lifetime(self, value) -> None: - ... - + def cache_lifetime(self, value) -> None: ... @property - def read_kms_url(self) -> None: - ... - + def read_kms_url(self) -> None: ... @read_kms_url.setter - def read_kms_url(self, value) -> None: - ... + def read_kms_url(self, value) -> None: ... class KmsConnectionConfig(_Weakrefable): __slots__ = () - def __init__(self, *, kms_instance_id=None, kms_instance_url=None, key_access_token=None, custom_kms_conf=None): - ... - + def __init__( + self, + *, + kms_instance_id=None, + kms_instance_url=None, + key_access_token=None, + custom_kms_conf=None, + ): ... @property - def kms_instance_id(self) -> None: - ... - + def kms_instance_id(self) -> None: ... @kms_instance_id.setter - def kms_instance_id(self, value) -> None: - ... - + def kms_instance_id(self, value) -> None: ... @property - def kms_instance_url(self) -> None: - ... - + def kms_instance_url(self) -> None: ... @kms_instance_url.setter - def kms_instance_url(self, value) -> None: - ... - + def kms_instance_url(self, value) -> None: ... @property - def key_access_token(self) -> None: - ... - + def key_access_token(self) -> None: ... @key_access_token.setter - def key_access_token(self, value) -> None: - ... - + def key_access_token(self, value) -> None: ... @property - def custom_kms_conf(self) -> None: - ... - + def custom_kms_conf(self) -> None: ... @custom_kms_conf.setter - def custom_kms_conf(self, value: dict[Any, Any]) -> None: - ... - - def refresh_key_access_token(self, value) -> None: - ... + def custom_kms_conf(self, value: dict[Any, Any]) -> None: ... + def refresh_key_access_token(self, value) -> None: ... class KmsClient(_Weakrefable): - - def __init__(self): - ... - - def wrap_key(self, key_bytes, master_key_identifier): - ... - - def unwrap_key(self, wrapped_key, master_key_identifier): - ... + def __init__(self): ... + def wrap_key(self, key_bytes, master_key_identifier): ... + def unwrap_key(self, wrapped_key, master_key_identifier): ... class CryptoFactory(_Weakrefable): __slots__ = () - def __init__(self, kms_client_factory): - ... - - def file_encryption_properties(self, kms_connection_config: KmsConnectionConfig, encryption_config: EncryptionConfiguration, parquet_file_path=None, filesystem: FileSystem | None=None) -> pyarrow.parquet.FileEncryptionProperties: - ... - - def file_decryption_properties(self, kms_connection_config: KmsConnectionConfig, decryption_config: DecryptionConfiguration | None=None, parquet_file_path=None, filesystem: FileSystem | None=None) -> pyarrow.parquet.FileDecryptionProperties: - ... - - def remove_cache_entries_for_token(self, access_token) -> None: - ... - - def remove_cache_entries_for_all_tokens(self) -> None: - ... - - def rotate_master_keys(self, kms_connection_config: KmsConnectionConfig, parquet_file_path, filesystem: FileSystem | None=None, double_wrapping: bool=True, cache_lifetime_seconds: int | float=600, read_kms_url: bool=False) -> None: - ... + def __init__(self, kms_client_factory): ... + def file_encryption_properties( + self, + kms_connection_config: KmsConnectionConfig, + encryption_config: EncryptionConfiguration, + parquet_file_path=None, + filesystem: FileSystem | None = None, + ) -> pyarrow.parquet.FileEncryptionProperties: ... + def file_decryption_properties( + self, + kms_connection_config: KmsConnectionConfig, + decryption_config: DecryptionConfiguration | None = None, + parquet_file_path=None, + filesystem: FileSystem | None = None, + ) -> pyarrow.parquet.FileDecryptionProperties: ... + def remove_cache_entries_for_token(self, access_token) -> None: ... + def remove_cache_entries_for_all_tokens(self) -> None: ... + def rotate_master_keys( + self, + kms_connection_config: KmsConnectionConfig, + parquet_file_path, + filesystem: FileSystem | None = None, + double_wrapping: bool = True, + cache_lifetime_seconds: float = 600, + read_kms_url: bool = False, + ) -> None: ... class KeyMaterial(_Weakrefable): - @property - def is_footer_key(self): - ... - + def is_footer_key(self): ... @property - def is_double_wrapped(self): - ... - + def is_double_wrapped(self): ... @property - def master_key_id(self) -> str: - ... - + def master_key_id(self) -> str: ... @property - def wrapped_dek(self) -> str: - ... - + def wrapped_dek(self) -> str: ... @property - def kek_id(self) -> str: - ... - + def kek_id(self) -> str: ... @property - def wrapped_kek(self) -> str: - ... - + def wrapped_kek(self) -> str: ... @property - def kms_instance_id(self) -> str: - ... - + def kms_instance_id(self) -> str: ... @property - def kms_instance_url(self) -> str: - ... - + def kms_instance_url(self) -> str: ... @staticmethod - def parse(key_material_string: bytes) -> KeyMaterial: - ... + def parse(key_material_string: bytes) -> KeyMaterial: ... class FileSystemKeyMaterialStore(_Weakrefable): - - def get_key_material(self, key_id): - ... - - def get_key_id_set(self): - ... - + def get_key_material(self, key_id): ... + def get_key_id_set(self): ... @classmethod - def for_file(cls, parquet_file_path, filesystem: FileSystem | None=None) -> FileSystemKeyMaterialStore: - ... + def for_file( + cls, parquet_file_path, filesystem: FileSystem | None = None + ) -> FileSystemKeyMaterialStore: ... diff --git a/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi b/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi index 5593c5fb2d2b..7766baa44734 100644 --- a/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi +++ b/python/pyarrow-stubs/pyarrow/_pyarrow_cpp_tests.pyi @@ -16,25 +16,9 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_pyarrow_cpp_tests.pyx -import os -import re -import pyarrow -from typing import Any, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from typing import Callable -from _typeshed import Incomplete -from typing_extensions import TypeAlias - -def get_cpp_tests() -> list: - ... +def get_cpp_tests() -> list: ... class CppTestCase: - @property - def name(self) -> str: - ... - - def __repr__(self) -> str: - ... - - def __call__(self) -> None: - ... + def name(self) -> str: ... + def __call__(self) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/_s3fs.pyi b/python/pyarrow-stubs/pyarrow/_s3fs.pyi index c0c611c821ed..199d368b2376 100644 --- a/python/pyarrow-stubs/pyarrow/_s3fs.pyi +++ b/python/pyarrow-stubs/pyarrow/_s3fs.pyi @@ -16,28 +16,16 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_s3fs.pyx -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete from enum import IntEnum from pyarrow._fs import FileSystem -def initialize_s3(log_level: S3LogLevel=S3LogLevel.Fatal, num_event_loop_threads: int=1) -> None: - ... - -def ensure_s3_initialized() -> None: - ... - -def finalize_s3() -> None: - ... - -def ensure_s3_finalized() -> None: - ... - -def resolve_s3_region(bucket: str) -> str: - ... +def initialize_s3( + log_level: S3LogLevel = ..., num_event_loop_threads: int = 1 +) -> None: ... +def ensure_s3_initialized() -> None: ... +def finalize_s3() -> None: ... +def ensure_s3_finalized() -> None: ... +def resolve_s3_region(bucket: str) -> str: ... class S3LogLevel(IntEnum): Off = ... @@ -49,28 +37,41 @@ class S3LogLevel(IntEnum): Trace = ... class S3RetryStrategy: + def __init__(self, max_attempts: int = 3): ... - def __init__(self, max_attempts: int=3): - ... - -class AwsStandardS3RetryStrategy(S3RetryStrategy): - ... - -class AwsDefaultS3RetryStrategy(S3RetryStrategy): - ... +class AwsStandardS3RetryStrategy(S3RetryStrategy): ... +class AwsDefaultS3RetryStrategy(S3RetryStrategy): ... class S3FileSystem(FileSystem): - - def __init__(self, *, access_key=None, secret_key=None, session_token=None, anonymous: bool=False, region=None, request_timeout=None, connect_timeout=None, scheme=None, endpoint_override=None, background_writes: bool=True, default_metadata=None, role_arn=None, session_name=None, external_id=None, load_frequency: int=900, proxy_options=None, allow_delayed_open: bool=False, allow_bucket_creation: bool=False, allow_bucket_deletion: bool=False, check_directory_existence_before_creation: bool=False, retry_strategy: S3RetryStrategy=AwsStandardS3RetryStrategy(max_attempts=3), force_virtual_addressing: bool=False, tls_ca_file_path=None): - ... - + def __init__( + self, + *, + access_key=None, + secret_key=None, + session_token=None, + anonymous: bool = False, + region=None, + request_timeout=None, + connect_timeout=None, + scheme=None, + endpoint_override=None, + background_writes: bool = True, + default_metadata=None, + role_arn=None, + session_name=None, + external_id=None, + load_frequency: int = 900, + proxy_options=None, + allow_delayed_open: bool = False, + allow_bucket_creation: bool = False, + allow_bucket_deletion: bool = False, + check_directory_existence_before_creation: bool = False, + retry_strategy: S3RetryStrategy = ..., + force_virtual_addressing: bool = False, + tls_ca_file_path=None, + ): ... @staticmethod - def _reconstruct(kwargs) -> S3FileSystem: - ... - - def __reduce__(self) -> tuple: - ... - + def _reconstruct(kwargs) -> S3FileSystem: ... + def __reduce__(self) -> tuple: ... @property - def region(self) -> str: - ... + def region(self) -> str: ... diff --git a/python/pyarrow-stubs/pyarrow/_substrait.pyi b/python/pyarrow-stubs/pyarrow/_substrait.pyi index 11e067fb5514..ea3c8cd58d80 100644 --- a/python/pyarrow-stubs/pyarrow/_substrait.pyi +++ b/python/pyarrow-stubs/pyarrow/_substrait.pyi @@ -16,62 +16,42 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/_substrait.pyx -import os -import re import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence import pyarrow.compute from pyarrow.lib import _Weakrefable -from _typeshed import Incomplete from pyarrow.lib import * from types import ModuleType -def run_query(plan, *, table_provider: object | None=None, use_threads: bool=True) -> pyarrow.RecordBatchReader: - ... +def run_query( + plan, *, table_provider: object | None = None, use_threads: bool = True +) -> pyarrow.RecordBatchReader: ... +def _parse_json_plan(plan: bytes) -> pyarrow.Buffer: ... +def serialize_schema(schema: pyarrow.Schema) -> SubstraitSchema: ... +def _serialize_namedstruct_schema(schema): ... +def deserialize_schema( + buf: pyarrow.Buffer | bytes | SubstraitSchema, +) -> pyarrow.Schema: ... +def serialize_expressions( + exprs: list[pyarrow.compute.Expression], + names: list[str], + schema: pyarrow.Schema, + *, + allow_arrow_extensions: bool = False, +) -> pyarrow.Buffer: ... +def deserialize_expressions(buf: pyarrow.Buffer | bytes) -> BoundExpressions: ... +def get_supported_functions() -> list: ... -def _parse_json_plan(plan: bytes) -> pyarrow.Buffer: - ... - -def serialize_schema(schema: pyarrow.Schema) -> SubstraitSchema: - ... - -def _serialize_namedstruct_schema(schema): - ... - -def deserialize_schema(buf: pyarrow.Buffer | bytes | SubstraitSchema) -> pyarrow.Schema: - ... - -def serialize_expressions(exprs: list[pyarrow.compute.Expression], names: list[str], schema: pyarrow.Schema, *, allow_arrow_extensions: bool=False) -> pyarrow.Buffer: - ... - -def deserialize_expressions(buf: pyarrow.Buffer | bytes) -> BoundExpressions: - ... - -def get_supported_functions() -> list: - ... py_substrait: ModuleType | None class SubstraitSchema: - - def __init__(self, schema, expression): - ... - - def to_pysubstrait(self): - ... + def __init__(self, schema, expression): ... + def to_pysubstrait(self): ... class BoundExpressions(_Weakrefable): - - def __init__(self): - ... - + def __init__(self): ... @property - def schema(self) -> pyarrow.Schema: - ... - + def schema(self) -> pyarrow.Schema: ... @property - def expressions(self) -> dict: - ... - + def expressions(self) -> dict: ... @classmethod - def from_substrait(cls, message) -> BoundExpressions: - ... + def from_substrait(cls, message) -> BoundExpressions: ... diff --git a/python/pyarrow-stubs/pyarrow/acero.pyi b/python/pyarrow-stubs/pyarrow/acero.pyi index c8f84d4846f4..6a94d7377622 100644 --- a/python/pyarrow-stubs/pyarrow/acero.pyi +++ b/python/pyarrow-stubs/pyarrow/acero.pyi @@ -17,42 +17,81 @@ from pyarrow._compute import CountOptions, PivotWiderOptions from pyarrow._dataset import FileSystemDataset, InMemoryDataset, UnionDataset -from typing import List, Optional, Tuple, Type, Union -import os -import re import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from typing import Any from _typeshed import Incomplete import pyarrow.compute import pyarrow.dataset -from pyarrow._acero import AggregateNodeOptions as AggregateNodeOptions, AsofJoinNodeOptions as AsofJoinNodeOptions, Declaration as Declaration, ExecNodeOptions as ExecNodeOptions, FilterNodeOptions as FilterNodeOptions, HashJoinNodeOptions as HashJoinNodeOptions, OrderByNodeOptions as OrderByNodeOptions, ProjectNodeOptions as ProjectNodeOptions, RecordBatchReaderSourceNodeOptions as RecordBatchReaderSourceNodeOptions, TableSourceNodeOptions as TableSourceNodeOptions +from pyarrow._acero import ( + AggregateNodeOptions as AggregateNodeOptions, + AsofJoinNodeOptions as AsofJoinNodeOptions, + Declaration as Declaration, + ExecNodeOptions as ExecNodeOptions, + FilterNodeOptions as FilterNodeOptions, + HashJoinNodeOptions as HashJoinNodeOptions, + OrderByNodeOptions as OrderByNodeOptions, + ProjectNodeOptions as ProjectNodeOptions, + RecordBatchReaderSourceNodeOptions as RecordBatchReaderSourceNodeOptions, + TableSourceNodeOptions as TableSourceNodeOptions, +) from pyarrow._dataset import ScanNodeOptions as ScanNodeOptions from pyarrow.compute import Expression as Expression, field as field from pyarrow.lib import RecordBatch as RecordBatch, Table as Table, array as array -ds: Incomplete - -def _dataset_to_decl(dataset: FileSystemDataset | UnionDataset | InMemoryDataset, use_threads: bool=True, implicit_ordering: bool=False) -> Declaration: - ... - -def _perform_join(join_type: str, left_operand: Table | pyarrow.dataset.Dataset, left_keys: str | list[str], right_operand: Table | pyarrow.dataset.Dataset, right_keys: str | list[str], left_suffix: str | None=None, right_suffix: str | None=None, use_threads: bool=True, coalesce_keys: bool=False, output_type=..., filter_expression: pyarrow.compute.Expression | None=None) -> Table | pyarrow.dataset.InMemoryDataset: - ... - -def _perform_join_asof(left_operand: Table | pyarrow.dataset.Dataset, left_on: str, left_by: list[Any] | str | list[str], right_operand: Table | pyarrow.dataset.Dataset, right_on: str | list[str], right_by: list[Any] | str | list[str], tolerance: int, use_threads: bool=True, output_type=...) -> Table | pyarrow.dataset.InMemoryDataset: - ... -def _filter_table(table: Table | RecordBatch, expression: Expression) -> Table | RecordBatch: - ... - -def _sort_source(table_or_dataset: FileSystemDataset | InMemoryDataset, sort_keys: list[tuple[str, str, str]], output_type=..., **kwargs) -> InMemoryDataset: - ... +ds: Incomplete -def _group_by(table: Table, aggregates: list[tuple[tuple[str, str], str, None, str] | tuple[list[Any], str, None, str] | tuple[list[str], str, CountOptions, str] | Any | tuple[list[str], str, None, str] | tuple[tuple[str, str], str, PivotWiderOptions, str]], keys: list[str], use_threads: bool=True) -> Table: - ... +def _dataset_to_decl( + dataset: FileSystemDataset | UnionDataset | InMemoryDataset, + use_threads: bool = True, + implicit_ordering: bool = False, +) -> Declaration: ... +def _perform_join( + join_type: str, + left_operand: Table | pyarrow.dataset.Dataset, + left_keys: str | list[str], + right_operand: Table | pyarrow.dataset.Dataset, + right_keys: str | list[str], + left_suffix: str | None = None, + right_suffix: str | None = None, + use_threads: bool = True, + coalesce_keys: bool = False, + output_type=..., + filter_expression: pyarrow.compute.Expression | None = None, +) -> Table | pyarrow.dataset.InMemoryDataset: ... +def _perform_join_asof( + left_operand: Table | pyarrow.dataset.Dataset, + left_on: str, + left_by: list[Any] | str | list[str], + right_operand: Table | pyarrow.dataset.Dataset, + right_on: str | list[str], + right_by: list[Any] | str | list[str], + tolerance: int, + use_threads: bool = True, + output_type=..., +) -> Table | pyarrow.dataset.InMemoryDataset: ... +def _filter_table( + table: Table | RecordBatch, expression: Expression +) -> Table | RecordBatch: ... +def _sort_source( + table_or_dataset: FileSystemDataset | InMemoryDataset, + sort_keys: list[tuple[str, str, str]], + output_type=..., + **kwargs, +) -> InMemoryDataset: ... +def _group_by( + table: Table, + aggregates: list[ + tuple[tuple[str, str], str, None, str] + | tuple[list[Any], str, None, str] + | tuple[list[str], str, CountOptions, str] + | Any + | tuple[list[str], str, None, str] + | tuple[tuple[str, str], str, PivotWiderOptions, str] + ], + keys: list[str], + use_threads: bool = True, +) -> Table: ... class DatasetModuleStub: - - class Dataset: - ... - - class InMemoryDataset: - ... + class Dataset: ... + class InMemoryDataset: ... diff --git a/python/pyarrow-stubs/pyarrow/benchmark.pyi b/python/pyarrow-stubs/pyarrow/benchmark.pyi index 8fa8a22c1956..68aa3ab3db99 100644 --- a/python/pyarrow-stubs/pyarrow/benchmark.pyi +++ b/python/pyarrow-stubs/pyarrow/benchmark.pyi @@ -15,9 +15,4 @@ # specific language governing permissions and limitations # under the License. -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete from pyarrow.lib import benchmark_PandasObjectIsNull as benchmark_PandasObjectIsNull diff --git a/python/pyarrow-stubs/pyarrow/cffi.pyi b/python/pyarrow-stubs/pyarrow/cffi.pyi index 8dac34060d11..69456bd3fc53 100644 --- a/python/pyarrow-stubs/pyarrow/cffi.pyi +++ b/python/pyarrow-stubs/pyarrow/cffi.pyi @@ -15,10 +15,7 @@ # specific language governing permissions and limitations # under the License. -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence from _typeshed import Incomplete + c_source: str ffi: Incomplete diff --git a/python/pyarrow-stubs/pyarrow/compute.pyi b/python/pyarrow-stubs/pyarrow/compute.pyi index 231cbf040bec..e2f51437889b 100644 --- a/python/pyarrow-stubs/pyarrow/compute.pyi +++ b/python/pyarrow-stubs/pyarrow/compute.pyi @@ -15,893 +15,1358 @@ # specific language governing permissions and limitations # under the License. -from pyarrow._compute import MetaFunction -from pyarrow.lib import BinaryArray, BinaryScalar, BinaryViewArray, Bool8Array, Bool8Type, BooleanArray, ChunkedArray, DataType, Date32Array, Date32Scalar, Date64Array, Date64Scalar, Decimal128Array, Decimal128Scalar, Decimal128Type, Decimal256Array, Decimal256Scalar, Decimal256Type, Decimal32Scalar, Decimal32Type, Decimal64Scalar, Decimal64Type, DenseUnionType, DictionaryArray, DictionaryType, DoubleArray, DoubleScalar, DurationArray, DurationType, ExtensionArray, FixedShapeTensorArray, FixedShapeTensorType, FixedSizeBinaryArray, FixedSizeBinaryType, FixedSizeListArray, FixedSizeListType, FloatArray, FloatScalar, HalfFloatArray, Int16Array, Int16Scalar, Int32Array, Int64Array, Int64Scalar, Int8Array, Int8Scalar, JsonArray, JsonType, LargeBinaryArray, LargeListArray, LargeListType, LargeStringArray, LargeStringScalar, ListArray, ListType, MapArray, MapType, MonthDayNanoIntervalArray, NullArray, NullScalar, OpaqueArray, OpaqueType, RecordBatch, SparseUnionType, StringArray, StringScalar, StringViewArray, StructArray, StructType, Table, Time32Array, Time64Array, TimestampArray, TimestampScalar, TimestampType, UInt16Array, UInt16Scalar, UInt32Array, UInt32Scalar, UInt64Array, UInt64Scalar, UInt8Array, UInt8Scalar -from pyarrow.tests.test_extension_type import ExampleUuidType, ExampleUuidType2, IntegerEmbeddedType, IntegerType, MyFixedListType, MyListType, PeriodType, PeriodTypeWithClass, PeriodTypeWithToPandasDtype, TinyIntType -from typing import Dict, List, Optional, Tuple, Type, Union -import os -import re -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from pyarrow.lib import ( + BinaryArray, + BinaryScalar, + Bool8Array, + BooleanArray, + ChunkedArray, + Date32Array, + Date64Array, + Decimal128Array, + DictionaryArray, + DoubleArray, + DoubleScalar, + DurationArray, + ExtensionArray, + FixedShapeTensorArray, + FixedSizeBinaryArray, + FixedSizeListArray, + FloatArray, + FloatScalar, + HalfFloatArray, + Int16Array, + Int16Scalar, + Int32Array, + Int64Array, + Int64Scalar, + Int8Array, + Int8Scalar, + JsonArray, + LargeBinaryArray, + LargeListArray, + LargeStringArray, + ListArray, + MapArray, + NullArray, + NullScalar, + OpaqueArray, + RecordBatch, + StringArray, + StringScalar, + StringViewArray, + StructArray, + Table, + Time32Array, + Time64Array, + TimestampArray, + TimestampScalar, + UInt16Array, + UInt16Scalar, + UInt32Array, + UInt64Array, + UInt64Scalar, + UInt8Array, + UInt8Scalar, +) +from collections.abc import Sequence import pyarrow.lib from _typeshed import Incomplete -from pyarrow._compute import ArraySortOptions as ArraySortOptions, AssumeTimezoneOptions as AssumeTimezoneOptions, CastOptions as CastOptions, CountOptions as CountOptions, CumulativeOptions as CumulativeOptions, CumulativeSumOptions as CumulativeSumOptions, DayOfWeekOptions as DayOfWeekOptions, DictionaryEncodeOptions as DictionaryEncodeOptions, ElementWiseAggregateOptions as ElementWiseAggregateOptions, Expression as Expression, ExtractRegexOptions as ExtractRegexOptions, ExtractRegexSpanOptions as ExtractRegexSpanOptions, FilterOptions as FilterOptions, Function as Function, FunctionOptions as FunctionOptions, FunctionRegistry as FunctionRegistry, HashAggregateFunction as HashAggregateFunction, HashAggregateKernel as HashAggregateKernel, IndexOptions as IndexOptions, InversePermutationOptions as InversePermutationOptions, JoinOptions as JoinOptions, Kernel as Kernel, ListFlattenOptions as ListFlattenOptions, ListSliceOptions as ListSliceOptions, MakeStructOptions as MakeStructOptions, MapLookupOptions as MapLookupOptions, MatchSubstringOptions as MatchSubstringOptions, ModeOptions as ModeOptions, NullOptions as NullOptions, PadOptions as PadOptions, PairwiseOptions as PairwiseOptions, PartitionNthOptions as PartitionNthOptions, PivotWiderOptions as PivotWiderOptions, QuantileOptions as QuantileOptions, RandomOptions as RandomOptions, RankOptions as RankOptions, RankQuantileOptions as RankQuantileOptions, ReplaceSliceOptions as ReplaceSliceOptions, ReplaceSubstringOptions as ReplaceSubstringOptions, RoundBinaryOptions as RoundBinaryOptions, RoundOptions as RoundOptions, RoundTemporalOptions as RoundTemporalOptions, RoundToMultipleOptions as RoundToMultipleOptions, RunEndEncodeOptions as RunEndEncodeOptions, ScalarAggregateFunction as ScalarAggregateFunction, ScalarAggregateKernel as ScalarAggregateKernel, ScalarAggregateOptions as ScalarAggregateOptions, ScalarFunction as ScalarFunction, ScalarKernel as ScalarKernel, ScatterOptions as ScatterOptions, SearchSortedOptions as SearchSortedOptions, SelectKOptions as SelectKOptions, SetLookupOptions as SetLookupOptions, SkewOptions as SkewOptions, SliceOptions as SliceOptions, SortOptions as SortOptions, SplitOptions as SplitOptions, SplitPatternOptions as SplitPatternOptions, StrftimeOptions as StrftimeOptions, StrptimeOptions as StrptimeOptions, StructFieldOptions as StructFieldOptions, TDigestOptions as TDigestOptions, TakeOptions as TakeOptions, TrimOptions as TrimOptions, UdfContext as UdfContext, Utf8NormalizeOptions as Utf8NormalizeOptions, VarianceOptions as VarianceOptions, VectorFunction as VectorFunction, VectorKernel as VectorKernel, WeekOptions as WeekOptions, WinsorizeOptions as WinsorizeOptions, ZeroFillOptions as ZeroFillOptions, call_function as call_function, call_tabular_function as call_tabular_function, function_registry as function_registry, get_function as get_function, list_functions as list_functions, register_aggregate_function as register_aggregate_function, register_scalar_function as register_scalar_function, register_tabular_function as register_tabular_function, register_vector_function as register_vector_function +from pyarrow._compute import ( + ArraySortOptions as ArraySortOptions, + AssumeTimezoneOptions as AssumeTimezoneOptions, + CastOptions as CastOptions, + CountOptions as CountOptions, + CumulativeOptions as CumulativeOptions, + CumulativeSumOptions as CumulativeSumOptions, + DayOfWeekOptions as DayOfWeekOptions, + DictionaryEncodeOptions as DictionaryEncodeOptions, + ElementWiseAggregateOptions as ElementWiseAggregateOptions, + Expression as Expression, + ExtractRegexOptions as ExtractRegexOptions, + ExtractRegexSpanOptions as ExtractRegexSpanOptions, + FilterOptions as FilterOptions, + Function as Function, + FunctionOptions as FunctionOptions, + FunctionRegistry as FunctionRegistry, + HashAggregateFunction as HashAggregateFunction, + HashAggregateKernel as HashAggregateKernel, + IndexOptions as IndexOptions, + InversePermutationOptions as InversePermutationOptions, + JoinOptions as JoinOptions, + Kernel as Kernel, + ListFlattenOptions as ListFlattenOptions, + ListSliceOptions as ListSliceOptions, + MakeStructOptions as MakeStructOptions, + MapLookupOptions as MapLookupOptions, + MatchSubstringOptions as MatchSubstringOptions, + ModeOptions as ModeOptions, + NullOptions as NullOptions, + PadOptions as PadOptions, + PairwiseOptions as PairwiseOptions, + PartitionNthOptions as PartitionNthOptions, + PivotWiderOptions as PivotWiderOptions, + QuantileOptions as QuantileOptions, + RandomOptions as RandomOptions, + RankOptions as RankOptions, + RankQuantileOptions as RankQuantileOptions, + ReplaceSliceOptions as ReplaceSliceOptions, + ReplaceSubstringOptions as ReplaceSubstringOptions, + RoundBinaryOptions as RoundBinaryOptions, + RoundOptions as RoundOptions, + RoundTemporalOptions as RoundTemporalOptions, + RoundToMultipleOptions as RoundToMultipleOptions, + RunEndEncodeOptions as RunEndEncodeOptions, + ScalarAggregateFunction as ScalarAggregateFunction, + ScalarAggregateKernel as ScalarAggregateKernel, + ScalarAggregateOptions as ScalarAggregateOptions, + ScalarFunction as ScalarFunction, + ScalarKernel as ScalarKernel, + ScatterOptions as ScatterOptions, + SearchSortedOptions as SearchSortedOptions, + SelectKOptions as SelectKOptions, + SetLookupOptions as SetLookupOptions, + SkewOptions as SkewOptions, + SliceOptions as SliceOptions, + SortOptions as SortOptions, + SplitOptions as SplitOptions, + SplitPatternOptions as SplitPatternOptions, + StrftimeOptions as StrftimeOptions, + StrptimeOptions as StrptimeOptions, + StructFieldOptions as StructFieldOptions, + TDigestOptions as TDigestOptions, + TakeOptions as TakeOptions, + TrimOptions as TrimOptions, + UdfContext as UdfContext, + Utf8NormalizeOptions as Utf8NormalizeOptions, + VarianceOptions as VarianceOptions, + VectorFunction as VectorFunction, + VectorKernel as VectorKernel, + WeekOptions as WeekOptions, + WinsorizeOptions as WinsorizeOptions, + ZeroFillOptions as ZeroFillOptions, + call_function as call_function, + call_tabular_function as call_tabular_function, + function_registry as function_registry, + get_function as get_function, + list_functions as list_functions, + register_aggregate_function as register_aggregate_function, + register_scalar_function as register_scalar_function, + register_tabular_function as register_tabular_function, + register_vector_function as register_vector_function, +) from typing import ClassVar -def abs(x, *, memory_pool: Incomplete | None=...): - ... - -def abs_checked(x, *, memory_pool: Incomplete | None=...): - ... - -def acos(x, *, memory_pool: Incomplete | None=...): - ... - -def acos_checked(x, *, memory_pool: Incomplete | None=...): - ... - -def acosh(x, *, memory_pool: Incomplete | None=...): - ... - -def acosh_checked(x, *, memory_pool: Incomplete | None=...): - ... - -def add(x, y, *, memory_pool: Incomplete | None=...): - ... - -def add_checked(x, y, *, memory_pool: Incomplete | None=...): - ... - -def all(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def and_(x, y, *, memory_pool: Incomplete | None=...): - ... - -def and_kleene(x, y, *, memory_pool: Incomplete | None=...): - ... - -def and_not(x, y, *, memory_pool: Incomplete | None=...): - ... - -def and_not_kleene(x, y, *, memory_pool: Incomplete | None=...): - ... - -def any(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def approximate_median(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def array_filter(array, selection_filter, null_selection_behavior: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def array_sort_indices(array, order: str=..., *, null_placement: str=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def array_take(array, indices, *, boundscheck: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def ascii_capitalize(strings, *, memory_pool: Incomplete | None=...): - ... - -def ascii_center(strings, width: int, padding: str=..., lean_left_on_odd_padding: bool=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def ascii_is_alnum(strings, *, memory_pool: Incomplete | None=...): - ... - -def ascii_is_alpha(strings, *, memory_pool: Incomplete | None=...): - ... - -def ascii_is_decimal(strings, *, memory_pool: Incomplete | None=...): - ... - -def ascii_is_lower(strings, *, memory_pool: Incomplete | None=...): - ... - -def ascii_is_printable(strings, *, memory_pool: Incomplete | None=...): - ... - -def ascii_is_space(strings, *, memory_pool: Incomplete | None=...): - ... - -def ascii_is_title(strings, *, memory_pool: Incomplete | None=...): - ... - -def ascii_is_upper(strings, *, memory_pool: Incomplete | None=...): - ... - -def ascii_lower(strings, *, memory_pool: Incomplete | None=...): - ... - -def ascii_lpad(strings, width: int, padding: str=..., lean_left_on_odd_padding: bool=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def ascii_ltrim(strings, characters: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def ascii_ltrim_whitespace(strings, *, memory_pool: Incomplete | None=...): - ... - -def ascii_reverse(strings, *, memory_pool: Incomplete | None=...): - ... - -def ascii_rpad(strings, width: int, padding: str=..., lean_left_on_odd_padding: bool=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def ascii_rtrim(strings, characters: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def ascii_rtrim_whitespace(strings, *, memory_pool: Incomplete | None=...): - ... - -def ascii_split_whitespace(strings, *, max_splits: Incomplete | None=..., reverse: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def ascii_swapcase(strings, *, memory_pool: Incomplete | None=...): - ... - -def ascii_title(strings, *, memory_pool: Incomplete | None=...): - ... - -def ascii_trim(strings, characters: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def ascii_trim_whitespace(strings, *, memory_pool: Incomplete | None=...): - ... - -def ascii_upper(strings, *, memory_pool: Incomplete | None=...): - ... - -def asin(x, *, memory_pool: Incomplete | None=...): - ... - -def asin_checked(x, *, memory_pool: Incomplete | None=...): - ... - -def asinh(x, *, memory_pool: Incomplete | None=...): - ... - -def assume_timezone(timestamps, timezone: str, *, ambiguous: str=..., nonexistent: str=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def atan(x, *, memory_pool: Incomplete | None=...): - ... - -def atan2(y, x, *, memory_pool: Incomplete | None=...): - ... - -def atanh(x, *, memory_pool: Incomplete | None=...): - ... - -def atanh_checked(x, *, memory_pool: Incomplete | None=...): - ... - -def binary_join(strings, separator, *, memory_pool: Incomplete | None=...): - ... - -def binary_join_element_wise(*strings, null_handling: str=..., null_replacement: str=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def binary_length(strings, *, memory_pool: Incomplete | None=...): - ... - -def binary_repeat(strings, num_repeats, *, memory_pool: Incomplete | None=...): - ... - -def binary_replace_slice(strings, start: int, stop: int, replacement: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def binary_reverse(strings, *, memory_pool: Incomplete | None=...): - ... - -def binary_slice(strings, start: int, stop: Incomplete | None=..., step: int=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def bit_wise_and(x, y, *, memory_pool: Incomplete | None=...): - ... - -def bit_wise_not(x, *, memory_pool: Incomplete | None=...): - ... - -def bit_wise_or(x, y, *, memory_pool: Incomplete | None=...): - ... - -def bit_wise_xor(x, y, *, memory_pool: Incomplete | None=...): - ... - -def case_when(cond, *cases, memory_pool: Incomplete | None=...): - ... - -def cast(arr: Table | Int16Array | StringViewArray | UInt64Array | Date32Array | JsonArray | DurationArray | UInt16Array | HalfFloatArray | Date64Array | Int64Scalar | ChunkedArray | StringScalar | FloatScalar | FixedShapeTensorArray | DoubleScalar | Int32Array | DictionaryArray | FloatArray | TimestampArray | Decimal128Array | LargeStringArray | ListArray | NullArray | StringArray | OpaqueArray | BinaryScalar | Int8Array | UInt32Array | MapArray | DoubleArray | Time32Array | LargeListArray | BooleanArray | ExtensionArray | BinaryArray | TimestampScalar | Bool8Array | UInt8Array | Int64Array | FixedSizeBinaryArray | Time64Array | StructArray | FixedSizeListArray | Int8Scalar, target_type: Incomplete | None=..., safe: Incomplete | None=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...) -> pyarrow.Array: - ... - -def ceil(x, *, memory_pool: Incomplete | None=...): - ... - -def ceil_temporal(timestamps, multiple: int=..., unit: str=..., *, week_starts_monday: bool=..., ceil_is_strictly_greater: bool=..., calendar_based_origin: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def choose(indices, *values, memory_pool: Incomplete | None=...): - ... - -def coalesce(*values, memory_pool: Incomplete | None=...): - ... - -def cos(x, *, memory_pool: Incomplete | None=...): - ... - -def cos_checked(x, *, memory_pool: Incomplete | None=...): - ... - -def cosh(x, *, memory_pool: Incomplete | None=...): - ... - -def count(array, mode: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def count_distinct(array, mode: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def count_substring(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def count_substring_regex(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def cumulative_max(values, start: Incomplete | None=..., *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def cumulative_mean(values, start: Incomplete | None=..., *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def cumulative_min(values, start: Incomplete | None=..., *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def cumulative_prod(values, start: Incomplete | None=..., *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def cumulative_prod_checked(values, start: Incomplete | None=..., *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def cumulative_sum(values, start: Incomplete | None=..., *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def cumulative_sum_checked(values, start: Incomplete | None=..., *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def day(values, *, memory_pool: Incomplete | None=...): - ... - -def day_of_week(values, *, count_from_zero: bool=..., week_start: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def day_of_year(values, *, memory_pool: Incomplete | None=...): - ... - -def day_time_interval_between(start, end, *, memory_pool: Incomplete | None=...): - ... - -def days_between(start, end, *, memory_pool: Incomplete | None=...): - ... - -def dictionary_decode(dictionary_array, *, memory_pool: Incomplete | None=...): - ... - -def dictionary_encode(array, null_encoding: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def divide(dividend, divisor, *, memory_pool: Incomplete | None=...): - ... - -def divide_checked(dividend, divisor, *, memory_pool: Incomplete | None=...): - ... - -def drop_null(input, *, memory_pool: Incomplete | None=...): - ... - -def ends_with(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def equal(x, y, *, memory_pool: Incomplete | None=...): - ... - -def exp(exponent, *, memory_pool: Incomplete | None=...): - ... - -def expm1(exponent, *, memory_pool: Incomplete | None=...): - ... - -def extract_regex(strings, pattern: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def extract_regex_span(strings, pattern: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def fill_null_backward(values, *, memory_pool: Incomplete | None=...): - ... - -def fill_null_forward(values, *, memory_pool: Incomplete | None=...): - ... - -def filter(input, selection_filter, null_selection_behavior: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def find_substring(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def find_substring_regex(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def first(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def first_last(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def floor(x, *, memory_pool: Incomplete | None=...): - ... - -def floor_temporal(timestamps, multiple: int=..., unit: str=..., *, week_starts_monday: bool=..., ceil_is_strictly_greater: bool=..., calendar_based_origin: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def greater(x, y, *, memory_pool: Incomplete | None=...): - ... - -def greater_equal(x, y, *, memory_pool: Incomplete | None=...): - ... - -def hour(values, *, memory_pool: Incomplete | None=...): - ... - -def hours_between(start, end, *, memory_pool: Incomplete | None=...): - ... - -def hypot(x, y, *, memory_pool: Incomplete | None=...): - ... - -def if_else(cond, left, right, *, memory_pool: Incomplete | None=...): - ... - -def index(data: ChunkedArray | Int64Array, value: Int64Scalar | int | Int8Scalar | None, start: Incomplete | None=..., end: Incomplete | None=..., *, memory_pool: Incomplete | None=...) -> int: - ... - -def index_in(values, value_set: pyarrow.Array, *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def index_in_meta_binary(values, value_set, *, memory_pool: Incomplete | None=...): - ... - -def indices_nonzero(values, *, memory_pool: Incomplete | None=...): - ... - -def inverse_permutation(indices, max_index: int=..., output_type: Incomplete | None=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def invert(values, *, memory_pool: Incomplete | None=...): - ... - -def is_dst(values, *, memory_pool: Incomplete | None=...): - ... - -def is_finite(values, *, memory_pool: Incomplete | None=...): - ... - -def is_in(values, value_set: pyarrow.Array, *, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def is_in_meta_binary(values, value_set, *, memory_pool: Incomplete | None=...): - ... - -def is_inf(values, *, memory_pool: Incomplete | None=...): - ... - -def is_leap_year(values, *, memory_pool: Incomplete | None=...): - ... - -def is_nan(values, *, memory_pool: Incomplete | None=...): - ... - -def is_null(values, *, nan_is_null: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def is_valid(values, *, memory_pool: Incomplete | None=...): - ... - -def iso_calendar(values, *, memory_pool: Incomplete | None=...): - ... - -def iso_week(values, *, memory_pool: Incomplete | None=...): - ... - -def iso_year(values, *, memory_pool: Incomplete | None=...): - ... - -def kurtosis(array, *, skip_nulls: bool=..., biased: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def last(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def less(x, y, *, memory_pool: Incomplete | None=...): - ... - -def less_equal(x, y, *, memory_pool: Incomplete | None=...): - ... - -def list_element(lists, index, *, memory_pool: Incomplete | None=...): - ... - -def list_flatten(lists, recursive: bool=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def list_parent_indices(lists, *, memory_pool: Incomplete | None=...): - ... - -def list_slice(lists, start: int, stop: Incomplete | None=..., step: int=..., return_fixed_size_list: Incomplete | None=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def list_value_length(lists, *, memory_pool: Incomplete | None=...): - ... - -def ln(x, *, memory_pool: Incomplete | None=...): - ... - -def ln_checked(x, *, memory_pool: Incomplete | None=...): - ... - -def local_timestamp(values, *, memory_pool: Incomplete | None=...): - ... - -def log10(x, *, memory_pool: Incomplete | None=...): - ... - -def log10_checked(x, *, memory_pool: Incomplete | None=...): - ... - -def log1p(x, *, memory_pool: Incomplete | None=...): - ... - -def log1p_checked(x, *, memory_pool: Incomplete | None=...): - ... - -def log2(x, *, memory_pool: Incomplete | None=...): - ... - -def log2_checked(x, *, memory_pool: Incomplete | None=...): - ... - -def logb(x, b, *, memory_pool: Incomplete | None=...): - ... - -def logb_checked(x, b, *, memory_pool: Incomplete | None=...): - ... - -def make_struct(*args, field_names: tuple=..., field_nullability: Incomplete | None=..., field_metadata: Incomplete | None=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def map_lookup(container, query_key, occurrence: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def match_like(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def match_substring(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def match_substring_regex(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def max(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def max_element_wise(*args, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def mean(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def microsecond(values, *, memory_pool: Incomplete | None=...): - ... - -def microseconds_between(start, end, *, memory_pool: Incomplete | None=...): - ... - -def millisecond(values, *, memory_pool: Incomplete | None=...): - ... - -def milliseconds_between(start, end, *, memory_pool: Incomplete | None=...): - ... - -def min(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def min_element_wise(*args, skip_nulls: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def min_max(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def minute(values, *, memory_pool: Incomplete | None=...): - ... - -def minutes_between(start, end, *, memory_pool: Incomplete | None=...): - ... - -def mode(array, n: int=..., *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def modulo(dividend, divisor, *, memory_pool: Incomplete | None=...): - ... - -def modulo_checked(dividend, divisor, *, memory_pool: Incomplete | None=...): - ... - -def month(values, *, memory_pool: Incomplete | None=...): - ... - -def month_day_nano_interval_between(start, end, *, memory_pool: Incomplete | None=...): - ... - -def month_interval_between(start, end, *, memory_pool: Incomplete | None=...): - ... - -def multiply(x, y, *, memory_pool: Incomplete | None=...): - ... - -def multiply_checked(x, y, *, memory_pool: Incomplete | None=...): - ... - -def nanosecond(values, *, memory_pool: Incomplete | None=...): - ... - -def nanoseconds_between(start, end, *, memory_pool: Incomplete | None=...): - ... - -def negate(x, *, memory_pool: Incomplete | None=...): - ... - -def negate_checked(x, *, memory_pool: Incomplete | None=...): - ... - -def not_equal(x, y, *, memory_pool: Incomplete | None=...): - ... - -def or_(x, y, *, memory_pool: Incomplete | None=...): - ... - -def or_kleene(x, y, *, memory_pool: Incomplete | None=...): - ... - -def pairwise_diff(input, period: int=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def pairwise_diff_checked(input, period: int=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def partition_nth_indices(array, pivot: int, *, null_placement: str=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def pivot_wider(pivot_keys, pivot_values, key_names: Sequence[str], *, unexpected_key_behavior: str=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def power(base, exponent, *, memory_pool: Incomplete | None=...): - ... - -def power_checked(base, exponent, *, memory_pool: Incomplete | None=...): - ... - -def product(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def quantile(array, q: float=..., *, interpolation: str=..., skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def quarter(values, *, memory_pool: Incomplete | None=...): - ... - -def quarters_between(start, end, *, memory_pool: Incomplete | None=...): - ... - -def random(n: int, *, initializer: str=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...) -> DoubleArray: - ... - -def rank(input, sort_keys: str=..., *, null_placement: Incomplete | None=..., tiebreaker: str=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def rank_normal(input, sort_keys: str=..., *, null_placement: Incomplete | None=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def rank_quantile(input, sort_keys: str=..., *, null_placement: Incomplete | None=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def remainder(dividend, divisor, *, memory_pool: Incomplete | None=...): - ... - -def remainder_checked(dividend, divisor, *, memory_pool: Incomplete | None=...): - ... - -def replace_substring(strings, pattern: str, replacement: str, *, max_replacements: Incomplete | None=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def replace_substring_regex(strings, pattern: str, replacement: str, *, max_replacements: Incomplete | None=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def replace_with_mask(values, mask, replacements, *, memory_pool: Incomplete | None=...): - ... - -def round(x, ndigits: int=..., round_mode: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def round_binary(x, s, round_mode: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def round_temporal(timestamps, multiple: int=..., unit: str=..., *, week_starts_monday: bool=..., ceil_is_strictly_greater: bool=..., calendar_based_origin: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def round_to_multiple(x, multiple: float=..., round_mode: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def run_end_decode(array, *, memory_pool: Incomplete | None=...): - ... - -def run_end_encode(array, run_end_type: pyarrow.lib.DataType=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def scatter(values, indices, max_index: int=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def search_sorted(values, needles, side: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def second(values, *, memory_pool: Incomplete | None=...): - ... - -def seconds_between(start, end, *, memory_pool: Incomplete | None=...): - ... - -def select_k_unstable(input, k: int, sort_keys, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def shift_left(x, y, *, memory_pool: Incomplete | None=...): - ... - -def shift_left_checked(x, y, *, memory_pool: Incomplete | None=...): - ... - -def shift_right(x, y, *, memory_pool: Incomplete | None=...): - ... - -def shift_right_checked(x, y, *, memory_pool: Incomplete | None=...): - ... - -def sign(x, *, memory_pool: Incomplete | None=...): - ... - -def sin(x, *, memory_pool: Incomplete | None=...): - ... - -def sin_checked(x, *, memory_pool: Incomplete | None=...): - ... - -def sinh(x, *, memory_pool: Incomplete | None=...): - ... - -def skew(array, *, skip_nulls: bool=..., biased: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def sort_indices(input, sort_keys: tuple=..., *, null_placement: Incomplete | None=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def split_pattern(strings, pattern: str, *, max_splits: Incomplete | None=..., reverse: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def split_pattern_regex(strings, pattern: str, *, max_splits: Incomplete | None=..., reverse: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def sqrt(x, *, memory_pool: Incomplete | None=...): - ... - -def sqrt_checked(x, *, memory_pool: Incomplete | None=...): - ... - -def starts_with(strings, pattern: str, *, ignore_case: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def stddev(array, *, ddof: int=..., skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def strftime(timestamps, format: str=..., locale: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def string_is_ascii(strings, *, memory_pool: Incomplete | None=...): - ... - -def strptime(strings, format: str, unit: str, error_is_null: bool=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def struct_field(values, indices: list[str] | list[bytes] | list[int] | Expression | bytes | str | int, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def subsecond(values, *, memory_pool: Incomplete | None=...): - ... - -def subtract(x, y, *, memory_pool: Incomplete | None=...): - ... - -def subtract_checked(x, y, *, memory_pool: Incomplete | None=...): - ... - -def sum(array, *, skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def take(data: pyarrow.Array | pyarrow.ChunkedArray | pyarrow.RecordBatch | pyarrow.Table, indices: pyarrow.Array | pyarrow.ChunkedArray, *, boundscheck: bool=..., memory_pool: Incomplete | None=...) -> Table | Int16Array | UInt64Array | UInt16Array | ChunkedArray | Int32Array | DictionaryArray | FloatArray | ListArray | NullArray | StringArray | Int8Array | UInt32Array | DoubleArray | LargeBinaryArray | LargeListArray | BooleanArray | ExtensionArray | BinaryArray | UInt8Array | Int64Array | FixedSizeBinaryArray | RecordBatch | StructArray: - ... - -def tan(x, *, memory_pool: Incomplete | None=...): - ... - -def tan_checked(x, *, memory_pool: Incomplete | None=...): - ... - -def tanh(x, *, memory_pool: Incomplete | None=...): - ... - -def tdigest(array, q: float=..., *, delta: int=..., buffer_size: int=..., skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def true_unless_null(values, *, memory_pool: Incomplete | None=...): - ... - -def trunc(x, *, memory_pool: Incomplete | None=...): - ... - -def unique(array, *, memory_pool: Incomplete | None=...): - ... - -def us_week(values, *, memory_pool: Incomplete | None=...): - ... - -def us_year(values, *, memory_pool: Incomplete | None=...): - ... - -def utf8_capitalize(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_center(strings, width: int, padding: str=..., lean_left_on_odd_padding: bool=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def utf8_is_alnum(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_is_alpha(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_is_decimal(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_is_digit(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_is_lower(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_is_numeric(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_is_printable(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_is_space(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_is_title(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_is_upper(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_length(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_lower(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_lpad(strings, width: int, padding: str=..., lean_left_on_odd_padding: bool=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def utf8_ltrim(strings, characters: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def utf8_ltrim_whitespace(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_normalize(strings, form: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def utf8_replace_slice(strings, start: int, stop: int, replacement: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def utf8_reverse(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_rpad(strings, width: int, padding: str=..., lean_left_on_odd_padding: bool=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def utf8_rtrim(strings, characters: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def utf8_rtrim_whitespace(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_slice_codeunits(strings, start: int, stop: Incomplete | None=..., step: int=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def utf8_split_whitespace(strings, *, max_splits: Incomplete | None=..., reverse: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def utf8_swapcase(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_title(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_trim(strings, characters: str, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def utf8_trim_whitespace(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_upper(strings, *, memory_pool: Incomplete | None=...): - ... - -def utf8_zero_fill(strings, width: int, padding: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def value_counts(array, *, memory_pool: Incomplete | None=...): - ... - -def variance(array, *, ddof: int=..., skip_nulls: bool=..., min_count: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def week(values, *, week_starts_monday: bool=..., count_from_zero: bool=..., first_week_is_fully_in_year: bool=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def weeks_between(start, end, *, count_from_zero: bool=..., week_start: int=..., options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def winsorize(array, lower_limit, upper_limit, *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def xor(x, y, *, memory_pool: Incomplete | None=...): - ... - -def year(values, *, memory_pool: Incomplete | None=...): - ... - -def year_month_day(values, *, memory_pool: Incomplete | None=...): - ... - -def years_between(start, end, *, memory_pool: Incomplete | None=...): - ... - -def utf8_zfill(strings, width: int, padding: str=..., *, options: Incomplete | None=..., memory_pool: Incomplete | None=...): - ... - -def fill_null(values: ChunkedArray | Int16Array | UInt64Array | StringArray | NullArray | UInt8Array | Int64Array | Int8Array | DoubleArray | Int32Array | FloatArray | LargeBinaryArray | UInt16Array | BooleanArray, fill_value: Int64Scalar | NullScalar | FloatScalar | bool | str | Int8Array | UInt16Scalar | UInt8Scalar | UInt64Scalar | DoubleScalar | Int16Scalar | int | Int8Scalar | None) -> ChunkedArray | Int16Array | UInt64Array | StringArray | NullArray | UInt8Array | Int64Array | Int8Array | DoubleArray | Int32Array | FloatArray | LargeBinaryArray | UInt16Array | BooleanArray: - ... - -def top_k_unstable(values: pyarrow.Array | pyarrow.ChunkedArray | pyarrow.RecordBatch | pyarrow.Table, k: int, sort_keys: Incomplete | None=..., null_placements: Incomplete | None=..., *, memory_pool: Incomplete | None=...) -> pyarrow.Array: - ... - -def bottom_k_unstable(values: pyarrow.Array | pyarrow.ChunkedArray | pyarrow.RecordBatch | pyarrow.Table, k: int, sort_keys: Incomplete | None=..., null_placements: Incomplete | None=..., *, memory_pool: Incomplete | None=...) -> UInt64Array: - ... - -def field(*name_or_index) -> Expression: - ... - -def scalar(value: bool | int | float | str) -> Expression: - ... +def abs(x, *, memory_pool: Incomplete | None = ...): ... +def abs_checked(x, *, memory_pool: Incomplete | None = ...): ... +def acos(x, *, memory_pool: Incomplete | None = ...): ... +def acos_checked(x, *, memory_pool: Incomplete | None = ...): ... +def acosh(x, *, memory_pool: Incomplete | None = ...): ... +def acosh_checked(x, *, memory_pool: Incomplete | None = ...): ... +def add(x, y, *, memory_pool: Incomplete | None = ...): ... +def add_checked(x, y, *, memory_pool: Incomplete | None = ...): ... +def all( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def and_(x, y, *, memory_pool: Incomplete | None = ...): ... +def and_kleene(x, y, *, memory_pool: Incomplete | None = ...): ... +def and_not(x, y, *, memory_pool: Incomplete | None = ...): ... +def and_not_kleene(x, y, *, memory_pool: Incomplete | None = ...): ... +def any( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def approximate_median( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def array_filter( + array, + selection_filter, + null_selection_behavior: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def array_sort_indices( + array, + order: str = ..., + *, + null_placement: str = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def array_take( + array, + indices, + *, + boundscheck: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def ascii_capitalize(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_center( + strings, + width: int, + padding: str = ..., + lean_left_on_odd_padding: bool = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def ascii_is_alnum(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_alpha(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_decimal(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_lower(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_printable(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_space(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_title(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_is_upper(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_lower(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_lpad( + strings, + width: int, + padding: str = ..., + lean_left_on_odd_padding: bool = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def ascii_ltrim( + strings, + characters: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def ascii_ltrim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_reverse(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_rpad( + strings, + width: int, + padding: str = ..., + lean_left_on_odd_padding: bool = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def ascii_rtrim( + strings, + characters: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def ascii_rtrim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_split_whitespace( + strings, + *, + max_splits: Incomplete | None = ..., + reverse: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def ascii_swapcase(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_title(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_trim( + strings, + characters: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def ascii_trim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def ascii_upper(strings, *, memory_pool: Incomplete | None = ...): ... +def asin(x, *, memory_pool: Incomplete | None = ...): ... +def asin_checked(x, *, memory_pool: Incomplete | None = ...): ... +def asinh(x, *, memory_pool: Incomplete | None = ...): ... +def assume_timezone( + timestamps, + timezone: str, + *, + ambiguous: str = ..., + nonexistent: str = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def atan(x, *, memory_pool: Incomplete | None = ...): ... +def atan2(y, x, *, memory_pool: Incomplete | None = ...): ... +def atanh(x, *, memory_pool: Incomplete | None = ...): ... +def atanh_checked(x, *, memory_pool: Incomplete | None = ...): ... +def binary_join(strings, separator, *, memory_pool: Incomplete | None = ...): ... +def binary_join_element_wise( + *strings, + null_handling: str = ..., + null_replacement: str = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def binary_length(strings, *, memory_pool: Incomplete | None = ...): ... +def binary_repeat(strings, num_repeats, *, memory_pool: Incomplete | None = ...): ... +def binary_replace_slice( + strings, + start: int, + stop: int, + replacement: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def binary_reverse(strings, *, memory_pool: Incomplete | None = ...): ... +def binary_slice( + strings, + start: int, + stop: Incomplete | None = ..., + step: int = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def bit_wise_and(x, y, *, memory_pool: Incomplete | None = ...): ... +def bit_wise_not(x, *, memory_pool: Incomplete | None = ...): ... +def bit_wise_or(x, y, *, memory_pool: Incomplete | None = ...): ... +def bit_wise_xor(x, y, *, memory_pool: Incomplete | None = ...): ... +def case_when(cond, *cases, memory_pool: Incomplete | None = ...): ... +def cast( + arr: Table + | Int16Array + | StringViewArray + | UInt64Array + | Date32Array + | JsonArray + | DurationArray + | UInt16Array + | HalfFloatArray + | Date64Array + | Int64Scalar + | ChunkedArray + | StringScalar + | FloatScalar + | FixedShapeTensorArray + | DoubleScalar + | Int32Array + | DictionaryArray + | FloatArray + | TimestampArray + | Decimal128Array + | LargeStringArray + | ListArray + | NullArray + | StringArray + | OpaqueArray + | BinaryScalar + | Int8Array + | UInt32Array + | MapArray + | DoubleArray + | Time32Array + | LargeListArray + | BooleanArray + | ExtensionArray + | BinaryArray + | TimestampScalar + | Bool8Array + | UInt8Array + | Int64Array + | FixedSizeBinaryArray + | Time64Array + | StructArray + | FixedSizeListArray + | Int8Scalar, + target_type: Incomplete | None = ..., + safe: Incomplete | None = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +) -> pyarrow.Array: ... +def ceil(x, *, memory_pool: Incomplete | None = ...): ... +def ceil_temporal( + timestamps, + multiple: int = ..., + unit: str = ..., + *, + week_starts_monday: bool = ..., + ceil_is_strictly_greater: bool = ..., + calendar_based_origin: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def choose(indices, *values, memory_pool: Incomplete | None = ...): ... +def coalesce(*values, memory_pool: Incomplete | None = ...): ... +def cos(x, *, memory_pool: Incomplete | None = ...): ... +def cos_checked(x, *, memory_pool: Incomplete | None = ...): ... +def cosh(x, *, memory_pool: Incomplete | None = ...): ... +def count( + array, + mode: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def count_distinct( + array, + mode: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def count_substring( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def count_substring_regex( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def cumulative_max( + values, + start: Incomplete | None = ..., + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def cumulative_mean( + values, + start: Incomplete | None = ..., + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def cumulative_min( + values, + start: Incomplete | None = ..., + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def cumulative_prod( + values, + start: Incomplete | None = ..., + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def cumulative_prod_checked( + values, + start: Incomplete | None = ..., + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def cumulative_sum( + values, + start: Incomplete | None = ..., + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def cumulative_sum_checked( + values, + start: Incomplete | None = ..., + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def day(values, *, memory_pool: Incomplete | None = ...): ... +def day_of_week( + values, + *, + count_from_zero: bool = ..., + week_start: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def day_of_year(values, *, memory_pool: Incomplete | None = ...): ... +def day_time_interval_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def days_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def dictionary_decode(dictionary_array, *, memory_pool: Incomplete | None = ...): ... +def dictionary_encode( + array, + null_encoding: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def divide(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def divide_checked(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def drop_null(input, *, memory_pool: Incomplete | None = ...): ... +def ends_with( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def equal(x, y, *, memory_pool: Incomplete | None = ...): ... +def exp(exponent, *, memory_pool: Incomplete | None = ...): ... +def expm1(exponent, *, memory_pool: Incomplete | None = ...): ... +def extract_regex( + strings, + pattern: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def extract_regex_span( + strings, + pattern: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def fill_null_backward(values, *, memory_pool: Incomplete | None = ...): ... +def fill_null_forward(values, *, memory_pool: Incomplete | None = ...): ... +def filter( + input, + selection_filter, + null_selection_behavior: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def find_substring( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def find_substring_regex( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def first( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def first_last( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def floor(x, *, memory_pool: Incomplete | None = ...): ... +def floor_temporal( + timestamps, + multiple: int = ..., + unit: str = ..., + *, + week_starts_monday: bool = ..., + ceil_is_strictly_greater: bool = ..., + calendar_based_origin: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def greater(x, y, *, memory_pool: Incomplete | None = ...): ... +def greater_equal(x, y, *, memory_pool: Incomplete | None = ...): ... +def hour(values, *, memory_pool: Incomplete | None = ...): ... +def hours_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def hypot(x, y, *, memory_pool: Incomplete | None = ...): ... +def if_else(cond, left, right, *, memory_pool: Incomplete | None = ...): ... +def index( + data: ChunkedArray | Int64Array, + value: Int64Scalar | int | Int8Scalar | None, + start: Incomplete | None = ..., + end: Incomplete | None = ..., + *, + memory_pool: Incomplete | None = ..., +) -> int: ... +def index_in( + values, + value_set: pyarrow.Array, + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def index_in_meta_binary( + values, value_set, *, memory_pool: Incomplete | None = ... +): ... +def indices_nonzero(values, *, memory_pool: Incomplete | None = ...): ... +def inverse_permutation( + indices, + max_index: int = ..., + output_type: Incomplete | None = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def invert(values, *, memory_pool: Incomplete | None = ...): ... +def is_dst(values, *, memory_pool: Incomplete | None = ...): ... +def is_finite(values, *, memory_pool: Incomplete | None = ...): ... +def is_in( + values, + value_set: pyarrow.Array, + *, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def is_in_meta_binary(values, value_set, *, memory_pool: Incomplete | None = ...): ... +def is_inf(values, *, memory_pool: Incomplete | None = ...): ... +def is_leap_year(values, *, memory_pool: Incomplete | None = ...): ... +def is_nan(values, *, memory_pool: Incomplete | None = ...): ... +def is_null( + values, + *, + nan_is_null: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def is_valid(values, *, memory_pool: Incomplete | None = ...): ... +def iso_calendar(values, *, memory_pool: Incomplete | None = ...): ... +def iso_week(values, *, memory_pool: Incomplete | None = ...): ... +def iso_year(values, *, memory_pool: Incomplete | None = ...): ... +def kurtosis( + array, + *, + skip_nulls: bool = ..., + biased: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def last( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def less(x, y, *, memory_pool: Incomplete | None = ...): ... +def less_equal(x, y, *, memory_pool: Incomplete | None = ...): ... +def list_element(lists, index, *, memory_pool: Incomplete | None = ...): ... +def list_flatten( + lists, + recursive: bool = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def list_parent_indices(lists, *, memory_pool: Incomplete | None = ...): ... +def list_slice( + lists, + start: int, + stop: Incomplete | None = ..., + step: int = ..., + return_fixed_size_list: Incomplete | None = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def list_value_length(lists, *, memory_pool: Incomplete | None = ...): ... +def ln(x, *, memory_pool: Incomplete | None = ...): ... +def ln_checked(x, *, memory_pool: Incomplete | None = ...): ... +def local_timestamp(values, *, memory_pool: Incomplete | None = ...): ... +def log10(x, *, memory_pool: Incomplete | None = ...): ... +def log10_checked(x, *, memory_pool: Incomplete | None = ...): ... +def log1p(x, *, memory_pool: Incomplete | None = ...): ... +def log1p_checked(x, *, memory_pool: Incomplete | None = ...): ... +def log2(x, *, memory_pool: Incomplete | None = ...): ... +def log2_checked(x, *, memory_pool: Incomplete | None = ...): ... +def logb(x, b, *, memory_pool: Incomplete | None = ...): ... +def logb_checked(x, b, *, memory_pool: Incomplete | None = ...): ... +def make_struct( + *args, + field_names: tuple = ..., + field_nullability: Incomplete | None = ..., + field_metadata: Incomplete | None = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def map_lookup( + container, + query_key, + occurrence: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def match_like( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def match_substring( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def match_substring_regex( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def max( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def max_element_wise( + *args, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def mean( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def microsecond(values, *, memory_pool: Incomplete | None = ...): ... +def microseconds_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def millisecond(values, *, memory_pool: Incomplete | None = ...): ... +def milliseconds_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def min( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def min_element_wise( + *args, + skip_nulls: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def min_max( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def minute(values, *, memory_pool: Incomplete | None = ...): ... +def minutes_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def mode( + array, + n: int = ..., + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def modulo(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def modulo_checked(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def month(values, *, memory_pool: Incomplete | None = ...): ... +def month_day_nano_interval_between( + start, end, *, memory_pool: Incomplete | None = ... +): ... +def month_interval_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def multiply(x, y, *, memory_pool: Incomplete | None = ...): ... +def multiply_checked(x, y, *, memory_pool: Incomplete | None = ...): ... +def nanosecond(values, *, memory_pool: Incomplete | None = ...): ... +def nanoseconds_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def negate(x, *, memory_pool: Incomplete | None = ...): ... +def negate_checked(x, *, memory_pool: Incomplete | None = ...): ... +def not_equal(x, y, *, memory_pool: Incomplete | None = ...): ... +def or_(x, y, *, memory_pool: Incomplete | None = ...): ... +def or_kleene(x, y, *, memory_pool: Incomplete | None = ...): ... +def pairwise_diff( + input, + period: int = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def pairwise_diff_checked( + input, + period: int = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def partition_nth_indices( + array, + pivot: int, + *, + null_placement: str = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def pivot_wider( + pivot_keys, + pivot_values, + key_names: Sequence[str], + *, + unexpected_key_behavior: str = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def power(base, exponent, *, memory_pool: Incomplete | None = ...): ... +def power_checked(base, exponent, *, memory_pool: Incomplete | None = ...): ... +def product( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def quantile( + array, + q: float = ..., + *, + interpolation: str = ..., + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def quarter(values, *, memory_pool: Incomplete | None = ...): ... +def quarters_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def random( + n: int, + *, + initializer: str = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +) -> DoubleArray: ... +def rank( + input, + sort_keys: str = ..., + *, + null_placement: Incomplete | None = ..., + tiebreaker: str = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def rank_normal( + input, + sort_keys: str = ..., + *, + null_placement: Incomplete | None = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def rank_quantile( + input, + sort_keys: str = ..., + *, + null_placement: Incomplete | None = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def remainder(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def remainder_checked(dividend, divisor, *, memory_pool: Incomplete | None = ...): ... +def replace_substring( + strings, + pattern: str, + replacement: str, + *, + max_replacements: Incomplete | None = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def replace_substring_regex( + strings, + pattern: str, + replacement: str, + *, + max_replacements: Incomplete | None = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def replace_with_mask( + values, mask, replacements, *, memory_pool: Incomplete | None = ... +): ... +def round( + x, + ndigits: int = ..., + round_mode: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def round_binary( + x, + s, + round_mode: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def round_temporal( + timestamps, + multiple: int = ..., + unit: str = ..., + *, + week_starts_monday: bool = ..., + ceil_is_strictly_greater: bool = ..., + calendar_based_origin: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def round_to_multiple( + x, + multiple: float = ..., + round_mode: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def run_end_decode(array, *, memory_pool: Incomplete | None = ...): ... +def run_end_encode( + array, + run_end_type: pyarrow.lib.DataType = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def scatter( + values, + indices, + max_index: int = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def search_sorted( + values, + needles, + side: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def second(values, *, memory_pool: Incomplete | None = ...): ... +def seconds_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def select_k_unstable( + input, + k: int, + sort_keys, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def shift_left(x, y, *, memory_pool: Incomplete | None = ...): ... +def shift_left_checked(x, y, *, memory_pool: Incomplete | None = ...): ... +def shift_right(x, y, *, memory_pool: Incomplete | None = ...): ... +def shift_right_checked(x, y, *, memory_pool: Incomplete | None = ...): ... +def sign(x, *, memory_pool: Incomplete | None = ...): ... +def sin(x, *, memory_pool: Incomplete | None = ...): ... +def sin_checked(x, *, memory_pool: Incomplete | None = ...): ... +def sinh(x, *, memory_pool: Incomplete | None = ...): ... +def skew( + array, + *, + skip_nulls: bool = ..., + biased: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def sort_indices( + input, + sort_keys: tuple = ..., + *, + null_placement: Incomplete | None = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def split_pattern( + strings, + pattern: str, + *, + max_splits: Incomplete | None = ..., + reverse: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def split_pattern_regex( + strings, + pattern: str, + *, + max_splits: Incomplete | None = ..., + reverse: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def sqrt(x, *, memory_pool: Incomplete | None = ...): ... +def sqrt_checked(x, *, memory_pool: Incomplete | None = ...): ... +def starts_with( + strings, + pattern: str, + *, + ignore_case: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def stddev( + array, + *, + ddof: int = ..., + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def strftime( + timestamps, + format: str = ..., + locale: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def string_is_ascii(strings, *, memory_pool: Incomplete | None = ...): ... +def strptime( + strings, + format: str, + unit: str, + error_is_null: bool = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def struct_field( + values, + indices: list[str] | list[bytes] | list[int] | Expression | bytes | str | int, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def subsecond(values, *, memory_pool: Incomplete | None = ...): ... +def subtract(x, y, *, memory_pool: Incomplete | None = ...): ... +def subtract_checked(x, y, *, memory_pool: Incomplete | None = ...): ... +def sum( + array, + *, + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def take( + data: pyarrow.Array | pyarrow.ChunkedArray | pyarrow.RecordBatch | pyarrow.Table, + indices: pyarrow.Array | pyarrow.ChunkedArray, + *, + boundscheck: bool = ..., + memory_pool: Incomplete | None = ..., +) -> ( + Table + | Int16Array + | UInt64Array + | UInt16Array + | ChunkedArray + | Int32Array + | DictionaryArray + | FloatArray + | ListArray + | NullArray + | StringArray + | Int8Array + | UInt32Array + | DoubleArray + | LargeBinaryArray + | LargeListArray + | BooleanArray + | ExtensionArray + | BinaryArray + | UInt8Array + | Int64Array + | FixedSizeBinaryArray + | RecordBatch + | StructArray +): ... +def tan(x, *, memory_pool: Incomplete | None = ...): ... +def tan_checked(x, *, memory_pool: Incomplete | None = ...): ... +def tanh(x, *, memory_pool: Incomplete | None = ...): ... +def tdigest( + array, + q: float = ..., + *, + delta: int = ..., + buffer_size: int = ..., + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def true_unless_null(values, *, memory_pool: Incomplete | None = ...): ... +def trunc(x, *, memory_pool: Incomplete | None = ...): ... +def unique(array, *, memory_pool: Incomplete | None = ...): ... +def us_week(values, *, memory_pool: Incomplete | None = ...): ... +def us_year(values, *, memory_pool: Incomplete | None = ...): ... +def utf8_capitalize(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_center( + strings, + width: int, + padding: str = ..., + lean_left_on_odd_padding: bool = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_is_alnum(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_alpha(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_decimal(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_digit(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_lower(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_numeric(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_printable(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_space(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_title(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_is_upper(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_length(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_lower(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_lpad( + strings, + width: int, + padding: str = ..., + lean_left_on_odd_padding: bool = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_ltrim( + strings, + characters: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_ltrim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_normalize( + strings, + form: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_replace_slice( + strings, + start: int, + stop: int, + replacement: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_reverse(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_rpad( + strings, + width: int, + padding: str = ..., + lean_left_on_odd_padding: bool = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_rtrim( + strings, + characters: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_rtrim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_slice_codeunits( + strings, + start: int, + stop: Incomplete | None = ..., + step: int = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_split_whitespace( + strings, + *, + max_splits: Incomplete | None = ..., + reverse: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_swapcase(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_title(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_trim( + strings, + characters: str, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def utf8_trim_whitespace(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_upper(strings, *, memory_pool: Incomplete | None = ...): ... +def utf8_zero_fill( + strings, + width: int, + padding: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def value_counts(array, *, memory_pool: Incomplete | None = ...): ... +def variance( + array, + *, + ddof: int = ..., + skip_nulls: bool = ..., + min_count: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def week( + values, + *, + week_starts_monday: bool = ..., + count_from_zero: bool = ..., + first_week_is_fully_in_year: bool = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def weeks_between( + start, + end, + *, + count_from_zero: bool = ..., + week_start: int = ..., + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def winsorize( + array, + lower_limit, + upper_limit, + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def xor(x, y, *, memory_pool: Incomplete | None = ...): ... +def year(values, *, memory_pool: Incomplete | None = ...): ... +def year_month_day(values, *, memory_pool: Incomplete | None = ...): ... +def years_between(start, end, *, memory_pool: Incomplete | None = ...): ... +def utf8_zfill( + strings, + width: int, + padding: str = ..., + *, + options: Incomplete | None = ..., + memory_pool: Incomplete | None = ..., +): ... +def fill_null( + values: ChunkedArray + | Int16Array + | UInt64Array + | StringArray + | NullArray + | UInt8Array + | Int64Array + | Int8Array + | DoubleArray + | Int32Array + | FloatArray + | LargeBinaryArray + | UInt16Array + | BooleanArray, + fill_value: Int64Scalar + | NullScalar + | FloatScalar + | bool + | str + | Int8Array + | UInt16Scalar + | UInt8Scalar + | UInt64Scalar + | DoubleScalar + | Int16Scalar + | int + | Int8Scalar + | None, +) -> ( + ChunkedArray + | Int16Array + | UInt64Array + | StringArray + | NullArray + | UInt8Array + | Int64Array + | Int8Array + | DoubleArray + | Int32Array + | FloatArray + | LargeBinaryArray + | UInt16Array + | BooleanArray +): ... +def top_k_unstable( + values: pyarrow.Array | pyarrow.ChunkedArray | pyarrow.RecordBatch | pyarrow.Table, + k: int, + sort_keys: Incomplete | None = ..., + null_placements: Incomplete | None = ..., + *, + memory_pool: Incomplete | None = ..., +) -> pyarrow.Array: ... +def bottom_k_unstable( + values: pyarrow.Array | pyarrow.ChunkedArray | pyarrow.RecordBatch | pyarrow.Table, + k: int, + sort_keys: Incomplete | None = ..., + null_placements: Incomplete | None = ..., + *, + memory_pool: Incomplete | None = ..., +) -> UInt64Array: ... +def field(*name_or_index) -> Expression: ... +def scalar(value: bool | float | str) -> Expression: ... class _OptionsClassDoc(tuple): _fields: ClassVar[tuple] = ... @@ -909,11 +1374,6 @@ class _OptionsClassDoc(tuple): __match_args__: ClassVar[tuple] = ... params: Incomplete - def __init__(self, _cls, params) -> None: - ... - - def __replace__(self, **kwds): - ... - - def __getnewargs__(self): - ... + def __init__(self, _cls, params) -> None: ... + def __replace__(self, **kwds): ... + def __getnewargs__(self): ... diff --git a/python/pyarrow-stubs/pyarrow/csv.pyi b/python/pyarrow-stubs/pyarrow/csv.pyi index 8020c3bbcd5e..2ea79dda6b89 100644 --- a/python/pyarrow-stubs/pyarrow/csv.pyi +++ b/python/pyarrow-stubs/pyarrow/csv.pyi @@ -15,9 +15,16 @@ # specific language governing permissions and limitations # under the License. -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete -from pyarrow._csv import CSVStreamingReader as CSVStreamingReader, CSVWriter as CSVWriter, ConvertOptions as ConvertOptions, ISO8601 as ISO8601, InvalidRow as InvalidRow, ParseOptions as ParseOptions, ReadOptions as ReadOptions, WriteOptions as WriteOptions, open_csv as open_csv, read_csv as read_csv, write_csv as write_csv +from pyarrow._csv import ( + CSVStreamingReader as CSVStreamingReader, + CSVWriter as CSVWriter, + ConvertOptions as ConvertOptions, + ISO8601 as ISO8601, + InvalidRow as InvalidRow, + ParseOptions as ParseOptions, + ReadOptions as ReadOptions, + WriteOptions as WriteOptions, + open_csv as open_csv, + read_csv as read_csv, + write_csv as write_csv, +) diff --git a/python/pyarrow-stubs/pyarrow/cuda.pyi b/python/pyarrow-stubs/pyarrow/cuda.pyi index d09469c99319..707a2e56d89f 100644 --- a/python/pyarrow-stubs/pyarrow/cuda.pyi +++ b/python/pyarrow-stubs/pyarrow/cuda.pyi @@ -15,4 +15,15 @@ # specific language governing permissions and limitations # under the License. -from pyarrow._cuda import BufferReader as BufferReader, BufferWriter as BufferWriter, Context as Context, CudaBuffer as CudaBuffer, HostBuffer as HostBuffer, IpcMemHandle as IpcMemHandle, new_host_buffer as new_host_buffer, read_message as read_message, read_record_batch as read_record_batch, serialize_record_batch as serialize_record_batch +from pyarrow._cuda import ( + BufferReader as BufferReader, + BufferWriter as BufferWriter, + Context as Context, + CudaBuffer as CudaBuffer, + HostBuffer as HostBuffer, + IpcMemHandle as IpcMemHandle, + new_host_buffer as new_host_buffer, + read_message as read_message, + read_record_batch as read_record_batch, + serialize_record_batch as serialize_record_batch, +) diff --git a/python/pyarrow-stubs/pyarrow/dataset.pyi b/python/pyarrow-stubs/pyarrow/dataset.pyi index e9cbe7950502..b59431d9bb16 100644 --- a/python/pyarrow-stubs/pyarrow/dataset.pyi +++ b/python/pyarrow-stubs/pyarrow/dataset.pyi @@ -15,64 +15,226 @@ # specific language governing permissions and limitations # under the License. -from pathlib import Path -from pyarrow._fs import FileSelector, LocalFileSystem, PyFileSystem, SubTreeFileSystem, _MockFileSystem +from pyarrow._fs import ( + FileSelector, + LocalFileSystem, + PyFileSystem, + SubTreeFileSystem, + _MockFileSystem, +) from pyarrow.lib import RecordBatch, RecordBatchReader, Schema, Table -from pyarrow.tests.util import FSProtocolClass -from typing import List, Optional, Tuple, Union import os -import re import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete +from typing import Any +from collections.abc import Callable import pyarrow.dataset import pyarrow.fs -from pyarrow._dataset import CsvFileFormat as CsvFileFormat, CsvFragmentScanOptions as CsvFragmentScanOptions, Dataset as Dataset, DatasetFactory as DatasetFactory, DirectoryPartitioning as DirectoryPartitioning, FeatherFileFormat as FeatherFileFormat, FileFormat as FileFormat, FileFragment as FileFragment, FileSystemDataset as FileSystemDataset, FileSystemDatasetFactory as FileSystemDatasetFactory, FileSystemFactoryOptions as FileSystemFactoryOptions, FileWriteOptions as FileWriteOptions, FilenamePartitioning as FilenamePartitioning, Fragment as Fragment, FragmentScanOptions as FragmentScanOptions, HivePartitioning as HivePartitioning, InMemoryDataset as InMemoryDataset, IpcFileFormat as IpcFileFormat, IpcFileWriteOptions as IpcFileWriteOptions, JsonFileFormat as JsonFileFormat, JsonFragmentScanOptions as JsonFragmentScanOptions, Partitioning as Partitioning, PartitioningFactory as PartitioningFactory, Scanner as Scanner, TaggedRecordBatch as TaggedRecordBatch, UnionDataset as UnionDataset, UnionDatasetFactory as UnionDatasetFactory, WrittenFile as WrittenFile, _filesystemdataset_write as _filesystemdataset_write, get_partition_keys as get_partition_keys +from pyarrow._dataset import ( + CsvFileFormat as CsvFileFormat, + CsvFragmentScanOptions as CsvFragmentScanOptions, + Dataset as Dataset, + DatasetFactory as DatasetFactory, + DirectoryPartitioning as DirectoryPartitioning, + FeatherFileFormat as FeatherFileFormat, + FileFormat as FileFormat, + FileFragment as FileFragment, + FileSystemDataset as FileSystemDataset, + FileSystemDatasetFactory as FileSystemDatasetFactory, + FileSystemFactoryOptions as FileSystemFactoryOptions, + FileWriteOptions as FileWriteOptions, + FilenamePartitioning as FilenamePartitioning, + Fragment as Fragment, + FragmentScanOptions as FragmentScanOptions, + HivePartitioning as HivePartitioning, + InMemoryDataset as InMemoryDataset, + IpcFileFormat as IpcFileFormat, + IpcFileWriteOptions as IpcFileWriteOptions, + JsonFileFormat as JsonFileFormat, + JsonFragmentScanOptions as JsonFragmentScanOptions, + Partitioning as Partitioning, + PartitioningFactory as PartitioningFactory, + Scanner as Scanner, + TaggedRecordBatch as TaggedRecordBatch, + UnionDataset as UnionDataset, + UnionDatasetFactory as UnionDatasetFactory, + WrittenFile as WrittenFile, + _filesystemdataset_write as _filesystemdataset_write, + get_partition_keys as get_partition_keys, +) from pyarrow._dataset_orc import OrcFileFormat as OrcFileFormat -from pyarrow._dataset_parquet import ParquetDatasetFactory as ParquetDatasetFactory, ParquetFactoryOptions as ParquetFactoryOptions, ParquetFileFormat as ParquetFileFormat, ParquetFileFragment as ParquetFileFragment, ParquetFileWriteOptions as ParquetFileWriteOptions, ParquetFragmentScanOptions as ParquetFragmentScanOptions, ParquetReadOptions as ParquetReadOptions, RowGroupInfo as RowGroupInfo -from pyarrow._dataset_parquet_encryption import ParquetDecryptionConfig as ParquetDecryptionConfig, ParquetEncryptionConfig as ParquetEncryptionConfig +from pyarrow._dataset_parquet import ( + ParquetDatasetFactory as ParquetDatasetFactory, + ParquetFactoryOptions as ParquetFactoryOptions, + ParquetFileFormat as ParquetFileFormat, + ParquetFileFragment as ParquetFileFragment, + ParquetFileWriteOptions as ParquetFileWriteOptions, + ParquetFragmentScanOptions as ParquetFragmentScanOptions, + ParquetReadOptions as ParquetReadOptions, + RowGroupInfo as RowGroupInfo, +) +from pyarrow._dataset_parquet_encryption import ( + ParquetDecryptionConfig as ParquetDecryptionConfig, + ParquetEncryptionConfig as ParquetEncryptionConfig, +) from pyarrow.compute import Expression as Expression, field as field, scalar as scalar -from pyarrow.util import _is_iterable as _is_iterable, _is_path_like as _is_path_like, _stringify_path as _stringify_path +from pyarrow.util import ( + _is_iterable as _is_iterable, + _is_path_like as _is_path_like, + _stringify_path as _stringify_path, +) + _orc_available: bool _orc_msg: str _parquet_available: bool _parquet_msg: str -def __getattr__(name: str) -> None: - ... - -def partitioning(schema: pyarrow.Schema | None=None, field_names: list[str] | None=None, flavor: str | None=None, dictionaries: str | None=None) -> Partitioning | PartitioningFactory: - ... - -def _ensure_partitioning(scheme: str | list[str] | PartitioningFactory | FilenamePartitioning | DirectoryPartitioning | int | HivePartitioning | None) -> FilenamePartitioning | PartitioningFactory | HivePartitioning | DirectoryPartitioning | None: - ... - -def _ensure_format(obj: str | OrcFileFormat | JsonFileFormat | IpcFileFormat | CsvFileFormat | ParquetFileFormat) -> OrcFileFormat | JsonFileFormat | IpcFileFormat | CsvFileFormat | FeatherFileFormat | ParquetFileFormat: - ... - -def _ensure_multiple_sources(paths: list[os.PathLike[str]], filesystem: pyarrow.fs.FileSystem | str | None=None) -> tuple[_MockFileSystem, list[str]] | tuple[LocalFileSystem, list[Any]] | tuple[LocalFileSystem, list[str]] | tuple[SubTreeFileSystem, list[str]]: - ... - -def _ensure_single_source(path: os.PathLike[str], filesystem: pyarrow.fs.FileSystem | str | None=None) -> tuple[PyFileSystem, FileSelector] | tuple[_MockFileSystem, FileSelector] | tuple[LocalFileSystem, list[str]] | tuple[LocalFileSystem, FileSelector] | tuple[SubTreeFileSystem, FileSelector] | tuple[SubTreeFileSystem, list[str]]: - ... - -def _filesystem_dataset(source, schema: Schema | None=None, filesystem: _MockFileSystem | PyFileSystem | str | SubTreeFileSystem | LocalFileSystem | None=None, partitioning: FilenamePartitioning | str | list[str] | PartitioningFactory | HivePartitioning | DirectoryPartitioning | int | None=None, format: str | OrcFileFormat | JsonFileFormat | IpcFileFormat | CsvFileFormat | ParquetFileFormat | None=None, partition_base_dir=None, exclude_invalid_files=None, selector_ignore_prefixes: list[str] | None=None) -> FileSystemDataset: - ... - -def _in_memory_dataset(source: Table | list[RecordBatch] | RecordBatch | list[Table] | RecordBatchReader, schema=None, **kwargs) -> InMemoryDataset: - ... - -def _union_dataset(children: list[FileSystemDataset] | tuple[InMemoryDataset, InMemoryDataset] | tuple[FileSystemDataset, FileSystemDataset], schema: Schema | None=None, **kwargs) -> UnionDataset: - ... - -def parquet_dataset(metadata_path: str, schema: pyarrow.Schema | None=None, filesystem=None, format: ParquetFileFormat | None=None, partitioning: Partitioning | PartitioningFactory | str | list[str] | None=None, partition_base_dir: str | None=None) -> FileSystemDataset: - ... - -def dataset(source, schema: pyarrow.Schema | None=None, format: FileFormat | str | None=None, filesystem: _MockFileSystem | PyFileSystem | str | SubTreeFileSystem | LocalFileSystem | None=None, partitioning: Partitioning | PartitioningFactory | str | list[str] | None=None, partition_base_dir: str | None=None, exclude_invalid_files: bool | None=None, ignore_prefixes: list | None=None) -> Dataset: - ... - -def _ensure_write_partitioning(part: list[str] | FilenamePartitioning | DirectoryPartitioning | HivePartitioning | None, schema: Schema, flavor: str | None) -> DirectoryPartitioning | HivePartitioning | FilenamePartitioning: - ... - -def write_dataset(data: Table | FileSystemDataset | Scanner | list[Table] | InMemoryDataset, base_dir: str, *, basename_template: str | None=None, format: FileFormat | str | None=None, partitioning: Partitioning | list[str] | None=None, partitioning_flavor: str | None=None, schema: pyarrow.Schema | None=None, filesystem: pyarrow.fs.FileSystem | None=None, file_options: pyarrow.dataset.FileWriteOptions | None=None, use_threads: bool=True, preserve_order: bool=False, max_partitions: int | None=None, max_open_files: int | None=None, max_rows_per_file: int | None=None, min_rows_per_group: int | None=None, max_rows_per_group: int | None=None, file_visitor: Callable[..., Any] | None=None, existing_data_behavior: str='error', create_dir: bool=True) -> None: - ... +def __getattr__(name: str) -> None: ... +def partitioning( + schema: pyarrow.Schema | None = None, + field_names: list[str] | None = None, + flavor: str | None = None, + dictionaries: str | None = None, +) -> Partitioning | PartitioningFactory: ... +def _ensure_partitioning( + scheme: str + | list[str] + | PartitioningFactory + | FilenamePartitioning + | DirectoryPartitioning + | int + | HivePartitioning + | None, +) -> ( + FilenamePartitioning + | PartitioningFactory + | HivePartitioning + | DirectoryPartitioning + | None +): ... +def _ensure_format( + obj: str + | OrcFileFormat + | JsonFileFormat + | IpcFileFormat + | CsvFileFormat + | ParquetFileFormat, +) -> ( + OrcFileFormat + | JsonFileFormat + | IpcFileFormat + | CsvFileFormat + | FeatherFileFormat + | ParquetFileFormat +): ... +def _ensure_multiple_sources( + paths: list[os.PathLike[str]], filesystem: pyarrow.fs.FileSystem | str | None = None +) -> ( + tuple[_MockFileSystem, list[str]] + | tuple[LocalFileSystem, list[Any]] + | tuple[LocalFileSystem, list[str]] + | tuple[SubTreeFileSystem, list[str]] +): ... +def _ensure_single_source( + path: os.PathLike[str], filesystem: pyarrow.fs.FileSystem | str | None = None +) -> ( + tuple[PyFileSystem, FileSelector] + | tuple[_MockFileSystem, FileSelector] + | tuple[LocalFileSystem, list[str]] + | tuple[LocalFileSystem, FileSelector] + | tuple[SubTreeFileSystem, FileSelector] + | tuple[SubTreeFileSystem, list[str]] +): ... +def _filesystem_dataset( + source, + schema: Schema | None = None, + filesystem: _MockFileSystem + | PyFileSystem + | str + | SubTreeFileSystem + | LocalFileSystem + | None = None, + partitioning: FilenamePartitioning + | str + | list[str] + | PartitioningFactory + | HivePartitioning + | DirectoryPartitioning + | int + | None = None, + format: str + | OrcFileFormat + | JsonFileFormat + | IpcFileFormat + | CsvFileFormat + | ParquetFileFormat + | None = None, + partition_base_dir=None, + exclude_invalid_files=None, + selector_ignore_prefixes: list[str] | None = None, +) -> FileSystemDataset: ... +def _in_memory_dataset( + source: Table | list[RecordBatch] | RecordBatch | list[Table] | RecordBatchReader, + schema=None, + **kwargs, +) -> InMemoryDataset: ... +def _union_dataset( + children: list[FileSystemDataset] + | tuple[InMemoryDataset, InMemoryDataset] + | tuple[FileSystemDataset, FileSystemDataset], + schema: Schema | None = None, + **kwargs, +) -> UnionDataset: ... +def parquet_dataset( + metadata_path: str, + schema: pyarrow.Schema | None = None, + filesystem=None, + format: ParquetFileFormat | None = None, + partitioning: Partitioning | PartitioningFactory | str | list[str] | None = None, + partition_base_dir: str | None = None, +) -> FileSystemDataset: ... +def dataset( + source, + schema: pyarrow.Schema | None = None, + format: FileFormat | str | None = None, + filesystem: _MockFileSystem + | PyFileSystem + | str + | SubTreeFileSystem + | LocalFileSystem + | None = None, + partitioning: Partitioning | PartitioningFactory | str | list[str] | None = None, + partition_base_dir: str | None = None, + exclude_invalid_files: bool | None = None, + ignore_prefixes: list | None = None, +) -> Dataset: ... +def _ensure_write_partitioning( + part: list[str] + | FilenamePartitioning + | DirectoryPartitioning + | HivePartitioning + | None, + schema: Schema, + flavor: str | None, +) -> DirectoryPartitioning | HivePartitioning | FilenamePartitioning: ... +def write_dataset( + data: Table | FileSystemDataset | Scanner | list[Table] | InMemoryDataset, + base_dir: str, + *, + basename_template: str | None = None, + format: FileFormat | str | None = None, + partitioning: Partitioning | list[str] | None = None, + partitioning_flavor: str | None = None, + schema: pyarrow.Schema | None = None, + filesystem: pyarrow.fs.FileSystem | None = None, + file_options: pyarrow.dataset.FileWriteOptions | None = None, + use_threads: bool = True, + preserve_order: bool = False, + max_partitions: int | None = None, + max_open_files: int | None = None, + max_rows_per_file: int | None = None, + min_rows_per_group: int | None = None, + max_rows_per_group: int | None = None, + file_visitor: Callable[..., Any] | None = None, + existing_data_behavior: str = "error", + create_dir: bool = True, +) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/feather.pyi b/python/pyarrow-stubs/pyarrow/feather.pyi index 8e824a3a4a53..548d319c8506 100644 --- a/python/pyarrow-stubs/pyarrow/feather.pyi +++ b/python/pyarrow-stubs/pyarrow/feather.pyi @@ -18,47 +18,63 @@ from io import BytesIO from pathlib import Path from pyarrow.lib import ChunkedArray -from typing import List, Optional, Union -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from typing import Any +from collections.abc import Iterator, Sequence from _typeshed import Incomplete from pyarrow import _feather as _feather from pyarrow._feather import FeatherError as FeatherError -from pyarrow.lib import Codec as Codec, Table as Table, concat_tables as concat_tables, schema as schema +from pyarrow.lib import ( + Codec as Codec, + Table as Table, + concat_tables as concat_tables, + schema as schema, +) from pyarrow.pandas_compat import _pandas_api as _pandas_api -def check_chunked_overflow(name: str, col: ChunkedArray) -> None: - ... -_FEATHER_SUPPORTED_CODECS: Incomplete - -def write_feather(df, dest: str, compression: str | None=None, compression_level: int | None=None, chunksize: int | None=None, version: int=2) -> None: - ... - -def read_feather(source: BytesIO | Path | str, columns: Sequence[Any] | None=None, use_threads: bool=True, memory_map: bool=False, **kwargs): - ... +def check_chunked_overflow(name: str, col: ChunkedArray) -> None: ... -def _read_table_internal(source: Path | BytesIO | str, columns: list[int] | Iterator[Any] | list[str] | None=None, memory_map: bool=False, use_threads: bool=True) -> Table: - ... +_FEATHER_SUPPORTED_CODECS: Incomplete -def read_table(source: Path | str, columns: Sequence[Any] | None=None, memory_map: bool=False, use_threads: bool=True) -> Table: - ... +def write_feather( + df, + dest: str, + compression: str | None = None, + compression_level: int | None = None, + chunksize: int | None = None, + version: int = 2, +) -> None: ... +def read_feather( + source: BytesIO | Path | str, + columns: Sequence[Any] | None = None, + use_threads: bool = True, + memory_map: bool = False, + **kwargs, +): ... +def _read_table_internal( + source: Path | BytesIO | str, + columns: list[int] | Iterator[Any] | list[str] | None = None, + memory_map: bool = False, + use_threads: bool = True, +) -> Table: ... +def read_table( + source: Path | str, + columns: Sequence[Any] | None = None, + memory_map: bool = False, + use_threads: bool = True, +) -> Table: ... class FeatherDataset: paths: Incomplete validate_schema: Incomplete - def __init__(self, path_or_paths: list[str], validate_schema: bool=True) -> None: - ... + def __init__( + self, path_or_paths: list[str], validate_schema: bool = True + ) -> None: ... _tables: Incomplete schema: Incomplete - def read_table(self, columns: list[str] | None=None) -> Table: - ... - - def validate_schemas(self, piece: str, table: Table) -> None: - ... - - def read_pandas(self, columns: list[str] | None=None, use_threads: bool=True): - ... + def read_table(self, columns: list[str] | None = None) -> Table: ... + def validate_schemas(self, piece: str, table: Table) -> None: ... + def read_pandas( + self, columns: list[str] | None = None, use_threads: bool = True + ): ... diff --git a/python/pyarrow-stubs/pyarrow/flight.pyi b/python/pyarrow-stubs/pyarrow/flight.pyi index dcebdef09c4a..2ce10eeaf679 100644 --- a/python/pyarrow-stubs/pyarrow/flight.pyi +++ b/python/pyarrow-stubs/pyarrow/flight.pyi @@ -15,9 +15,50 @@ # specific language governing permissions and limitations # under the License. -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete -from pyarrow._flight import Action as Action, ActionType as ActionType, BasicAuth as BasicAuth, CallInfo as CallInfo, CertKeyPair as CertKeyPair, ClientAuthHandler as ClientAuthHandler, ClientMiddleware as ClientMiddleware, ClientMiddlewareFactory as ClientMiddlewareFactory, DescriptorType as DescriptorType, FlightCallOptions as FlightCallOptions, FlightCancelledError as FlightCancelledError, FlightClient as FlightClient, FlightDataStream as FlightDataStream, FlightDescriptor as FlightDescriptor, FlightEndpoint as FlightEndpoint, FlightError as FlightError, FlightInfo as FlightInfo, FlightInternalError as FlightInternalError, FlightMetadataReader as FlightMetadataReader, FlightMetadataWriter as FlightMetadataWriter, FlightMethod as FlightMethod, FlightServerBase as FlightServerBase, FlightServerError as FlightServerError, FlightStreamChunk as FlightStreamChunk, FlightStreamReader as FlightStreamReader, FlightStreamWriter as FlightStreamWriter, FlightTimedOutError as FlightTimedOutError, FlightUnauthenticatedError as FlightUnauthenticatedError, FlightUnauthorizedError as FlightUnauthorizedError, FlightUnavailableError as FlightUnavailableError, FlightWriteSizeExceededError as FlightWriteSizeExceededError, GeneratorStream as GeneratorStream, Location as Location, MetadataRecordBatchReader as MetadataRecordBatchReader, MetadataRecordBatchWriter as MetadataRecordBatchWriter, RecordBatchStream as RecordBatchStream, Result as Result, SchemaResult as SchemaResult, ServerAuthHandler as ServerAuthHandler, ServerCallContext as ServerCallContext, ServerMiddleware as ServerMiddleware, ServerMiddlewareFactory as ServerMiddlewareFactory, Ticket as Ticket, TracingServerMiddlewareFactory as TracingServerMiddlewareFactory, connect as connect +from pyarrow._flight import ( + Action as Action, + ActionType as ActionType, + BasicAuth as BasicAuth, + CallInfo as CallInfo, + CertKeyPair as CertKeyPair, + ClientAuthHandler as ClientAuthHandler, + ClientMiddleware as ClientMiddleware, + ClientMiddlewareFactory as ClientMiddlewareFactory, + DescriptorType as DescriptorType, + FlightCallOptions as FlightCallOptions, + FlightCancelledError as FlightCancelledError, + FlightClient as FlightClient, + FlightDataStream as FlightDataStream, + FlightDescriptor as FlightDescriptor, + FlightEndpoint as FlightEndpoint, + FlightError as FlightError, + FlightInfo as FlightInfo, + FlightInternalError as FlightInternalError, + FlightMetadataReader as FlightMetadataReader, + FlightMetadataWriter as FlightMetadataWriter, + FlightMethod as FlightMethod, + FlightServerBase as FlightServerBase, + FlightServerError as FlightServerError, + FlightStreamChunk as FlightStreamChunk, + FlightStreamReader as FlightStreamReader, + FlightStreamWriter as FlightStreamWriter, + FlightTimedOutError as FlightTimedOutError, + FlightUnauthenticatedError as FlightUnauthenticatedError, + FlightUnauthorizedError as FlightUnauthorizedError, + FlightUnavailableError as FlightUnavailableError, + FlightWriteSizeExceededError as FlightWriteSizeExceededError, + GeneratorStream as GeneratorStream, + Location as Location, + MetadataRecordBatchReader as MetadataRecordBatchReader, + MetadataRecordBatchWriter as MetadataRecordBatchWriter, + RecordBatchStream as RecordBatchStream, + Result as Result, + SchemaResult as SchemaResult, + ServerAuthHandler as ServerAuthHandler, + ServerCallContext as ServerCallContext, + ServerMiddleware as ServerMiddleware, + ServerMiddlewareFactory as ServerMiddlewareFactory, + Ticket as Ticket, + TracingServerMiddlewareFactory as TracingServerMiddlewareFactory, + connect as connect, +) diff --git a/python/pyarrow-stubs/pyarrow/fs.pyi b/python/pyarrow-stubs/pyarrow/fs.pyi index e9040396d0cb..4e183355b039 100644 --- a/python/pyarrow-stubs/pyarrow/fs.pyi +++ b/python/pyarrow-stubs/pyarrow/fs.pyi @@ -16,96 +16,112 @@ # under the License. from io import BufferedReader, BufferedWriter, BytesIO -from pathlib import Path from pyarrow.lib import Buffer, BufferOutputStream, BufferReader, NativeFile, OSFile -from pyarrow.tests.util import FSProtocolClass -from typing import Optional, Tuple, Union -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from typing import Any +from collections.abc import Mapping from _typeshed import Incomplete from pyarrow._azurefs import AzureFileSystem as AzureFileSystem -from pyarrow._fs import FileInfo as FileInfo, FileSelector as FileSelector, FileSystem as FileSystem, FileSystemHandler as FileSystemHandler, FileType as FileType, LocalFileSystem as LocalFileSystem, PyFileSystem as PyFileSystem, SubTreeFileSystem as SubTreeFileSystem, _MockFileSystem as _MockFileSystem, _copy_files as _copy_files, _copy_files_selector as _copy_files_selector +from pyarrow._fs import ( + FileInfo as FileInfo, + FileSelector as FileSelector, + FileSystem as FileSystem, + FileSystemHandler as FileSystemHandler, + FileType as FileType, + LocalFileSystem as LocalFileSystem, + PyFileSystem as PyFileSystem, + SubTreeFileSystem as SubTreeFileSystem, + _MockFileSystem as _MockFileSystem, + _copy_files as _copy_files, + _copy_files_selector as _copy_files_selector, +) from pyarrow._gcsfs import GcsFileSystem as GcsFileSystem from pyarrow._hdfs import HadoopFileSystem as HadoopFileSystem -from pyarrow._s3fs import AwsDefaultS3RetryStrategy as AwsDefaultS3RetryStrategy, AwsStandardS3RetryStrategy as AwsStandardS3RetryStrategy, S3FileSystem as S3FileSystem, S3LogLevel as S3LogLevel, S3RetryStrategy as S3RetryStrategy, ensure_s3_finalized as ensure_s3_finalized, ensure_s3_initialized as ensure_s3_initialized, finalize_s3 as finalize_s3, initialize_s3 as initialize_s3, resolve_s3_region as resolve_s3_region -from pyarrow.util import _is_path_like as _is_path_like, _stringify_path as _stringify_path +from pyarrow._s3fs import ( + AwsDefaultS3RetryStrategy as AwsDefaultS3RetryStrategy, + AwsStandardS3RetryStrategy as AwsStandardS3RetryStrategy, + S3FileSystem as S3FileSystem, + S3LogLevel as S3LogLevel, + S3RetryStrategy as S3RetryStrategy, + ensure_s3_finalized as ensure_s3_finalized, + ensure_s3_initialized as ensure_s3_initialized, + finalize_s3 as finalize_s3, + initialize_s3 as initialize_s3, + resolve_s3_region as resolve_s3_region, +) +from pyarrow.util import ( + _is_path_like as _is_path_like, + _stringify_path as _stringify_path, +) + FileStats = FileInfo _not_imported: Incomplete -def __getattr__(name: str) -> None: - ... - -def _ensure_filesystem(filesystem: PyFileSystem | _MockFileSystem | str | SubTreeFileSystem | LocalFileSystem, *, use_mmap: bool=False) -> PyFileSystem | SubTreeFileSystem | _MockFileSystem | LocalFileSystem: - ... - -def _resolve_filesystem_and_path(path, filesystem: _MockFileSystem | PyFileSystem | str | SubTreeFileSystem | LocalFileSystem | None=None, *, memory_map: bool=False) -> tuple[PyFileSystem, str] | tuple[None, BufferedReader] | tuple[None, NativeFile] | tuple[None, BufferOutputStream] | tuple[LocalFileSystem, str] | tuple[_MockFileSystem, str] | tuple[None, None] | tuple[None, BytesIO] | tuple[None, OSFile] | tuple[SubTreeFileSystem, str] | tuple[None, Buffer] | tuple[None, BufferedWriter] | tuple[None, BufferReader]: - ... - -def copy_files(source: str, destination: str, source_filesystem: FileSystem | None=None, destination_filesystem: FileSystem | None=None, *, chunk_size: int=..., use_threads: bool=True) -> None: - ... +def __getattr__(name: str) -> None: ... +def _ensure_filesystem( + filesystem: PyFileSystem + | _MockFileSystem + | str + | SubTreeFileSystem + | LocalFileSystem, + *, + use_mmap: bool = False, +) -> PyFileSystem | SubTreeFileSystem | _MockFileSystem | LocalFileSystem: ... +def _resolve_filesystem_and_path( + path, + filesystem: _MockFileSystem + | PyFileSystem + | str + | SubTreeFileSystem + | LocalFileSystem + | None = None, + *, + memory_map: bool = False, +) -> ( + tuple[PyFileSystem, str] + | tuple[None, BufferedReader] + | tuple[None, NativeFile] + | tuple[None, BufferOutputStream] + | tuple[LocalFileSystem, str] + | tuple[_MockFileSystem, str] + | tuple[None, None] + | tuple[None, BytesIO] + | tuple[None, OSFile] + | tuple[SubTreeFileSystem, str] + | tuple[None, Buffer] + | tuple[None, BufferedWriter] + | tuple[None, BufferReader] +): ... +def copy_files( + source: str, + destination: str, + source_filesystem: FileSystem | None = None, + destination_filesystem: FileSystem | None = None, + *, + chunk_size: int = ..., + use_threads: bool = True, +) -> None: ... class FSSpecHandler(FileSystemHandler): fs: Incomplete - def __init__(self, fs) -> None: - ... - - def __eq__(self, other): - ... - - def __ne__(self, other): - ... - - def get_type_name(self): - ... - - def normalize_path(self, path: str): - ... - + def __init__(self, fs) -> None: ... + def __eq__(self, other): ... + def __ne__(self, other): ... + def get_type_name(self): ... + def normalize_path(self, path: str): ... @staticmethod - def _create_file_info(path, info): - ... - - def get_file_info(self, paths: list[str]): - ... - - def get_file_info_selector(self, selector: FileSelector): - ... - - def create_dir(self, path: str, recursive: bool) -> None: - ... - - def delete_dir(self, path: str) -> None: - ... - - def _delete_dir_contents(self, path, missing_dir_ok) -> None: - ... - - def delete_dir_contents(self, path: str, missing_dir_ok: bool) -> None: - ... - - def delete_root_dir_contents(self) -> None: - ... - - def delete_file(self, path: str) -> None: - ... - - def move(self, src: str, dest: str) -> None: - ... - - def copy_file(self, src: str, dest: str) -> None: - ... - - def open_input_stream(self, path: str): - ... - - def open_input_file(self, path: str): - ... - - def open_output_stream(self, path: str, metadata: Mapping[Any, Any]): - ... - - def open_append_stream(self, path: str, metadata: Mapping[Any, Any]): - ... + def _create_file_info(path, info): ... + def get_file_info(self, paths: list[str]): ... + def get_file_info_selector(self, selector: FileSelector): ... + def create_dir(self, path: str, recursive: bool) -> None: ... + def delete_dir(self, path: str) -> None: ... + def _delete_dir_contents(self, path, missing_dir_ok) -> None: ... + def delete_dir_contents(self, path: str, missing_dir_ok: bool) -> None: ... + def delete_root_dir_contents(self) -> None: ... + def delete_file(self, path: str) -> None: ... + def move(self, src: str, dest: str) -> None: ... + def copy_file(self, src: str, dest: str) -> None: ... + def open_input_stream(self, path: str): ... + def open_input_file(self, path: str): ... + def open_output_stream(self, path: str, metadata: Mapping[Any, Any]): ... + def open_append_stream(self, path: str, metadata: Mapping[Any, Any]): ... diff --git a/python/pyarrow-stubs/pyarrow/gandiva.pyi b/python/pyarrow-stubs/pyarrow/gandiva.pyi index 13d0aa18c25f..a80e03671e58 100644 --- a/python/pyarrow-stubs/pyarrow/gandiva.pyi +++ b/python/pyarrow-stubs/pyarrow/gandiva.pyi @@ -16,12 +16,9 @@ # under the License. # Generated by stubgen-pyx from python/pyarrow/gandiva.pyx -from _typeshed import Incomplete from pyarrow.includes.libarrow import * -from pyarrow.lib import (DataType, Field, MemoryPool, RecordBatch, Schema, - _Weakrefable) - +from pyarrow.lib import DataType, Field, MemoryPool, RecordBatch, Schema, _Weakrefable class Node(_Weakrefable): def __init__(self): ... @@ -50,20 +47,24 @@ class Projector(_Weakrefable): def __init__(self): ... @property def llvm_ir(self): ... - def evaluate(self, batch: RecordBatch, selection: SelectionVector | None=None): ... + def evaluate( + self, batch: RecordBatch, selection: SelectionVector | None = None + ): ... class Filter(_Weakrefable): def __init__(self): ... @property def llvm_ir(self): ... - def evaluate(self, batch: RecordBatch, pool: MemoryPool, dtype='int32'): ... + def evaluate(self, batch: RecordBatch, pool: MemoryPool, dtype="int32"): ... class TreeExprBuilder(_Weakrefable): def make_literal(self, value, dtype): ... def make_expression(self, root_node: Node, return_field: Field): ... def make_function(self, name, children, return_type: DataType): ... def make_field(self, field: Field): ... - def make_if(self, condition: Node, this_node: Node, else_node: Node, return_type: DataType): ... + def make_if( + self, condition: Node, this_node: Node, else_node: Node, return_type: DataType + ): ... def make_and(self, children): ... def make_or(self, children): ... def _make_in_expression_int32(self, node: Node, values): ... @@ -79,7 +80,7 @@ class TreeExprBuilder(_Weakrefable): def make_condition(self, condition: Node): ... class Configuration(_Weakrefable): - def __init__(self, optimize: bool=True, dump_ir: bool=False): ... + def __init__(self, optimize: bool = True, dump_ir: bool = False): ... class FunctionSignature(_Weakrefable): def __init__(self): ... @@ -88,6 +89,14 @@ class FunctionSignature(_Weakrefable): def name(self): ... def __repr__(self): ... -def make_projector(schema: Schema, children, pool: MemoryPool, selection_mode: str='NONE', configuration: Configuration | None=None): ... -def make_filter(schema: Schema, condition: Condition, configuration: Configuration | None=None): ... +def make_projector( + schema: Schema, + children, + pool: MemoryPool, + selection_mode: str = "NONE", + configuration: Configuration | None = None, +): ... +def make_filter( + schema: Schema, condition: Condition, configuration: Configuration | None = None +): ... def get_registered_function_signatures(): ... diff --git a/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi b/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi index c5cb5ed212af..3d04612d9651 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/__init__.pyi @@ -15,9 +15,4 @@ # specific language governing permissions and limitations # under the License. -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete from .from_dataframe import from_dataframe as from_dataframe diff --git a/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi b/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi index 84b19a6e967b..4fa39a8cdbf1 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/buffer.pyi @@ -15,10 +15,6 @@ # specific language governing permissions and limitations # under the License. -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence import enum import pyarrow as pa from _typeshed import Incomplete @@ -36,22 +32,10 @@ class DlpackDeviceType(enum.IntEnum): class _PyArrowBuffer: _x: Incomplete - def __init__(self, x: pa.Buffer, allow_copy: bool=True) -> None: - ... - + def __init__(self, x: pa.Buffer, allow_copy: bool = True) -> None: ... @property - def bufsize(self) -> int: - ... - + def bufsize(self) -> int: ... @property - def ptr(self) -> int: - ... - - def __dlpack__(self) -> None: - ... - - def __dlpack_device__(self) -> tuple[DlpackDeviceType, int | None]: - ... - - def __repr__(self) -> str: - ... + def ptr(self) -> int: ... + def __dlpack__(self) -> None: ... + def __dlpack_device__(self) -> tuple[DlpackDeviceType, int | None]: ... diff --git a/python/pyarrow-stubs/pyarrow/interchange/column.pyi b/python/pyarrow-stubs/pyarrow/interchange/column.pyi index 6461bfafa1ca..5790a3f9d05e 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/column.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/column.pyi @@ -15,16 +15,13 @@ # specific language governing permissions and limitations # under the License. -from typing import Optional -import os -import re -import pyarrow -from typing import Callable, IO, Iterator, Literal, Mapping, NamedTuple, Self, Sequence import enum import pyarrow as pa from _typeshed import Incomplete from pyarrow.interchange.buffer import _PyArrowBuffer as _PyArrowBuffer -from typing import Any, Iterable, TypedDict +from typing import Any, TypedDict, TypeAlias +from collections.abc import Iterable + _PYARROW_KINDS: Incomplete class DtypeKind(enum.IntEnum): @@ -35,7 +32,8 @@ class DtypeKind(enum.IntEnum): STRING = 21 DATETIME = 22 CATEGORICAL = 23 -Dtype = tuple[DtypeKind, int, str, str] + +Dtype: TypeAlias = tuple[DtypeKind, int, str, str] class ColumnNullType(enum.IntEnum): NON_NULLABLE = 0 @@ -60,61 +58,35 @@ class Endianness: NATIVE: str NA: str -class NoBufferPresent(Exception): - ... +class NoBufferPresent(Exception): ... class _PyArrowColumn: _allow_copy: Incomplete _dtype: Incomplete _col: Incomplete - def __init__(self, column: pa.Array | pa.ChunkedArray, allow_copy: bool=True) -> None: - ... - - def size(self) -> int: - ... - + def __init__( + self, column: pa.Array | pa.ChunkedArray, allow_copy: bool = True + ) -> None: ... + def size(self) -> int: ... @property - def offset(self) -> int: - ... - + def offset(self) -> int: ... @property - def dtype(self) -> tuple[DtypeKind, int, str, str]: - ... - - def _dtype_from_arrowdtype(self, dtype: pa.DataType, bit_width: int) -> tuple[DtypeKind, int, str, str]: - ... - + def dtype(self) -> tuple[DtypeKind, int, str, str]: ... + def _dtype_from_arrowdtype( + self, dtype: pa.DataType, bit_width: int + ) -> tuple[DtypeKind, int, str, str]: ... @property - def describe_categorical(self) -> CategoricalDescription: - ... - + def describe_categorical(self) -> CategoricalDescription: ... @property - def describe_null(self) -> tuple[ColumnNullType, Any]: - ... - + def describe_null(self) -> tuple[ColumnNullType, Any]: ... @property - def null_count(self) -> int: - ... - + def null_count(self) -> int: ... @property - def metadata(self) -> dict[str, Any]: - ... - - def num_chunks(self) -> int: - ... - - def get_chunks(self, n_chunks: int | None=None) -> Iterable[_PyArrowColumn]: - ... - - def get_buffers(self) -> ColumnBuffers: - ... - - def _get_data_buffer(self) -> tuple[_PyArrowBuffer, Any]: - ... - - def _get_validity_buffer(self) -> tuple[_PyArrowBuffer, Any]: - ... - - def _get_offsets_buffer(self) -> tuple[_PyArrowBuffer, Any]: - ... + def metadata(self) -> dict[str, Any]: ... + def num_chunks(self) -> int: ... + def get_chunks(self, n_chunks: int | None = None) -> Iterable[_PyArrowColumn]: ... + def get_buffers(self) -> ColumnBuffers: ... + def _get_data_buffer(self) -> tuple[_PyArrowBuffer, Any]: ... + def _get_validity_buffer(self) -> tuple[_PyArrowBuffer, Any]: ... + def _get_offsets_buffer(self) -> tuple[_PyArrowBuffer, Any]: ... diff --git a/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi b/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi index 82c922def5d4..283aea084506 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/dataframe.pyi @@ -15,57 +15,37 @@ # specific language governing permissions and limitations # under the License. -from typing import Optional -import os -import re -import pyarrow -from typing import Callable, IO, Iterator, Literal, Mapping, NamedTuple, Self import pyarrow as pa from _typeshed import Incomplete from pyarrow.interchange.column import _PyArrowColumn as _PyArrowColumn -from typing import Any, Iterable, Sequence +from typing import Any +from collections.abc import Iterable, Sequence class _PyArrowDataFrame: _df: Incomplete _nan_as_null: Incomplete _allow_copy: Incomplete - def __init__(self, df: pa.Table | pa.RecordBatch, nan_as_null: bool=False, allow_copy: bool=True) -> None: - ... - - def __dataframe__(self, nan_as_null: bool=False, allow_copy: bool=True) -> _PyArrowDataFrame: - ... - + def __init__( + self, + df: pa.Table | pa.RecordBatch, + nan_as_null: bool = False, + allow_copy: bool = True, + ) -> None: ... + def __dataframe__( + self, nan_as_null: bool = False, allow_copy: bool = True + ) -> _PyArrowDataFrame: ... @property - def metadata(self) -> dict[str, Any]: - ... - - def num_columns(self) -> int: - ... - - def num_rows(self) -> int: - ... - - def num_chunks(self) -> int: - ... - - def column_names(self) -> Iterable[str]: - ... - - def get_column(self, i: int) -> _PyArrowColumn: - ... - - def get_column_by_name(self, name: str) -> _PyArrowColumn: - ... - - def get_columns(self) -> Iterable[_PyArrowColumn]: - ... - - def select_columns(self, indices: Sequence[int]) -> _PyArrowDataFrame: - ... - - def select_columns_by_name(self, names: Sequence[str]) -> _PyArrowDataFrame: - ... - - def get_chunks(self, n_chunks: int | None=None) -> Iterable[_PyArrowDataFrame]: - ... + def metadata(self) -> dict[str, Any]: ... + def num_columns(self) -> int: ... + def num_rows(self) -> int: ... + def num_chunks(self) -> int: ... + def column_names(self) -> Iterable[str]: ... + def get_column(self, i: int) -> _PyArrowColumn: ... + def get_column_by_name(self, name: str) -> _PyArrowColumn: ... + def get_columns(self) -> Iterable[_PyArrowColumn]: ... + def select_columns(self, indices: Sequence[int]) -> _PyArrowDataFrame: ... + def select_columns_by_name(self, names: Sequence[str]) -> _PyArrowDataFrame: ... + def get_chunks( + self, n_chunks: int | None = None + ) -> Iterable[_PyArrowDataFrame]: ... diff --git a/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi b/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi index 56300ed43591..f0a05cbcde65 100644 --- a/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi +++ b/python/pyarrow-stubs/pyarrow/interchange/from_dataframe.pyi @@ -15,50 +15,57 @@ # specific language governing permissions and limitations # under the License. -from pyarrow.lib import DataType, Table, TimestampType -from typing import Tuple, Union -import os -import re +from pyarrow.lib import DataType, TimestampType import pyarrow -from typing import Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete import pyarrow as pa -from pyarrow.interchange.column import ColumnBuffers as ColumnBuffers, ColumnNullType as ColumnNullType, Dtype as Dtype, DtypeKind as DtypeKind -from typing import Any -DataFrameObject = Any -ColumnObject = Any -BufferObject = Any -_PYARROW_DTYPES: dict[DtypeKind, dict[int, Any]] - -def from_dataframe(df: DataFrameObject, allow_copy: bool=True) -> pa.Table: - ... - -def _from_dataframe(df: DataFrameObject, allow_copy: bool=True) -> pyarrow.Table: - ... - -def protocol_df_chunk_to_pyarrow(df: DataFrameObject, allow_copy: bool=True) -> pa.RecordBatch: - ... - -def column_to_array(col: ColumnObject, allow_copy: bool=True) -> pa.Array: - ... - -def bool_column_to_array(col: ColumnObject, allow_copy: bool=True) -> pa.Array: - ... +from pyarrow.interchange.column import ( + ColumnBuffers as ColumnBuffers, + ColumnNullType as ColumnNullType, + Dtype as Dtype, + DtypeKind as DtypeKind, +) +from typing import Any, TypeAlias -def categorical_column_to_dictionary(col: ColumnObject, allow_copy: bool=True) -> pa.DictionaryArray: - ... - -def parse_datetime_format_str(format_str: str) -> tuple[str, str]: - ... - -def map_date_type(data_type: tuple[DtypeKind, int, str, str] | tuple[DtypeKind, int, str, str]) -> DataType | TimestampType: - ... - -def buffers_to_array(buffers: ColumnBuffers, data_type: tuple[DtypeKind, int, str, str], length: int, describe_null: ColumnNullType, offset: int=0, allow_copy: bool=True) -> pa.Array: - ... - -def validity_buffer_from_mask(validity_buff: BufferObject, validity_dtype: Dtype, describe_null: ColumnNullType, length: int, offset: int=0, allow_copy: bool=True) -> pa.Buffer: - ... +DataFrameObject: TypeAlias = Any +ColumnObject: TypeAlias = Any +BufferObject: TypeAlias = Any +_PYARROW_DTYPES: dict[DtypeKind, dict[int, Any]] -def validity_buffer_nan_sentinel(data_pa_buffer: BufferObject, data_type: Dtype, describe_null: ColumnNullType, length: int, offset: int=0, allow_copy: bool=True) -> pa.Buffer: - ... +def from_dataframe(df: DataFrameObject, allow_copy: bool = True) -> pa.Table: ... +def _from_dataframe(df: DataFrameObject, allow_copy: bool = True) -> pyarrow.Table: ... +def protocol_df_chunk_to_pyarrow( + df: DataFrameObject, allow_copy: bool = True +) -> pa.RecordBatch: ... +def column_to_array(col: ColumnObject, allow_copy: bool = True) -> pa.Array: ... +def bool_column_to_array(col: ColumnObject, allow_copy: bool = True) -> pa.Array: ... +def categorical_column_to_dictionary( + col: ColumnObject, allow_copy: bool = True +) -> pa.DictionaryArray: ... +def parse_datetime_format_str(format_str: str) -> tuple[str, str]: ... +def map_date_type( + data_type: tuple[DtypeKind, int, str, str], +) -> DataType | TimestampType: ... +def buffers_to_array( + buffers: ColumnBuffers, + data_type: tuple[DtypeKind, int, str, str], + length: int, + describe_null: ColumnNullType, + offset: int = 0, + allow_copy: bool = True, +) -> pa.Array: ... +def validity_buffer_from_mask( + validity_buff: BufferObject, + validity_dtype: Dtype, + describe_null: ColumnNullType, + length: int, + offset: int = 0, + allow_copy: bool = True, +) -> pa.Buffer: ... +def validity_buffer_nan_sentinel( + data_pa_buffer: BufferObject, + data_type: Dtype, + describe_null: ColumnNullType, + length: int, + offset: int = 0, + allow_copy: bool = True, +) -> pa.Buffer: ... diff --git a/python/pyarrow-stubs/pyarrow/ipc.pyi b/python/pyarrow-stubs/pyarrow/ipc.pyi index 31eba3df47a4..bd5d5147d671 100644 --- a/python/pyarrow-stubs/pyarrow/ipc.pyi +++ b/python/pyarrow-stubs/pyarrow/ipc.pyi @@ -17,62 +17,128 @@ from io import BufferedWriter, BytesIO from pathlib import Path -from pyarrow.lib import Buffer, BufferOutputStream, BufferReader, CompressedInputStream, CompressedOutputStream, MockOutputStream, NativeFile, OSFile, Schema -from typing import Optional, Union -import os -import re +from pyarrow.lib import ( + Buffer, + BufferOutputStream, + BufferReader, + CompressedInputStream, + CompressedOutputStream, + MockOutputStream, + NativeFile, + OSFile, + Schema, +) import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from typing import Any, IO import pyarrow.ipc from pyarrow.lib import __doc__ as __doc__ import pyarrow.lib as lib from _typeshed import Incomplete -from pyarrow.lib import Alignment as Alignment, IpcReadOptions as IpcReadOptions, IpcWriteOptions as IpcWriteOptions, Message as Message, MessageReader as MessageReader, MetadataVersion as MetadataVersion, ReadStats as ReadStats, RecordBatchReader as RecordBatchReader, WriteStats as WriteStats, _ReadPandasMixin as _ReadPandasMixin, get_record_batch_size as get_record_batch_size, get_tensor_size as get_tensor_size, read_message as read_message, read_record_batch as read_record_batch, read_schema as read_schema, read_tensor as read_tensor, write_tensor as write_tensor +from pyarrow.lib import ( + Alignment as Alignment, + IpcReadOptions as IpcReadOptions, + IpcWriteOptions as IpcWriteOptions, + Message as Message, + MessageReader as MessageReader, + MetadataVersion as MetadataVersion, + ReadStats as ReadStats, + RecordBatchReader as RecordBatchReader, + WriteStats as WriteStats, + _ReadPandasMixin as _ReadPandasMixin, + get_record_batch_size as get_record_batch_size, + get_tensor_size as get_tensor_size, + read_message as read_message, + read_record_batch as read_record_batch, + read_schema as read_schema, + read_tensor as read_tensor, + write_tensor as write_tensor, +) + _ipc_writer_class_doc: str _ipc_file_writer_class_doc: Incomplete -def _get_legacy_format_default(options: IpcWriteOptions | None) -> IpcWriteOptions: - ... - -def _ensure_default_ipc_read_options(options: IpcReadOptions | bool | None) -> IpcReadOptions: - ... - -def new_stream(sink: str | pyarrow.NativeFile | IO[Any], schema: pyarrow.Schema, *, options: pyarrow.ipc.IpcWriteOptions | None=None) -> RecordBatchStreamWriter: - ... - -def open_stream(source: BytesIO | bytes | BufferReader | Buffer, *, options: pyarrow.ipc.IpcReadOptions | None=None, memory_pool: pyarrow.MemoryPool | None=None) -> RecordBatchStreamReader: - ... - -def new_file(sink: str | pyarrow.NativeFile | IO[Any], schema: pyarrow.Schema, *, options: pyarrow.ipc.IpcWriteOptions | None=None, metadata: dict | pyarrow.KeyValueMetadata | None=None) -> RecordBatchFileWriter: - ... - -def open_file(source: Path | BufferReader | Buffer | BytesIO | OSFile | bytes, footer_offset: int | None=None, *, options: pyarrow.ipc.IpcReadOptions | None=None, memory_pool: pyarrow.MemoryPool | None=None) -> RecordBatchFileReader: - ... - -def serialize_pandas(df, *, nthreads: int | None=None, preserve_index: bool | None=None) -> Buffer: - ... - -def deserialize_pandas(buf: Buffer, *, use_threads: bool=True): - ... +def _get_legacy_format_default(options: IpcWriteOptions | None) -> IpcWriteOptions: ... +def _ensure_default_ipc_read_options( + options: IpcReadOptions | bool | None, +) -> IpcReadOptions: ... +def new_stream( + sink: str | pyarrow.NativeFile | IO[Any], + schema: pyarrow.Schema, + *, + options: pyarrow.ipc.IpcWriteOptions | None = None, +) -> RecordBatchStreamWriter: ... +def open_stream( + source: BytesIO | bytes | BufferReader | Buffer, + *, + options: pyarrow.ipc.IpcReadOptions | None = None, + memory_pool: pyarrow.MemoryPool | None = None, +) -> RecordBatchStreamReader: ... +def new_file( + sink: str | pyarrow.NativeFile | IO[Any], + schema: pyarrow.Schema, + *, + options: pyarrow.ipc.IpcWriteOptions | None = None, + metadata: dict | pyarrow.KeyValueMetadata | None = None, +) -> RecordBatchFileWriter: ... +def open_file( + source: Path | BufferReader | Buffer | BytesIO | OSFile | bytes, + footer_offset: int | None = None, + *, + options: pyarrow.ipc.IpcReadOptions | None = None, + memory_pool: pyarrow.MemoryPool | None = None, +) -> RecordBatchFileReader: ... +def serialize_pandas( + df, *, nthreads: int | None = None, preserve_index: bool | None = None +) -> Buffer: ... +def deserialize_pandas(buf: Buffer, *, use_threads: bool = True): ... class RecordBatchStreamReader(lib._RecordBatchStreamReader): - - def __init__(self, source: BufferReader | CompressedInputStream | Buffer | BytesIO | bytes, *, options=None, memory_pool=None) -> None: - ... + def __init__( + self, + source: BufferReader | CompressedInputStream | Buffer | BytesIO | bytes, + *, + options=None, + memory_pool=None, + ) -> None: ... class RecordBatchStreamWriter(lib._RecordBatchStreamWriter): __doc__: Incomplete - def __init__(self, sink: BufferOutputStream | BufferedWriter | MockOutputStream | CompressedOutputStream | BytesIO, schema: Schema, *, options=None) -> None: - ... + def __init__( + self, + sink: BufferOutputStream + | BufferedWriter + | MockOutputStream + | CompressedOutputStream + | BytesIO, + schema: Schema, + *, + options=None, + ) -> None: ... class RecordBatchFileReader(lib._RecordBatchFileReader): - - def __init__(self, source: Path | BufferReader | Buffer | BytesIO | OSFile | bytes, footer_offset=None, *, options=None, memory_pool=None) -> None: - ... + def __init__( + self, + source: Path | BufferReader | Buffer | BytesIO | OSFile | bytes, + footer_offset=None, + *, + options=None, + memory_pool=None, + ) -> None: ... class RecordBatchFileWriter(lib._RecordBatchFileWriter): __doc__: Incomplete - def __init__(self, sink: BufferOutputStream | MockOutputStream | str | BytesIO | NativeFile | OSFile, schema: Schema, *, options=None, metadata=None) -> None: - ... + def __init__( + self, + sink: BufferOutputStream + | MockOutputStream + | str + | BytesIO + | NativeFile + | OSFile, + schema: Schema, + *, + options=None, + metadata=None, + ) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/json.pyi b/python/pyarrow-stubs/pyarrow/json.pyi index 559ea5be22b9..d67cf434b233 100644 --- a/python/pyarrow-stubs/pyarrow/json.pyi +++ b/python/pyarrow-stubs/pyarrow/json.pyi @@ -15,9 +15,9 @@ # specific language governing permissions and limitations # under the License. -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete -from pyarrow._json import ParseOptions as ParseOptions, ReadOptions as ReadOptions, open_json as open_json, read_json as read_json +from pyarrow._json import ( + ParseOptions as ParseOptions, + ReadOptions as ReadOptions, + open_json as open_json, + read_json as read_json, +) diff --git a/python/pyarrow-stubs/pyarrow/jvm.pyi b/python/pyarrow-stubs/pyarrow/jvm.pyi index 423f956be3db..657fa8234396 100644 --- a/python/pyarrow-stubs/pyarrow/jvm.pyi +++ b/python/pyarrow-stubs/pyarrow/jvm.pyi @@ -15,47 +15,22 @@ # specific language governing permissions and limitations # under the License. -import os -import re import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence from _typeshed import Incomplete -def jvm_buffer(jvm_buf) -> pyarrow.Buffer: - ... - -def _from_jvm_int_type(jvm_type) -> pyarrow.DataType: - ... - -def _from_jvm_float_type(jvm_type): - ... - -def _from_jvm_time_type(jvm_type): - ... - -def _from_jvm_timestamp_type(jvm_type): - ... - -def _from_jvm_date_type(jvm_type): - ... - -def field(jvm_field) -> pyarrow.Field: - ... - -def schema(jvm_schema) -> pyarrow.Schema: - ... - -def array(jvm_array) -> pyarrow.Array: - ... - -def record_batch(jvm_vector_schema_root): - ... +def jvm_buffer(jvm_buf) -> pyarrow.Buffer: ... +def _from_jvm_int_type(jvm_type) -> pyarrow.DataType: ... +def _from_jvm_float_type(jvm_type): ... +def _from_jvm_time_type(jvm_type): ... +def _from_jvm_timestamp_type(jvm_type): ... +def _from_jvm_date_type(jvm_type): ... +def field(jvm_field) -> pyarrow.Field: ... +def schema(jvm_schema) -> pyarrow.Schema: ... +def array(jvm_array) -> pyarrow.Array: ... +def record_batch(jvm_vector_schema_root): ... class _JvmBufferNanny: ref_manager: Incomplete - def __init__(self, jvm_buf) -> None: - ... - - def __del__(self) -> None: - ... + def __init__(self, jvm_buf) -> None: ... + def __del__(self) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/lib.pyi b/python/pyarrow-stubs/pyarrow/lib.pyi index 1359f11e6e98..5c5988186f42 100644 --- a/python/pyarrow-stubs/pyarrow/lib.pyi +++ b/python/pyarrow-stubs/pyarrow/lib.pyi @@ -17,17 +17,15 @@ # Generated by stubgen-pyx from python/pyarrow/lib.pyx import os -import re import pyarrow -from typing import Callable, IO, Iterable, Iterator, Literal, NamedTuple, Self +from typing import IO, NamedTuple, Self +from collections.abc import Callable, Iterable, Iterator import pyarrow.compute from _typeshed import Incomplete -import sys -from collections import namedtuple from collections.abc import Mapping, Sequence from enum import IntEnum from typing import Any -from typing_extensions import TypeAlias + MonthDayNano: Incomplete Type_NA: Incomplete Type_BOOL: Incomplete @@ -90,431 +88,233 @@ _python_extension_types_registry: list[Incomplete] _registry_nanny: _ExtensionRegistryNanny DEFAULT_BUFFER_SIZE: int -def cpu_count(): - ... - -def set_cpu_count(count: int) -> None: - ... - -def is_opentelemetry_enabled() -> bool: - ... - -def is_threading_enabled() -> bool: - ... - -def _pc(): - ... - -def _pac(): - ... - -def _ensure_cuda_loaded() -> None: - ... - -def _gdb_test_session() -> None: - ... - -def encode_file_path(path): - ... - -def tobytes(o: str | bytes): - ... - -def frombytes(o, *, safe: bool=False): - ... - -def enable_signal_handlers(enable: bool) -> None: - ... - -def runtime_info() -> RuntimeInfo: - ... - -def _build_info() -> BuildInfo: - ... - -def set_timezone_db_path(path: str) -> None: - ... - -def default_memory_pool() -> MemoryPool: - ... - -def proxy_memory_pool(parent: MemoryPool) -> ProxyMemoryPool: - ... - -def logging_memory_pool(parent: MemoryPool) -> LoggingMemoryPool: - ... - -def system_memory_pool() -> MemoryPool: - ... - -def jemalloc_memory_pool() -> MemoryPool: - ... - -def mimalloc_memory_pool() -> MemoryPool: - ... - -def set_memory_pool(pool: MemoryPool) -> None: - ... - -def log_memory_allocations(enable: bool=True) -> None: - ... - -def total_allocated_bytes(): - ... - -def jemalloc_set_decay_ms(decay_ms: int) -> None: - ... - -def supported_memory_backends() -> list: - ... - -def default_cpu_memory_manager() -> MemoryManager: - ... - -def _get_pandas_type_map(): - ... - -def _is_primitive(type: int): - ... - -def _is_nested(type: int): - ... - -def _get_pandas_type(arrow_type, coerce_to_ns: bool=False): - ... - -def _get_pandas_tz_type(arrow_type, coerce_to_ns: bool=False): - ... - -def _to_pandas_dtype(arrow_type, options=None): - ... - -def register_extension_type(ext_type) -> None: - ... - -def unregister_extension_type(type_name: str) -> None: - ... - -def ensure_metadata(meta: object, allow_none: bool=False) -> KeyValueMetadata: - ... - -def unify_schemas(schemas: list[Schema], *, promote_options: str='default') -> Schema: - ... - -def field(name: str | bytes, type: DataType | str | None=None, nullable: bool | None=None, metadata: dict | None=None) -> Field: - ... - -def null() -> DataType: - ... - -def bool_() -> DataType: - ... - -def uint8() -> DataType: - ... - -def int8() -> DataType: - ... - -def uint16() -> DataType: - ... - -def int16() -> DataType: - ... - -def uint32() -> DataType: - ... - -def int32() -> DataType: - ... - -def uint64() -> DataType: - ... - -def int64() -> DataType: - ... - -def tzinfo_to_string(tz) -> str: - ... - -def string_to_tzinfo(name, *, prefer_zoneinfo: bool=True): - ... - -def timestamp(unit: str, tz: str | None=None) -> TimestampType: - ... - -def time32(unit: str) -> Time32Type: - ... - -def time64(unit: str) -> Time64Type: - ... - -def duration(unit: str) -> DurationType: - ... - -def month_day_nano_interval() -> DataType: - ... - -def date32() -> DataType: - ... - -def date64() -> DataType: - ... - -def float16() -> DataType: - ... - -def float32() -> DataType: - ... - -def float64() -> DataType: - ... - -def decimal32(precision: int, scale: int=0) -> DataType: - ... - -def decimal64(precision: int, scale: int=0) -> DataType: - ... - -def decimal128(precision: int, scale: int=0) -> DataType: - ... - -def decimal256(precision: int, scale: int=0) -> DataType: - ... - -def string() -> DataType: - ... - -def utf8(): - ... - -def binary(length: int=-1) -> DataType: - ... - -def large_binary() -> DataType: - ... - -def large_string() -> DataType: - ... - -def large_utf8(): - ... - -def binary_view() -> DataType: - ... - -def string_view() -> DataType: - ... - -def list_(value_type: DataType | Field, list_size: int=-1) -> DataType: - ... - -def large_list(value_type: DataType | Field) -> LargeListType: - ... - -def list_view(value_type: DataType | Field) -> ListViewType: - ... - -def large_list_view(value_type: DataType | Field) -> LargeListViewType: - ... - -def map_(key_type: DataType | Field, item_type: DataType | Field, keys_sorted: bool=False) -> MapType: - ... - -def dictionary(index_type: DataType, value_type: DataType, ordered: bool=False) -> DictionaryType: - ... - -def struct(fields) -> DataType: - ... - -def sparse_union(child_fields, type_codes: list[int] | None=None) -> DataType: - ... - -def dense_union(child_fields, type_codes: list[int] | None=None) -> DataType: - ... - -def union(child_fields, mode: str, type_codes: list[int] | None=None) -> UnionType: - ... - -def run_end_encoded(run_end_type: DataType, value_type: DataType) -> DataType: - ... - -def json_(storage_type: DataType=utf8()) -> JsonType: - ... - -def uuid() -> UuidType: - ... - -def fixed_shape_tensor(value_type: DataType, shape: tuple | list[int], dim_names: tuple | list[str] | None=None, permutation=None) -> FixedShapeTensorType: - ... - -def bool8() -> Bool8Type: - ... - -def opaque(storage_type: DataType, type_name: str, vendor_name: str) -> OpaqueType: - ... - -def type_for_alias(name: str) -> DataType: - ... - -def ensure_type(ty: object, allow_none: bool=False) -> DataType: - ... - -def schema(fields, metadata: dict | None=None) -> Schema: - ... - -def from_numpy_dtype(dtype: object) -> DataType: - ... - -def is_boolean_value(obj: object): - ... - -def is_integer_value(obj: object): - ... - -def is_float_value(obj: object): - ... - -def _datetime_from_int(value: int, unit: int, tzinfo=None): - ... - -def scalar(value: Any, type: DataType | None=None, *, from_pandas: bool | None=None, memory_pool: MemoryPool | None=None) -> Scalar: - ... - -def _ndarray_to_arrow_type(values: object, type: DataType) -> DataType: - ... - -def _handle_arrow_array_protocol(obj, type, mask, size): - ... - -def array(obj: object, type: DataType | None=None, mask=None, size=None, from_pandas: bool | None=None, safe: bool=True, memory_pool: MemoryPool | None=None) -> Array | ChunkedArray: - ... - -def asarray(values, type: str | DataType | None=None) -> Array | ChunkedArray: - ... - -def nulls(size: int, type: DataType | None=None, memory_pool: MemoryPool | None=None) -> Array: - ... - -def repeat(value, size: int, memory_pool: MemoryPool | None=None) -> Array: - ... - -def infer_type(values, mask=None, from_pandas: bool=False) -> DataType: - ... - -def arange(start: int, stop: int, step: int=1, *, memory_pool: MemoryPool | None=None) -> Array: - ... - -def _normalize_slice(arrow_obj: object, key: slice): - ... - -def _restore_array(data) -> Array: - ... - -def _compute_binary_op(func_name, left, right): - ... - -def concat_arrays(arrays: Iterable[Array], memory_pool: MemoryPool | None=None) -> Array: - ... - -def _empty_array(type: DataType) -> DictionaryArray: - ... - -def chunked_array(arrays, type=None) -> ChunkedArray: - ... - -def _reconstruct_record_batch(columns, schema) -> RecordBatch: - ... - -def table_to_blocks(options, table: Table, categories, extension_columns): - ... - -def _reconstruct_table(arrays, schema) -> Table: - ... - -def record_batch(data, names: list | None=None, schema: Schema | None=None, metadata: dict | Mapping[Any, Any] | None=None) -> RecordBatch: - ... - -def table(data, names: list | None=None, schema: Schema | None=None, metadata: dict | Mapping[Any, Any] | None=None, nthreads: int | None=None) -> Table: - ... - -def concat_tables(tables, memory_pool: MemoryPool | None=None, promote_options: str='none', **kwargs) -> Table: - ... - -def concat_batches(recordbatches, memory_pool: MemoryPool | None=None) -> RecordBatch: - ... - -def _from_pydict(cls, mapping: dict | Mapping[Any, Any], schema: Schema | None, metadata: dict | Mapping[Any, Any] | None) -> Self: - ... - -def _from_pylist(cls, mapping, schema: Schema | None, metadata: dict | Mapping[Any, Any] | None) -> Self: - ... - -def io_thread_count(): - ... - -def set_io_thread_count(count: int) -> None: - ... - -def memory_map(path: str, mode: str='r') -> MemoryMappedFile: - ... - -def create_memory_map(path: str, size: int) -> MemoryMappedFile: - ... - -def allocate_buffer(size: int, memory_pool: MemoryPool | None=None, resizable: bool=False) -> ResizableBuffer | Buffer: - ... - -def transcoding_input_stream(stream: NativeFile, src_encoding: str, dest_encoding: str): - ... - -def py_buffer(obj: object) -> Buffer: - ... - -def foreign_buffer(address: int, size: int, base=None) -> Buffer: - ... +def cpu_count(): ... +def set_cpu_count(count: int) -> None: ... +def is_opentelemetry_enabled() -> bool: ... +def is_threading_enabled() -> bool: ... +def _pc(): ... +def _pac(): ... +def _ensure_cuda_loaded() -> None: ... +def _gdb_test_session() -> None: ... +def encode_file_path(path): ... +def tobytes(o: str | bytes): ... +def frombytes(o, *, safe: bool = False): ... +def enable_signal_handlers(enable: bool) -> None: ... +def runtime_info() -> RuntimeInfo: ... +def _build_info() -> BuildInfo: ... +def set_timezone_db_path(path: str) -> None: ... +def default_memory_pool() -> MemoryPool: ... +def proxy_memory_pool(parent: MemoryPool) -> ProxyMemoryPool: ... +def logging_memory_pool(parent: MemoryPool) -> LoggingMemoryPool: ... +def system_memory_pool() -> MemoryPool: ... +def jemalloc_memory_pool() -> MemoryPool: ... +def mimalloc_memory_pool() -> MemoryPool: ... +def set_memory_pool(pool: MemoryPool) -> None: ... +def log_memory_allocations(enable: bool = True) -> None: ... +def total_allocated_bytes(): ... +def jemalloc_set_decay_ms(decay_ms: int) -> None: ... +def supported_memory_backends() -> list: ... +def default_cpu_memory_manager() -> MemoryManager: ... +def _get_pandas_type_map(): ... +def _is_primitive(type: int): ... +def _is_nested(type: int): ... +def _get_pandas_type(arrow_type, coerce_to_ns: bool = False): ... +def _get_pandas_tz_type(arrow_type, coerce_to_ns: bool = False): ... +def _to_pandas_dtype(arrow_type, options=None): ... +def register_extension_type(ext_type) -> None: ... +def unregister_extension_type(type_name: str) -> None: ... +def ensure_metadata(meta: object, allow_none: bool = False) -> KeyValueMetadata: ... +def unify_schemas( + schemas: list[Schema], *, promote_options: str = "default" +) -> Schema: ... +def field( + name: str | bytes, + type: DataType | str | None = None, + nullable: bool | None = None, + metadata: dict | None = None, +) -> Field: ... +def null() -> DataType: ... +def bool_() -> DataType: ... +def uint8() -> DataType: ... +def int8() -> DataType: ... +def uint16() -> DataType: ... +def int16() -> DataType: ... +def uint32() -> DataType: ... +def int32() -> DataType: ... +def uint64() -> DataType: ... +def int64() -> DataType: ... +def tzinfo_to_string(tz) -> str: ... +def string_to_tzinfo(name, *, prefer_zoneinfo: bool = True): ... +def timestamp(unit: str, tz: str | None = None) -> TimestampType: ... +def time32(unit: str) -> Time32Type: ... +def time64(unit: str) -> Time64Type: ... +def duration(unit: str) -> DurationType: ... +def month_day_nano_interval() -> DataType: ... +def date32() -> DataType: ... +def date64() -> DataType: ... +def float16() -> DataType: ... +def float32() -> DataType: ... +def float64() -> DataType: ... +def decimal32(precision: int, scale: int = 0) -> DataType: ... +def decimal64(precision: int, scale: int = 0) -> DataType: ... +def decimal128(precision: int, scale: int = 0) -> DataType: ... +def decimal256(precision: int, scale: int = 0) -> DataType: ... +def string() -> DataType: ... +def utf8(): ... +def binary(length: int = -1) -> DataType: ... +def large_binary() -> DataType: ... +def large_string() -> DataType: ... +def large_utf8(): ... +def binary_view() -> DataType: ... +def string_view() -> DataType: ... +def list_(value_type: DataType | Field, list_size: int = -1) -> DataType: ... +def large_list(value_type: DataType | Field) -> LargeListType: ... +def list_view(value_type: DataType | Field) -> ListViewType: ... +def large_list_view(value_type: DataType | Field) -> LargeListViewType: ... +def map_( + key_type: DataType | Field, item_type: DataType | Field, keys_sorted: bool = False +) -> MapType: ... +def dictionary( + index_type: DataType, value_type: DataType, ordered: bool = False +) -> DictionaryType: ... +def struct(fields) -> DataType: ... +def sparse_union(child_fields, type_codes: list[int] | None = None) -> DataType: ... +def dense_union(child_fields, type_codes: list[int] | None = None) -> DataType: ... +def union( + child_fields, mode: str, type_codes: list[int] | None = None +) -> UnionType: ... +def run_end_encoded(run_end_type: DataType, value_type: DataType) -> DataType: ... +def json_(storage_type: DataType = ...) -> JsonType: ... +def uuid() -> UuidType: ... +def fixed_shape_tensor( + value_type: DataType, + shape: tuple | list[int], + dim_names: tuple | list[str] | None = None, + permutation=None, +) -> FixedShapeTensorType: ... +def bool8() -> Bool8Type: ... +def opaque(storage_type: DataType, type_name: str, vendor_name: str) -> OpaqueType: ... +def type_for_alias(name: str) -> DataType: ... +def ensure_type(ty: object, allow_none: bool = False) -> DataType: ... +def schema(fields, metadata: dict | None = None) -> Schema: ... +def from_numpy_dtype(dtype: object) -> DataType: ... +def is_boolean_value(obj: object): ... +def is_integer_value(obj: object): ... +def is_float_value(obj: object): ... +def _datetime_from_int(value: int, unit: int, tzinfo=None): ... +def scalar( + value: Any, + type: DataType | None = None, + *, + from_pandas: bool | None = None, + memory_pool: MemoryPool | None = None, +) -> Scalar: ... +def _ndarray_to_arrow_type(values: object, type: DataType) -> DataType: ... +def _handle_arrow_array_protocol(obj, type, mask, size): ... +def array( + obj: object, + type: DataType | None = None, + mask=None, + size=None, + from_pandas: bool | None = None, + safe: bool = True, + memory_pool: MemoryPool | None = None, +) -> Array | ChunkedArray: ... +def asarray(values, type: str | DataType | None = None) -> Array | ChunkedArray: ... +def nulls( + size: int, type: DataType | None = None, memory_pool: MemoryPool | None = None +) -> Array: ... +def repeat(value, size: int, memory_pool: MemoryPool | None = None) -> Array: ... +def infer_type(values, mask=None, from_pandas: bool = False) -> DataType: ... +def arange( + start: int, stop: int, step: int = 1, *, memory_pool: MemoryPool | None = None +) -> Array: ... +def _normalize_slice(arrow_obj: object, key: slice): ... +def _restore_array(data) -> Array: ... +def _compute_binary_op(func_name, left, right): ... +def concat_arrays( + arrays: Iterable[Array], memory_pool: MemoryPool | None = None +) -> Array: ... +def _empty_array(type: DataType) -> DictionaryArray: ... +def chunked_array(arrays, type=None) -> ChunkedArray: ... +def _reconstruct_record_batch(columns, schema) -> RecordBatch: ... +def table_to_blocks(options, table: Table, categories, extension_columns): ... +def _reconstruct_table(arrays, schema) -> Table: ... +def record_batch( + data, + names: list | None = None, + schema: Schema | None = None, + metadata: dict | Mapping[Any, Any] | None = None, +) -> RecordBatch: ... +def table( + data, + names: list | None = None, + schema: Schema | None = None, + metadata: dict | Mapping[Any, Any] | None = None, + nthreads: int | None = None, +) -> Table: ... +def concat_tables( + tables, + memory_pool: MemoryPool | None = None, + promote_options: str = "none", + **kwargs, +) -> Table: ... +def concat_batches( + recordbatches, memory_pool: MemoryPool | None = None +) -> RecordBatch: ... +def _from_pydict( + cls, + mapping: dict | Mapping[Any, Any], + schema: Schema | None, + metadata: dict | Mapping[Any, Any] | None, +) -> Self: ... +def _from_pylist( + cls, mapping, schema: Schema | None, metadata: dict | Mapping[Any, Any] | None +) -> Self: ... +def io_thread_count(): ... +def set_io_thread_count(count: int) -> None: ... +def memory_map(path: str, mode: str = "r") -> MemoryMappedFile: ... +def create_memory_map(path: str, size: int) -> MemoryMappedFile: ... +def allocate_buffer( + size: int, memory_pool: MemoryPool | None = None, resizable: bool = False +) -> ResizableBuffer | Buffer: ... +def transcoding_input_stream( + stream: NativeFile, src_encoding: str, dest_encoding: str +): ... +def py_buffer(obj: object) -> Buffer: ... +def foreign_buffer(address: int, size: int, base=None) -> Buffer: ... +def as_buffer(o: object): ... +def _detect_compression(path) -> str: ... +def compress( + buf: object, + codec: str = "lz4", + asbytes: bool = False, + memory_pool: MemoryPool | None = None, +) -> Buffer | bytes: ... +def decompress( + buf: object, + decompressed_size: int | None = None, + codec: str = "lz4", + asbytes: bool = False, + memory_pool: MemoryPool | None = None, +) -> Buffer | bytes: ... +def input_stream( + source, compression: str | None = "detect", buffer_size: int | None = None +) -> NativeFile: ... +def output_stream( + source, compression: str | None = "detect", buffer_size: int | None = None +) -> NativeFile: ... +def get_tensor_size(tensor: Tensor) -> int: ... +def get_record_batch_size(batch: RecordBatch) -> int: ... +def write_tensor(tensor: Tensor, dest: NativeFile) -> int: ... +def read_tensor(source: NativeFile) -> Tensor: ... +def read_message(source) -> Message: ... +def read_schema(obj, dictionary_memo: DictionaryMemo | None = None) -> Schema: ... +def read_record_batch( + obj, schema: Schema, dictionary_memo: DictionaryMemo | None = None +) -> RecordBatch: ... +def benchmark_PandasObjectIsNull(obj: list[Any]) -> None: ... -def as_buffer(o: object): - ... - -def _detect_compression(path) -> str: - ... - -def compress(buf: object, codec: str='lz4', asbytes: bool=False, memory_pool: MemoryPool | None=None) -> Buffer | bytes: - ... - -def decompress(buf: object, decompressed_size: int | None=None, codec: str='lz4', asbytes: bool=False, memory_pool: MemoryPool | None=None) -> Buffer | bytes: - ... - -def input_stream(source, compression: str | None='detect', buffer_size: int | None=None) -> NativeFile: - ... - -def output_stream(source, compression: str | None='detect', buffer_size: int | None=None) -> NativeFile: - ... - -def get_tensor_size(tensor: Tensor) -> int: - ... - -def get_record_batch_size(batch: RecordBatch) -> int: - ... - -def write_tensor(tensor: Tensor, dest: NativeFile) -> int: - ... - -def read_tensor(source: NativeFile) -> Tensor: - ... - -def read_message(source) -> Message: - ... - -def read_schema(obj, dictionary_memo: DictionaryMemo | None=None) -> Schema: - ... - -def read_record_batch(obj, schema: Schema, dictionary_memo: DictionaryMemo | None=None) -> RecordBatch: - ... - -def benchmark_PandasObjectIsNull(obj: list[Any]) -> None: - ... _NULL: NullScalar NA: NullScalar @@ -590,60 +390,31 @@ class Alignment(IntEnum): DataTypeSpecific = ... At64Byte = ... -class ArrowException(Exception): - ... - -class ArrowInvalid(ValueError, ArrowException): - ... - -class ArrowMemoryError(MemoryError, ArrowException): - ... +class ArrowException(Exception): ... +class ArrowInvalid(ValueError, ArrowException): ... +class ArrowMemoryError(MemoryError, ArrowException): ... class ArrowKeyError(KeyError, ArrowException): + def __str__(self): ... - def __str__(self): - ... - -class ArrowTypeError(TypeError, ArrowException): - ... - -class ArrowNotImplementedError(NotImplementedError, ArrowException): - ... - -class ArrowCapacityError(ArrowException): - ... - -class ArrowIndexError(IndexError, ArrowException): - ... - -class ArrowSerializationError(ArrowException): - ... +class ArrowTypeError(TypeError, ArrowException): ... +class ArrowNotImplementedError(NotImplementedError, ArrowException): ... +class ArrowCapacityError(ArrowException): ... +class ArrowIndexError(IndexError, ArrowException): ... +class ArrowSerializationError(ArrowException): ... class ArrowCancelled(ArrowException): + def __init__(self, message, signum=None): ... - def __init__(self, message, signum=None): - ... - -class StopToken: - ... +class StopToken: ... class SignalStopHandler: - - def __init__(self): - ... - - def _init_signals(self) -> None: - ... - - def __enter__(self) -> Self: - ... - - def __exit__(self, exc_type, exc_value, exc_tb) -> None: - ... - + def __init__(self): ... + def _init_signals(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type, exc_value, exc_tb) -> None: ... @property - def stop_token(self): - ... + def stop_token(self): ... class _PandasAPIShim(object): _loose_version: object @@ -664,3392 +435,1872 @@ class _PandasAPIShim(object): _is_ge_v3: bool _is_ge_v3_strict: bool - def __init__(self): - ... - - def series(self, *args, **kwargs): - ... - - def data_frame(self, *args, **kwargs): - ... - + def __init__(self): ... + def series(self, *args, **kwargs): ... + def data_frame(self, *args, **kwargs): ... @property - def have_pandas(self): - ... - + def have_pandas(self): ... @property - def compat(self): - ... - + def compat(self): ... @property - def pd(self): - ... - - def infer_dtype(self, obj): - ... - - def pandas_dtype(self, dtype): - ... - + def pd(self): ... + def infer_dtype(self, obj): ... + def pandas_dtype(self, dtype): ... @property - def loose_version(self): - ... - + def loose_version(self): ... @property - def version(self): - ... - - def is_ge_v23(self): - ... - - def is_ge_v3(self): - ... - - def is_ge_v3_strict(self): - ... - - def uses_string_dtype(self): - ... - + def version(self): ... + def is_ge_v23(self): ... + def is_ge_v3(self): ... + def is_ge_v3_strict(self): ... + def uses_string_dtype(self): ... @property - def categorical_type(self): - ... - + def categorical_type(self): ... @property - def datetimetz_type(self): - ... - + def datetimetz_type(self): ... @property - def extension_dtype(self): - ... - - def is_array_like(self, obj): - ... - - def is_categorical(self, obj): - ... - - def is_datetimetz(self, obj): - ... - - def is_extension_array_dtype(self, obj): - ... - - def is_sparse(self, obj): - ... - - def is_data_frame(self, obj): - ... + def extension_dtype(self): ... + def is_array_like(self, obj): ... + def is_categorical(self, obj): ... + def is_datetimetz(self, obj): ... + def is_extension_array_dtype(self, obj): ... + def is_sparse(self, obj): ... + def is_data_frame(self, obj): ... + def is_series(self, obj): ... + def is_index(self, obj): ... + def get_values(self, obj): ... - def is_series(self, obj): - ... - - def is_index(self, obj): - ... - - def get_values(self, obj): - ... - -class _Weakrefable: - ... +class _Weakrefable: ... class MemoryPool(_Weakrefable): - - def __init__(self): - ... - - def release_unused(self) -> None: - ... - - def bytes_allocated(self): - ... - - def total_bytes_allocated(self): - ... - - def max_memory(self): - ... - - def num_allocations(self): - ... - - def print_stats(self) -> None: - ... - + def __init__(self): ... + def release_unused(self) -> None: ... + def bytes_allocated(self): ... + def total_bytes_allocated(self): ... + def max_memory(self): ... + def num_allocations(self): ... + def print_stats(self) -> None: ... @property - def backend_name(self) -> str: - ... - - def __repr__(self) -> str: - ... + def backend_name(self) -> str: ... class LoggingMemoryPool(MemoryPool): - - def __init__(self): - ... + def __init__(self): ... class ProxyMemoryPool(MemoryPool): - - def __init__(self): - ... + def __init__(self): ... class Device(_Weakrefable): - - def __init__(self): - ... - - def __eq__(self, other): - ... - - def __repr__(self) -> str: - ... - + def __init__(self): ... + def __eq__(self, other): ... @property - def type_name(self) -> str: - ... - + def type_name(self) -> str: ... @property - def device_id(self): - ... - + def device_id(self): ... @property - def is_cpu(self): - ... - + def is_cpu(self): ... @property - def device_type(self): - ... + def device_type(self): ... class MemoryManager(_Weakrefable): - - def __init__(self): - ... - - def __repr__(self) -> str: - ... - + def __init__(self): ... @property - def device(self) -> Device: - ... - + def device(self) -> Device: ... @property - def is_cpu(self): - ... + def is_cpu(self): ... class DataType(_Weakrefable): - - def __init__(self): - ... - - def field(self, i: int) -> Field: - ... - + def __init__(self): ... + def field(self, i: int) -> Field: ... @property - def id(self): - ... - + def id(self): ... @property - def bit_width(self): - ... - + def bit_width(self): ... @property - def byte_width(self): - ... - + def byte_width(self): ... @property - def num_fields(self): - ... - + def num_fields(self): ... @property - def num_buffers(self): - ... - + def num_buffers(self): ... @property - def has_variadic_buffers(self): - ... - - def __str__(self) -> str: - ... - - def __hash__(self): - ... - - def __reduce__(self) -> tuple: - ... - - def __repr__(self) -> str: - ... - - def __eq__(self, other): - ... - - def equals(self, other, *, check_metadata: bool=False) -> bool: - ... - - def to_pandas_dtype(self): - ... - - def _export_to_c(self, out_ptr) -> None: - ... - + def has_variadic_buffers(self): ... + def __hash__(self): ... + def __reduce__(self) -> tuple: ... + def __eq__(self, other): ... + def equals(self, other, *, check_metadata: bool = False) -> bool: ... + def to_pandas_dtype(self): ... + def _export_to_c(self, out_ptr) -> None: ... @staticmethod - def _import_from_c(in_ptr) -> DataType: - ... - - def __arrow_c_schema__(self): - ... - + def _import_from_c(in_ptr) -> DataType: ... + def __arrow_c_schema__(self): ... @staticmethod - def _import_from_c_capsule(schema) -> DataType: - ... + def _import_from_c_capsule(schema) -> DataType: ... class DictionaryMemo(_Weakrefable): - - def __init__(self): - ... + def __init__(self): ... class DictionaryType(DataType): - - def __reduce__(self) -> tuple: - ... - + def __reduce__(self) -> tuple: ... @property - def ordered(self): - ... - + def ordered(self): ... @property - def index_type(self) -> DataType: - ... - + def index_type(self) -> DataType: ... @property - def value_type(self) -> DataType: - ... + def value_type(self) -> DataType: ... class ListType(DataType): - - def __reduce__(self) -> tuple: - ... - + def __reduce__(self) -> tuple: ... @property - def value_field(self) -> Field: - ... - + def value_field(self) -> Field: ... @property - def value_type(self) -> DataType: - ... + def value_type(self) -> DataType: ... class LargeListType(DataType): - - def __reduce__(self) -> tuple: - ... - + def __reduce__(self) -> tuple: ... @property - def value_field(self) -> Field: - ... - + def value_field(self) -> Field: ... @property - def value_type(self) -> DataType: - ... + def value_type(self) -> DataType: ... class ListViewType(DataType): - - def __reduce__(self) -> tuple: - ... - + def __reduce__(self) -> tuple: ... @property - def value_field(self) -> Field: - ... - + def value_field(self) -> Field: ... @property - def value_type(self) -> DataType: - ... + def value_type(self) -> DataType: ... class LargeListViewType(DataType): - - def __reduce__(self) -> tuple: - ... - + def __reduce__(self) -> tuple: ... @property - def value_field(self) -> Field: - ... - + def value_field(self) -> Field: ... @property - def value_type(self) -> DataType: - ... + def value_type(self) -> DataType: ... class MapType(DataType): - - def __reduce__(self) -> tuple: - ... - + def __reduce__(self) -> tuple: ... @property - def key_field(self) -> Field: - ... - + def key_field(self) -> Field: ... @property - def key_type(self) -> DataType: - ... - + def key_type(self) -> DataType: ... @property - def item_field(self) -> Field: - ... - + def item_field(self) -> Field: ... @property - def item_type(self) -> DataType: - ... - + def item_type(self) -> DataType: ... @property - def keys_sorted(self): - ... + def keys_sorted(self): ... class FixedSizeListType(DataType): - - def __reduce__(self) -> tuple: - ... - + def __reduce__(self) -> tuple: ... @property - def value_field(self) -> Field: - ... - + def value_field(self) -> Field: ... @property - def value_type(self) -> DataType: - ... - + def value_type(self) -> DataType: ... @property - def list_size(self): - ... + def list_size(self): ... class StructType(DataType): - - def get_field_index(self, name: str) -> int: - ... - - def field(self, i: int | str) -> Field: - ... - - def get_all_field_indices(self, name: str) -> list[int]: - ... - - def __len__(self): - ... - - def __iter__(self) -> Iterator[Any]: - ... - - def __getitem__(self, i): - ... - - def __reduce__(self) -> tuple: - ... - + def get_field_index(self, name: str) -> int: ... + def field(self, i: int | str) -> Field: ... + def get_all_field_indices(self, name: str) -> list[int]: ... + def __len__(self): ... + def __iter__(self) -> Iterator[Any]: ... + def __getitem__(self, i): ... + def __reduce__(self) -> tuple: ... @property - def names(self) -> list: - ... - + def names(self) -> list: ... @property - def fields(self) -> list: - ... + def fields(self) -> list: ... class UnionType(DataType): - @property - def mode(self) -> str: - ... - + def mode(self) -> str: ... @property - def type_codes(self): - ... - - def __len__(self): - ... - - def __iter__(self) -> Iterator[Any]: - ... - - def field(self, i: int) -> Field: - ... - - def __getitem__(self, i): - ... - - def __reduce__(self) -> tuple: - ... - -class SparseUnionType(UnionType): - ... + def type_codes(self): ... + def __len__(self): ... + def __iter__(self) -> Iterator[Any]: ... + def field(self, i: int) -> Field: ... + def __getitem__(self, i): ... + def __reduce__(self) -> tuple: ... -class DenseUnionType(UnionType): - ... +class SparseUnionType(UnionType): ... +class DenseUnionType(UnionType): ... class TimestampType(DataType): - @property - def unit(self): - ... - + def unit(self): ... @property - def tz(self) -> str | None: - ... - - def __reduce__(self) -> tuple: - ... + def tz(self) -> str | None: ... + def __reduce__(self) -> tuple: ... class Time32Type(DataType): - @property - def unit(self): - ... + def unit(self): ... class Time64Type(DataType): - @property - def unit(self): - ... + def unit(self): ... class DurationType(DataType): - @property - def unit(self): - ... + def unit(self): ... class FixedSizeBinaryType(DataType): - - def __reduce__(self) -> tuple: - ... + def __reduce__(self) -> tuple: ... class Decimal32Type(FixedSizeBinaryType): - - def __reduce__(self) -> tuple: - ... - + def __reduce__(self) -> tuple: ... @property - def precision(self): - ... - + def precision(self): ... @property - def scale(self): - ... + def scale(self): ... class Decimal64Type(FixedSizeBinaryType): - - def __reduce__(self) -> tuple: - ... - + def __reduce__(self) -> tuple: ... @property - def precision(self): - ... - + def precision(self): ... @property - def scale(self): - ... + def scale(self): ... class Decimal128Type(FixedSizeBinaryType): - - def __reduce__(self) -> tuple: - ... - + def __reduce__(self) -> tuple: ... @property - def precision(self): - ... - + def precision(self): ... @property - def scale(self): - ... + def scale(self): ... class Decimal256Type(FixedSizeBinaryType): - - def __reduce__(self) -> tuple: - ... - + def __reduce__(self) -> tuple: ... @property - def precision(self): - ... - + def precision(self): ... @property - def scale(self): - ... + def scale(self): ... class RunEndEncodedType(DataType): - - def __reduce__(self) -> tuple: - ... - + def __reduce__(self) -> tuple: ... @property - def run_end_type(self) -> DataType: - ... - + def run_end_type(self) -> DataType: ... @property - def value_type(self) -> DataType: - ... + def value_type(self) -> DataType: ... class BaseExtensionType(DataType): - - def __arrow_ext_class__(self): - ... - - def __arrow_ext_scalar_class__(self): - ... - + def __arrow_ext_class__(self): ... + def __arrow_ext_scalar_class__(self): ... @property - def extension_name(self) -> str: - ... - + def extension_name(self) -> str: ... @property - def storage_type(self) -> DataType: - ... - + def storage_type(self) -> DataType: ... @property - def byte_width(self): - ... - + def byte_width(self): ... @property - def bit_width(self): - ... - - def wrap_array(self, storage: Array | ChunkedArray) -> Array | ChunkedArray: - ... + def bit_width(self): ... + def wrap_array(self, storage: Array | ChunkedArray) -> Array | ChunkedArray: ... class ExtensionType(BaseExtensionType): - - def __init__(self, storage_type: DataType, extension_name): - ... - - def __eq__(self, other): - ... - - def __repr__(self) -> str: - ... - - def __arrow_ext_serialize__(self): - ... - + def __init__(self, storage_type: DataType, extension_name): ... + def __eq__(self, other): ... + def __arrow_ext_serialize__(self): ... @classmethod - def __arrow_ext_deserialize__(cls, storage_type, serialized): - ... - - def __reduce__(self) -> tuple: - ... - - def __arrow_ext_class__(self): - ... - - def __arrow_ext_scalar_class__(self): - ... + def __arrow_ext_deserialize__(cls, storage_type, serialized): ... + def __reduce__(self) -> tuple: ... + def __arrow_ext_class__(self): ... + def __arrow_ext_scalar_class__(self): ... class JsonType(BaseExtensionType): - - def __arrow_ext_class__(self): - ... - - def __reduce__(self) -> tuple: - ... - - def __arrow_ext_scalar_class__(self): - ... + def __arrow_ext_class__(self): ... + def __reduce__(self) -> tuple: ... + def __arrow_ext_scalar_class__(self): ... class UuidType(BaseExtensionType): - - def __arrow_ext_class__(self): - ... - - def __reduce__(self) -> tuple: - ... - - def __arrow_ext_scalar_class__(self): - ... + def __arrow_ext_class__(self): ... + def __reduce__(self) -> tuple: ... + def __arrow_ext_scalar_class__(self): ... class FixedShapeTensorType(BaseExtensionType): - @property - def value_type(self) -> DataType: - ... - + def value_type(self) -> DataType: ... @property - def shape(self): - ... - + def shape(self): ... @property - def dim_names(self) -> list | None: - ... - + def dim_names(self) -> list | None: ... @property - def permutation(self): - ... - - def __arrow_ext_class__(self): - ... - - def __reduce__(self) -> tuple: - ... - - def __arrow_ext_scalar_class__(self): - ... + def permutation(self): ... + def __arrow_ext_class__(self): ... + def __reduce__(self) -> tuple: ... + def __arrow_ext_scalar_class__(self): ... class Bool8Type(BaseExtensionType): - - def __arrow_ext_class__(self): - ... - - def __reduce__(self) -> tuple: - ... - - def __arrow_ext_scalar_class__(self): - ... + def __arrow_ext_class__(self): ... + def __reduce__(self) -> tuple: ... + def __arrow_ext_scalar_class__(self): ... class OpaqueType(BaseExtensionType): - @property - def type_name(self) -> str: - ... - + def type_name(self) -> str: ... @property - def vendor_name(self) -> str: - ... - - def __arrow_ext_class__(self): - ... - - def __reduce__(self) -> tuple: - ... - - def __arrow_ext_scalar_class__(self): - ... + def vendor_name(self) -> str: ... + def __arrow_ext_class__(self): ... + def __reduce__(self) -> tuple: ... + def __arrow_ext_scalar_class__(self): ... class UnknownExtensionType(ExtensionType): - - def __init__(self, storage_type: DataType, serialized): - ... - - def __arrow_ext_serialize__(self): - ... - + def __init__(self, storage_type: DataType, serialized): ... + def __arrow_ext_serialize__(self): ... @classmethod - def __arrow_ext_deserialize__(cls, storage_type, serialized) -> UnknownExtensionType: - ... + def __arrow_ext_deserialize__( + cls, storage_type, serialized + ) -> UnknownExtensionType: ... -class _Metadata(_Weakrefable): - ... +class _Metadata(_Weakrefable): ... class KeyValueMetadata(_Metadata, Mapping): + def __init__(self, __arg0__=None, **kwargs): ... + def equals(self, other: KeyValueMetadata) -> bool: ... + def __eq__(self, other): ... + def __len__(self): ... + def __contains__(self, key): ... + def __getitem__(self, key): ... + def __iter__(self): ... + def __reduce__(self) -> tuple: ... + def key(self, i: int): ... + def value(self, i: int): ... + def keys(self) -> Iterator[Any]: ... + def values(self) -> Iterator[Any]: ... + def items(self) -> Iterator[tuple]: ... + def get_all(self, key: str) -> list: ... + def to_dict(self): ... - def __init__(self, __arg0__=None, **kwargs): - ... +class Field(_Weakrefable): + type: DataType - def equals(self, other: KeyValueMetadata) -> bool: - ... + def __init__(self): ... + def equals(self, other: Field, check_metadata: bool = False) -> bool: ... + def __eq__(self, other): ... + def __reduce__(self) -> tuple: ... + def __repr__(self): ... + def __hash__(self): ... + @property + def nullable(self): ... + @property + def name(self) -> str: ... + @property + def metadata(self) -> dict | None: ... + def with_metadata(self, metadata: dict) -> Field: ... + def remove_metadata(self) -> Field: ... + def with_type(self, new_type: DataType) -> Field: ... + def with_name(self, name: str) -> Field: ... + def with_nullable(self, nullable: bool) -> Field: ... + def flatten(self) -> list: ... + def _export_to_c(self, out_ptr) -> None: ... + @staticmethod + def _import_from_c(in_ptr) -> Field: ... + def __arrow_c_schema__(self): ... + @staticmethod + def _import_from_c_capsule(schema) -> Field: ... - def __repr__(self) -> str: - ... +class Schema(_Weakrefable): + def __init__(self): ... + def __len__(self): ... + def __getitem__(self, key): ... + def __iter__(self) -> Iterator[Any]: ... + def __reduce__(self) -> tuple: ... + def __hash__(self): ... + def __sizeof__(self): ... + @property + def pandas_metadata(self): ... + @property + def names(self) -> list: ... + @property + def types(self) -> list: ... + @property + def metadata(self): ... + def __eq__(self, other): ... + def empty_table(self) -> Table: ... + def equals(self, other: Schema, check_metadata: bool = False) -> bool: ... + @classmethod + def from_pandas(cls, df, preserve_index: bool | None = None) -> Schema: ... + def field(self, i: int | str) -> Field: ... + def _field(self, i: int) -> Field: ... + def field_by_name(self, name: str) -> None | Field: ... + def get_field_index(self, name: str) -> int: ... + def get_all_field_indices(self, name: str) -> list[int]: ... + def append(self, field: Field): ... + def insert(self, i: int, field: Field) -> Schema: ... + def remove(self, i: int) -> Schema: ... + def set(self, i: int, field: Field) -> Schema: ... + def add_metadata(self, metadata: dict): ... + def with_metadata(self, metadata: dict) -> Schema: ... + def serialize(self, memory_pool: MemoryPool | None = None) -> Buffer: ... + def remove_metadata(self) -> Schema: ... + def to_string( + self, + truncate_metadata: bool = True, + show_field_metadata: bool = True, + show_schema_metadata: bool = True, + element_size_limit: int = 100, + ) -> str: ... + def _export_to_c(self, out_ptr) -> None: ... + @staticmethod + def _import_from_c(in_ptr) -> Schema: ... + def __str__(self): ... + def __repr__(self): ... + def __arrow_c_schema__(self): ... + @staticmethod + def _import_from_c_capsule(schema) -> Schema: ... - def __str__(self) -> str: - ... +class _ExtensionRegistryNanny(_Weakrefable): + def __init__(self): ... + def release_registry(self) -> None: ... - def __eq__(self, other): - ... +class Scalar(_Weakrefable): + def __init__(self): ... + @property + def type(self) -> DataType: ... + @property + def is_valid(self): ... + def cast( + self, + target_type: object | None = None, + safe: bool | None = None, + options: pyarrow.compute.CastOptions | None = None, + memory_pool: MemoryPool | None = None, + ): ... + def validate(self, *, full: bool = False) -> None: ... + def equals(self, other: Scalar) -> bool: ... + def __eq__(self, other): ... + def __hash__(self): ... + def __reduce__(self) -> tuple: ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __abs__(self): ... + def __add__(self, other: object): ... + def __radd__(self, other: object): ... + def __truediv__(self, other: object): ... + def __rtruediv__(self, other: object): ... + def __mul__(self, other: object): ... + def __rmul__(self, other: object): ... + def __neg__(self): ... + def __pow__(self, other: object, mod: None = None): ... + def __rpow__(self, other: object, mod: None = None): ... + def __sub__(self, other: object): ... + def __rsub__(self, other: object): ... + def __and__(self, other: object): ... + def __rand__(self, other: object): ... + def __or__(self, other: object): ... + def __ror__(self, other: object): ... + def __xor__(self, other: object): ... + def __rxor__(self, other: object): ... + def __lshift__(self, other: object): ... + def __rlshift__(self, other: object): ... + def __rshift__(self, other: object): ... + def __rrshift__(self, other: object): ... - def __len__(self): - ... +class NullScalar(Scalar): + def __init__(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None) -> None: ... - def __contains__(self, key): - ... +class BooleanScalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __bool__(self): ... - def __getitem__(self, key): - ... +class UInt8Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __index__(self): ... - def __iter__(self): - ... +class Int8Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __index__(self): ... - def __reduce__(self) -> tuple: - ... +class UInt16Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __index__(self): ... - def key(self, i: int): - ... +class Int16Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __index__(self): ... - def value(self, i: int): - ... +class UInt32Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __index__(self): ... - def keys(self) -> Iterator[Any]: - ... +class Int32Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __index__(self): ... - def values(self) -> Iterator[Any]: - ... +class UInt64Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __index__(self): ... - def items(self) -> Iterator[tuple]: - ... +class Int64Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __index__(self): ... - def get_all(self, key: str) -> list: - ... +class HalfFloatScalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __float__(self): ... + def __int__(self) -> int: ... - def to_dict(self): - ... +class FloatScalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __float__(self): ... + def __int__(self) -> int: ... -class Field(_Weakrefable): - type: DataType +class DoubleScalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __float__(self): ... + def __int__(self) -> int: ... - def __init__(self): - ... +class Decimal32Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... - def equals(self, other: Field, check_metadata: bool=False) -> bool: - ... +class Decimal64Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... - def __eq__(self, other): - ... +class Decimal128Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... - def __reduce__(self) -> tuple: - ... +class Decimal256Scalar(Scalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... - def __str__(self) -> str: - ... +class Date32Scalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... - def __repr__(self): - ... +class Date64Scalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... - def __hash__(self): - ... +class Time32Scalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... +class Time64Scalar(Scalar): @property - def nullable(self): - ... + def value(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... +class TimestampScalar(Scalar): @property - def name(self) -> str: - ... + def value(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... +class DurationScalar(Scalar): @property - def metadata(self) -> dict | None: - ... + def value(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... - def with_metadata(self, metadata: dict) -> Field: - ... +class MonthDayNanoIntervalScalar(Scalar): + @property + def value(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... - def remove_metadata(self) -> Field: - ... +class BinaryScalar(Scalar): + def as_buffer(self) -> Buffer | None: ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + def __bytes__(self): ... - def with_type(self, new_type: DataType) -> Field: - ... +class LargeBinaryScalar(BinaryScalar): ... +class FixedSizeBinaryScalar(BinaryScalar): ... - def with_name(self, name: str) -> Field: - ... +class StringScalar(BinaryScalar): + def as_py(self, *, maps_as_pydicts: str | None = None) -> None | str: ... - def with_nullable(self, nullable: bool) -> Field: - ... +class LargeStringScalar(StringScalar): ... +class BinaryViewScalar(BinaryScalar): ... +class StringViewScalar(StringScalar): ... - def flatten(self) -> list: - ... +class ListScalar(Scalar, Sequence): + @property + def values(self) -> Array | None: ... + def __len__(self) -> int: ... + def __getitem__(self, i): ... + def __iter__(self): ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... - def _export_to_c(self, out_ptr) -> None: - ... +class FixedSizeListScalar(ListScalar): ... +class LargeListScalar(ListScalar): ... +class ListViewScalar(ListScalar): ... +class LargeListViewScalar(ListScalar): ... - @staticmethod - def _import_from_c(in_ptr) -> Field: - ... +class StructScalar(Scalar, Mapping): + def __len__(self): ... + def __iter__(self) -> Iterator[str]: ... + def items(self): ... + def __contains__(self, key) -> bool: ... + def __getitem__(self, key) -> Scalar: ... + def as_py(self, *, maps_as_pydicts: str | None = None) -> dict | None: ... + def _as_py_tuple(self) -> list | None: ... - def __arrow_c_schema__(self): - ... +class MapScalar(ListScalar, Mapping): + def __getitem__(self, i): ... + def __iter__(self) -> Iterator[tuple]: ... + def as_py(self, *, maps_as_pydicts: str | None = None) -> None | list | dict: ... + def keys(self) -> list: ... +class DictionaryScalar(Scalar): @staticmethod - def _import_from_c_capsule(schema) -> Field: - ... - -class Schema(_Weakrefable): - - def __init__(self): - ... - - def __len__(self): - ... - - def __getitem__(self, key): - ... - - def __iter__(self) -> Iterator[Any]: - ... - - def __reduce__(self) -> tuple: - ... - - def __hash__(self): - ... - - def __sizeof__(self): - ... - + def _reconstruct(type, is_valid, index, dictionary) -> Scalar: ... + def __reduce__(self) -> tuple: ... @property - def pandas_metadata(self): - ... - + def index(self) -> Scalar: ... @property - def names(self) -> list: - ... - + def value(self) -> Scalar: ... @property - def types(self) -> list: - ... + def dictionary(self) -> Array: ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... +class RunEndEncodedScalar(Scalar): @property - def metadata(self): - ... - - def __eq__(self, other): - ... + def value(self) -> Scalar: ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... - def empty_table(self) -> Table: - ... - - def equals(self, other: Schema, check_metadata: bool=False) -> bool: - ... - - @classmethod - def from_pandas(cls, df, preserve_index: bool | None=None) -> Schema: - ... - - def field(self, i: int | str) -> Field: - ... +class UnionScalar(Scalar): + @property + def value(self) -> Scalar | None: ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + @property + def type_code(self): ... - def _field(self, i: int) -> Field: - ... +class ExtensionScalar(Scalar): + @property + def value(self) -> Scalar | None: ... + def as_py(self, *, maps_as_pydicts: str | None = None): ... + @staticmethod + def from_storage(typ: BaseExtensionType, value: object) -> Scalar: ... - def field_by_name(self, name: str) -> None | Field: - ... +class JsonScalar(ExtensionScalar): ... - def get_field_index(self, name: str) -> int: - ... +class UuidScalar(ExtensionScalar): + def as_py(self, *, maps_as_pydicts: str | None = None): ... - def get_all_field_indices(self, name: str) -> list[int]: - ... +class FixedShapeTensorScalar(ExtensionScalar): + def to_numpy(self): ... + def to_tensor(self) -> Tensor: ... + def __dlpack__( + self, *, stream=None, max_version=None, dl_device=None, copy=None + ): ... + def __dlpack_device__(self) -> tuple: ... - def append(self, field: Field): - ... +class OpaqueScalar(ExtensionScalar): ... - def insert(self, i: int, field: Field) -> Schema: - ... +class Bool8Scalar(ExtensionScalar): + def as_py(self, *, maps_as_pydicts: str | None = None) -> None | bool: ... - def remove(self, i: int) -> Schema: - ... +class ArrayStatistics(_Weakrefable): + def __init__(self): ... + @property + def null_count(self): ... + @property + def is_null_count_exact(self): ... + @property + def distinct_count(self): ... + @property + def is_distinct_count_exact(self): ... + @property + def min(self): ... + @property + def is_min_exact(self): ... + @property + def max(self): ... + @property + def is_max_exact(self): ... - def set(self, i: int, field: Field) -> Schema: - ... +class _PandasConvertible(_Weakrefable): + def to_pandas( + self, + memory_pool: MemoryPool | None = None, + categories: list | None = None, + strings_to_categorical: bool = False, + zero_copy_only: bool = False, + integer_object_nulls: bool = False, + date_as_object: bool = True, + timestamp_as_object: bool = False, + use_threads: bool = True, + deduplicate_objects: bool = True, + ignore_metadata: bool = False, + safe: bool = True, + split_blocks: bool = False, + self_destruct: bool = False, + maps_as_pydicts: str | None = None, + types_mapper: Callable[..., Any] | None = None, + coerce_temporal_nanoseconds: bool = False, + ): ... - def add_metadata(self, metadata: dict): - ... +class Array(_PandasConvertible): + type: DataType + _name: object - def with_metadata(self, metadata: dict) -> Schema: - ... + def __init__(self): ... + def _debug_print(self) -> None: ... + def diff(self, other: Array) -> str: ... + def cast( + self, + target_type: object | None = None, + safe: bool | None = None, + options: pyarrow.compute.CastOptions | None = None, + memory_pool: MemoryPool | None = None, + ) -> Array: ... + def view(self, target_type: object) -> Array: ... + def sum(self, **kwargs) -> Scalar: ... + def unique(self) -> Array: ... + def dictionary_encode(self, null_encoding: str = "mask") -> DictionaryArray: ... + def value_counts(self) -> StructArray: ... + @staticmethod + def from_pandas( + obj, + mask=None, + type: DataType | None = None, + safe: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Array | ChunkedArray: ... + def __reduce__(self) -> tuple: ... + @staticmethod + def from_buffers( + type: DataType, + length: int, + buffers, + null_count: int = -1, + offset: int = 0, + children: list[Array] | None = None, + ) -> Array: ... + @property + def null_count(self): ... + @property + def nbytes(self): ... + def get_total_buffer_size(self) -> int: ... + def __sizeof__(self): ... + def __iter__(self) -> Iterator[Any]: ... + def to_string( + self, + *, + indent: int = 2, + top_level_indent: int = 0, + window: int = 10, + container_window: int = 2, + skip_new_lines: bool = False, + element_size_limit: int = 100, + ) -> str: ... + def __str__(self): ... + def __eq__(self, other): ... + def equals(self: Array, other: Array) -> bool: ... + def __len__(self): ... + def is_null(self, *, nan_is_null: bool = False): ... + def is_nan(self): ... + def is_valid(self): ... + def fill_null(self, fill_value: Any) -> Array: ... + def __getitem__(self, key) -> Scalar | Array: ... + def slice( + self, offset: int | Scalar = 0, length: int | Scalar | None = None + ) -> Array: ... + def take(self, indices: object) -> Array: ... + def drop_null(self): ... + def filter( + self, mask: object, *, null_selection_behavior: str = "drop" + ) -> Array: ... + def index( + self, + value: Scalar | object, + start: int | None = None, + end: int | None = None, + *, + memory_pool: MemoryPool | None = None, + ) -> Int64Scalar: ... + def sort( + self, order: str = "ascending", null_placement: str = "at_end", **kwargs + ) -> Array: ... + def _to_pandas(self, options, types_mapper=None, **kwargs): ... + def __array__(self, dtype=None, copy=None): ... + def to_numpy(self, zero_copy_only: bool = True, writable: bool = False): ... + def to_tensor(self, *, allow_nulls: bool = False) -> Tensor: ... + def to_pylist(self, *, maps_as_pydicts: str | None = None) -> list: ... + def tolist(self): ... + def validate(self, *, full: bool = False) -> None: ... + @property + def offset(self): ... + def buffers(self) -> list: ... + def copy_to(self, destination: MemoryManager | Device) -> Array: ... + def _export_to_c(self, out_ptr, out_schema_ptr: int = 0) -> None: ... + @staticmethod + def _import_from_c(in_ptr, type) -> Array: ... + def __arrow_c_array__(self, requested_schema=None) -> tuple: ... + @staticmethod + def _import_from_c_capsule(schema_capsule, array_capsule) -> Array: ... + def _export_to_c_device(self, out_ptr, out_schema_ptr: int = 0) -> None: ... + @staticmethod + def _import_from_c_device(in_ptr, type) -> Array: ... + def __arrow_c_device_array__(self, requested_schema=None, **kwargs) -> tuple: ... + @staticmethod + def _import_from_c_device_capsule(schema_capsule, array_capsule) -> Array: ... + @staticmethod + def from_dlpack( + x: object, /, *, device=None, copy: bool | None = None + ) -> Array: ... + def __dlpack__( + self, + *, + stream: int | None = None, + max_version=None, + dl_device=None, + copy: bool | None = None, + ): ... + def __dlpack_device__(self) -> tuple: ... + @property + def device_type(self) -> DeviceAllocationType: ... + @property + def is_cpu(self) -> bool: ... + @property + def statistics(self) -> None | ArrayStatistics: ... + def __abs__(self): ... + def __add__(self, other: object): ... + def __radd__(self, other: object): ... + def __truediv__(self, other: object): ... + def __rtruediv__(self, other: object): ... + def __mul__(self, other: object): ... + def __rmul__(self, other: object): ... + def __neg__(self): ... + def __pow__(self, other: object, mod: None = None): ... + def __rpow__(self, other: object, mod: None = None): ... + def __sub__(self, other: object): ... + def __rsub__(self, other: object): ... + def __and__(self, other: object): ... + def __rand__(self, other: object): ... + def __or__(self, other: object): ... + def __ror__(self, other: object): ... + def __xor__(self, other: object): ... + def __rxor__(self, other: object): ... + def __lshift__(self, other: object): ... + def __rlshift__(self, other: object): ... + def __rshift__(self, other: object): ... + def __rrshift__(self, other: object): ... + +class NullArray(Array): ... - def serialize(self, memory_pool: MemoryPool | None=None) -> Buffer: - ... +class BooleanArray(Array): + @property + def false_count(self): ... + @property + def true_count(self): ... + +class NumericArray(Array): ... +class IntegerArray(NumericArray): ... +class FloatingPointArray(NumericArray): ... +class Int8Array(IntegerArray): ... +class UInt8Array(IntegerArray): ... +class Int16Array(IntegerArray): ... +class UInt16Array(IntegerArray): ... +class Int32Array(IntegerArray): ... +class UInt32Array(IntegerArray): ... +class Int64Array(IntegerArray): ... +class UInt64Array(IntegerArray): ... +class Date32Array(NumericArray): ... +class Date64Array(NumericArray): ... +class TimestampArray(NumericArray): ... +class Time32Array(NumericArray): ... +class Time64Array(NumericArray): ... +class DurationArray(NumericArray): ... - def remove_metadata(self) -> Schema: - ... +class MonthDayNanoIntervalArray(Array): + def to_pylist(self, *, maps_as_pydicts: str | None = None) -> list: ... - def to_string(self, truncate_metadata: bool=True, show_field_metadata: bool=True, show_schema_metadata: bool=True, element_size_limit: int=100) -> str: - ... +class HalfFloatArray(FloatingPointArray): ... +class FloatArray(FloatingPointArray): ... +class DoubleArray(FloatingPointArray): ... +class FixedSizeBinaryArray(Array): ... +class Decima32Array(FixedSizeBinaryArray): ... +class Decimal64Array(FixedSizeBinaryArray): ... +class Decimal128Array(FixedSizeBinaryArray): ... +class Decimal256Array(FixedSizeBinaryArray): ... - def _export_to_c(self, out_ptr) -> None: - ... +class BaseListArray(Array): + def flatten(self, recursive: bool = False) -> Array: ... + def value_parent_indices(self): ... + def value_lengths(self): ... +class ListArray(BaseListArray): @staticmethod - def _import_from_c(in_ptr) -> Schema: - ... - - def __str__(self): - ... - - def __repr__(self): - ... - - def __arrow_c_schema__(self): - ... + def from_arrays( + offsets: Array, + values: Array, + type: DataType | None = None, + pool: MemoryPool | None = None, + mask: Array | None = None, + ) -> Array: ... + @property + def values(self) -> Array: ... + @property + def offsets(self) -> Array: ... +class LargeListArray(BaseListArray): @staticmethod - def _import_from_c_capsule(schema) -> Schema: - ... - -class _ExtensionRegistryNanny(_Weakrefable): - - def __init__(self): - ... - - def release_registry(self) -> None: - ... - -class Scalar(_Weakrefable): + def from_arrays( + offsets: Array, + values: Array, + type: DataType | None = None, + pool: MemoryPool | None = None, + mask: Array | None = None, + ) -> Array: ... + @property + def values(self) -> Array: ... + @property + def offsets(self) -> Array: ... - def __init__(self): - ... +class ListViewArray(BaseListArray): + @staticmethod + def from_arrays( + offsets: Array, + sizes: Array, + values: Array, + type: DataType | None = None, + pool: MemoryPool | None = None, + mask: Array | None = None, + ) -> Array: ... + @property + def values(self) -> Array: ... + @property + def offsets(self) -> Array: ... + @property + def sizes(self) -> Array: ... +class LargeListViewArray(BaseListArray): + @staticmethod + def from_arrays( + offsets: Array, + sizes: Array, + values: Array, + type: DataType | None = None, + pool: MemoryPool | None = None, + mask: Array | None = None, + ) -> Array: ... @property - def type(self) -> DataType: - ... + def values(self) -> Array: ... + @property + def offsets(self) -> Array: ... + @property + def sizes(self) -> Array: ... +class MapArray(ListArray): + @staticmethod + def from_arrays( + offsets, + keys, + items, + type: DataType | None = None, + pool: MemoryPool | None = None, + mask: Array | None = None, + ) -> Array: ... @property - def is_valid(self): - ... + def keys(self) -> Array: ... + @property + def items(self) -> Array: ... - def cast(self, target_type: object | None=None, safe: bool | None=None, options: pyarrow.compute.CastOptions | None=None, memory_pool: MemoryPool | None=None): - ... +class FixedSizeListArray(BaseListArray): + @staticmethod + def from_arrays( + values: Array, + list_size: int | None = None, + type: DataType | None = None, + mask: Array | None = None, + ) -> Array: ... + @property + def values(self) -> Array: ... - def validate(self, *, full: bool=False) -> None: - ... +class UnionArray(Array): + def child(self, pos: int) -> Field: ... + def field(self, pos: int) -> Array: ... + @property + def type_codes(self) -> Array: ... + @property + def offsets(self) -> Array: ... + @staticmethod + def from_dense( + types: Array, + value_offsets: Array, + children: list[Any], + field_names: list[Any] | None = None, + type_codes: list[Any] | None = None, + ) -> Array: ... + @staticmethod + def from_sparse( + types: Array, + children: list[Any], + field_names: list[Any] | None = None, + type_codes: list[Any] | None = None, + ) -> Array: ... - def __repr__(self) -> str: - ... +class StringArray(Array): + @staticmethod + def from_buffers( + length: int, + value_offsets: Buffer, + data: Buffer, + null_bitmap: Buffer | None = None, + null_count: int = -1, + offset: int = 0, + ) -> Array: ... - def __str__(self) -> str: - ... +class LargeStringArray(Array): + @staticmethod + def from_buffers( + length: int, + value_offsets: Buffer, + data: Buffer, + null_bitmap: Buffer | None = None, + null_count: int = -1, + offset: int = 0, + ) -> Array: ... - def equals(self, other: Scalar) -> bool: - ... +class StringViewArray(Array): ... - def __eq__(self, other): - ... +class BinaryArray(Array): + @property + def total_values_length(self): ... - def __hash__(self): - ... +class LargeBinaryArray(Array): + @property + def total_values_length(self): ... - def __reduce__(self) -> tuple: - ... +class BinaryViewArray(Array): ... - def as_py(self, *, maps_as_pydicts: str | None=None): - ... +class DictionaryArray(Array): + def dictionary_encode(self) -> DictionaryArray: ... + def dictionary_decode(self): ... + @property + def dictionary(self): ... + @property + def indices(self): ... + @staticmethod + def from_buffers( + type: DataType, + length: int, + buffers, + dictionary: Array, + null_count: int = -1, + offset: int = 0, + ) -> Array: ... + @staticmethod + def from_arrays( + indices, + dictionary, + mask=None, + ordered: bool = False, + from_pandas: bool = False, + safe: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Array: ... - def __abs__(self): - ... +class StructArray(Array): + def field(self, index) -> Array: ... + def _flattened_field( + self, index, memory_pool: MemoryPool | None = None + ) -> Array: ... + def flatten(self, memory_pool: MemoryPool | None = None) -> list: ... + @staticmethod + def from_arrays( + arrays: Sequence[Array], + names: list[str] | None = None, + fields: list[Field] | None = None, + mask=None, + memory_pool: MemoryPool | None = None, + type: StructType | None = None, + ) -> StructArray: ... + def sort( + self, + order: str = "ascending", + null_placement: str = "at_end", + by: str | None = None, + **kwargs, + ) -> StructArray: ... - def __add__(self, other: object): - ... +class RunEndEncodedArray(Array): + @staticmethod + def _from_arrays( + type, allow_none_for_type, logical_length, run_ends, values, logical_offset + ) -> Array: ... + @staticmethod + def from_arrays( + run_ends, values: Array, type: DataType | None = None + ) -> RunEndEncodedArray: ... + @staticmethod + def from_buffers( + type: DataType, + length: int, + buffers: list[Buffer], + null_count: int = -1, + offset: int = 0, + children: list[Array] | None = None, + ) -> RunEndEncodedArray: ... + @property + def run_ends(self) -> Array: ... + @property + def values(self) -> Array: ... + def find_physical_offset(self): ... + def find_physical_length(self): ... - def __radd__(self, other: object): - ... +class ExtensionArray(Array): + @property + def storage(self) -> Array: ... + @staticmethod + def from_storage(typ: BaseExtensionType, storage: Array) -> Array: ... - def __truediv__(self, other: object): - ... +class JsonArray(ExtensionArray): ... +class UuidArray(ExtensionArray): ... - def __rtruediv__(self, other: object): - ... +class FixedShapeTensorArray(ExtensionArray): + def to_numpy_ndarray(self): ... + @staticmethod + def from_tensor(tensor: Tensor) -> Array: ... + @staticmethod + def from_numpy_ndarray( + obj, dim_names: tuple | list[str] | None = None + ) -> ExtensionArray: ... + @staticmethod + def from_dlpack( + x: object, /, *, device=None, copy: bool | None = None + ) -> FixedShapeTensorArray: ... + def __dlpack__( + self, *, stream=None, max_version=None, dl_device=None, copy=None + ): ... - def __mul__(self, other: object): - ... +class OpaqueArray(ExtensionArray): ... - def __rmul__(self, other: object): - ... +class Bool8Array(ExtensionArray): + def to_numpy(self, zero_copy_only: bool = True, writable: bool = False): ... + @staticmethod + def from_storage(storage: Int8Array) -> ExtensionArray: ... + @staticmethod + def from_numpy(obj) -> Bool8Array: ... - def __neg__(self): - ... +class StringBuilder(_Weakrefable): + def __init__(self, memory_pool: MemoryPool | None = None): ... + def append(self, value) -> None: ... + def append_values(self, values) -> None: ... + def finish(self) -> Array: ... + @property + def null_count(self): ... + def __len__(self): ... - def __pow__(self, other: object, mod: None=None): - ... +class StringViewBuilder(_Weakrefable): + def __init__(self, memory_pool: MemoryPool | None = None): ... + def append(self, value) -> None: ... + def append_values(self, values) -> None: ... + def finish(self) -> Array: ... + @property + def null_count(self): ... + def __len__(self): ... - def __rpow__(self, other: object, mod: None=None): - ... +class ChunkedArray(_PandasConvertible): + _name: object - def __sub__(self, other: object): - ... + def __init__(self): ... + def __reduce__(self) -> tuple: ... + @property + def data(self) -> ChunkedArray: ... + @property + def type(self) -> DataType: ... + def length(self): ... + def __len__(self): ... + def to_string( + self, + *, + indent: int = 0, + window: int = 5, + container_window: int = 2, + skip_new_lines: bool = False, + element_size_limit: int = 100, + ) -> str: ... + def format(self, **kwargs) -> str: ... + def __str__(self): ... + def validate(self, *, full: bool = False) -> None: ... + @property + def null_count(self) -> int: ... + @property + def nbytes(self): ... + def get_total_buffer_size(self) -> int: ... + def __sizeof__(self): ... + def __iter__(self) -> Iterator[Any]: ... + def __getitem__(self, key) -> Scalar | ChunkedArray: ... + def is_null(self, *, nan_is_null: bool = False): ... + def is_nan(self): ... + def is_valid(self): ... + def __eq__(self, other): ... + def fill_null(self, fill_value: Any) -> Array | ChunkedArray: ... + def equals(self, other: ChunkedArray) -> bool: ... + def _to_pandas(self, options, types_mapper=None, **kwargs): ... + def to_numpy(self, zero_copy_only: bool = False): ... + def __array__(self, dtype=None, copy=None): ... + def cast( + self, + target_type: object | None = None, + safe: bool | None = None, + options: pyarrow.compute.CastOptions | None = None, + ) -> Array | ChunkedArray: ... + def dictionary_encode(self, null_encoding: str = "mask") -> ChunkedArray: ... + def flatten(self, memory_pool: MemoryPool | None = None) -> list: ... + def combine_chunks(self, memory_pool: MemoryPool | None = None) -> Array: ... + def unique(self) -> Array: ... + def value_counts(self): ... + def slice( + self, offset: int | Scalar = 0, length: int | Scalar | None = None + ) -> ChunkedArray: ... + def filter( + self, mask, null_selection_behavior: object = "drop" + ) -> Array | ChunkedArray: ... + def index( + self, + value: Scalar | object, + start: int | None = None, + end: int | None = None, + *, + memory_pool: MemoryPool | None = None, + ) -> Int64Scalar: ... + def take(self, indices: object) -> Array | ChunkedArray: ... + def drop_null(self): ... + def sort( + self, order: str = "ascending", null_placement: str = "at_end", **kwargs + ) -> ChunkedArray: ... + def unify_dictionaries( + self, memory_pool: MemoryPool | None = None + ) -> ChunkedArray: ... + @property + def num_chunks(self) -> int: ... + def chunk(self, i: int) -> Array: ... + @property + def chunks(self) -> list: ... + def iterchunks(self) -> Iterator[Any]: ... + def to_pylist(self, *, maps_as_pydicts: str | None = None) -> list: ... + def __arrow_c_stream__(self, requested_schema=None): ... + @staticmethod + def _import_from_c_capsule(stream) -> ChunkedArray: ... + @property + def is_cpu(self): ... + def _assert_cpu(self) -> None: ... - def __rsub__(self, other: object): - ... +class _Tabular(_PandasConvertible): + def __init__(self): ... + def __array__(self, dtype=None, copy=None): ... + def __dataframe__(self, nan_as_null: bool = False, allow_copy: bool = True): ... + def __eq__(self, other): ... + def __getitem__(self, key): ... + def __len__(self): ... + def __repr__(self): ... + def _column(self, i: int): ... + def _ensure_integer_index(self, i): ... + def _is_initialized(self): ... + def column(self, i: int | str) -> Array | ChunkedArray: ... + @property + def column_names(self) -> list: ... + @property + def columns(self) -> list: ... + def drop_null(self) -> Table | RecordBatch: ... + def field(self, i: int | str) -> Field: ... + @classmethod + def from_pydict( + cls, + mapping: dict | Mapping[Any, Any], + schema: Schema | None = None, + metadata: dict | Mapping[Any, Any] | None = None, + ) -> Table | RecordBatch: ... + @classmethod + def from_pylist( + cls, + mapping, + schema: Schema | None = None, + metadata: dict | Mapping[Any, Any] | None = None, + ) -> Table | RecordBatch: ... + def itercolumns(self) -> Iterator[Any]: ... + @property + def num_columns(self): ... + @property + def num_rows(self): ... + @property + def shape(self) -> tuple: ... + @property + def schema(self): ... + def sort_by(self, sorting, **kwargs) -> Table | RecordBatch: ... + def take(self, indices: object) -> Table | RecordBatch: ... + def filter( + self, mask, null_selection_behavior: object = "drop" + ) -> Table | RecordBatch: ... + def to_pydict(self, *, maps_as_pydicts: str | None = None) -> dict: ... + def to_pylist(self, *, maps_as_pydicts: str | None = None) -> list: ... + def to_string( + self, *, show_metadata: bool = False, preview_cols: int = 0 + ) -> str: ... + def remove_column(self, i: int): ... + def drop_columns(self, columns: str | list[str]) -> _Tabular: ... + def add_column(self, i: int, field_, column): ... + def append_column(self, field_: str | Field, column) -> Table | RecordBatch: ... - def __and__(self, other: object): - ... +class RecordBatch(_Tabular): + def __init__(self): ... + def _is_initialized(self) -> bool: ... + def __reduce__(self) -> tuple: ... + def validate(self, *, full: bool = False) -> None: ... + def replace_schema_metadata(self, metadata: dict | None = None) -> RecordBatch: ... + @property + def num_columns(self) -> int: ... + @property + def num_rows(self) -> int: ... + @property + def schema(self) -> Schema: ... + def _column(self, i: int) -> Array: ... + @property + def nbytes(self): ... + def get_total_buffer_size(self) -> int: ... + def __sizeof__(self): ... + def add_column(self, i: int, field_: str | Field, column) -> RecordBatch: ... + def remove_column(self, i: int) -> RecordBatch: ... + def set_column(self, i: int, field_: str | Field, column) -> RecordBatch: ... + def rename_columns(self, names) -> RecordBatch: ... + def serialize(self, memory_pool: MemoryPool | None = None) -> Buffer: ... + def slice( + self, offset: int | Scalar = 0, length: int | Scalar | None = None + ) -> RecordBatch: ... + def equals(self, other: object, check_metadata: bool = False) -> bool: ... + def select(self, columns: object) -> RecordBatch: ... + def cast( + self, + target_schema: Schema, + safe: bool | None = None, + options: pyarrow.compute.CastOptions | None = None, + ) -> RecordBatch: ... + def _to_pandas(self, options, **kwargs): ... + @classmethod + def from_pandas( + cls, + df, + schema: Schema | None = None, + preserve_index: bool | None = None, + nthreads: int | None = None, + columns: list | None = None, + ) -> RecordBatch | Self: ... + @staticmethod + def from_arrays( + arrays: list[Any], + names: list[str] | None = None, + schema: Schema | None = None, + metadata: dict | Mapping[Any, Any] | None = None, + ) -> RecordBatch: ... + @staticmethod + def from_struct_array(struct_array: StructArray) -> RecordBatch: ... + def to_struct_array(self) -> Array: ... + def to_tensor( + self, + null_to_nan: bool = False, + row_major: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Tensor: ... + def copy_to(self, destination: MemoryManager | Device) -> RecordBatch: ... + def _export_to_c(self, out_ptr, out_schema_ptr: int = 0) -> None: ... + @staticmethod + def _import_from_c(in_ptr, schema) -> RecordBatch: ... + def __arrow_c_array__(self, requested_schema=None) -> tuple: ... + def __arrow_c_stream__(self, requested_schema=None): ... + @staticmethod + def _import_from_c_capsule(schema_capsule, array_capsule) -> RecordBatch: ... + def _export_to_c_device(self, out_ptr, out_schema_ptr: int = 0) -> None: ... + @staticmethod + def _import_from_c_device(in_ptr, schema) -> RecordBatch: ... + def __arrow_c_device_array__(self, requested_schema=None, **kwargs) -> tuple: ... + @staticmethod + def _import_from_c_device_capsule(schema_capsule, array_capsule) -> RecordBatch: ... + @property + def device_type(self) -> DeviceAllocationType: ... + @property + def is_cpu(self) -> bool: ... - def __rand__(self, other: object): - ... +class Table(_Tabular): + def __init__(self): ... + def _is_initialized(self) -> bool: ... + def validate(self, *, full: bool = False) -> None: ... + def __reduce__(self) -> tuple: ... + def slice( + self, offset: int | Scalar = 0, length: int | Scalar | None = None + ) -> Table: ... + def select(self, columns: object) -> Table: ... + def replace_schema_metadata(self, metadata: dict | None = None) -> Table: ... + def flatten(self, memory_pool: MemoryPool | None = None) -> Table: ... + def combine_chunks(self, memory_pool: MemoryPool | None = None) -> Table: ... + def unify_dictionaries(self, memory_pool: MemoryPool | None = None) -> Table: ... + def equals(self, other: Table, check_metadata: bool = False) -> bool: ... + def cast( + self, + target_schema: Schema, + safe: bool | None = None, + options: pyarrow.compute.CastOptions | None = None, + ) -> Table: ... + @classmethod + def from_pandas( + cls, + df, + schema: Schema | None = None, + preserve_index: bool | None = None, + nthreads: int | None = None, + columns: list | None = None, + safe: bool = True, + ) -> Table | Self: ... + @staticmethod + def from_arrays( + arrays: list[Array] | ChunkedArray, + names: list[str] | None = None, + schema: Schema | None = None, + metadata: dict | Mapping[Any, Any] | None = None, + ) -> Table: ... + @staticmethod + def from_struct_array(struct_array: StructArray | ChunkedArray) -> Table: ... + def to_struct_array(self, max_chunksize: int | None = None) -> ChunkedArray: ... + @staticmethod + def from_batches( + batches: Sequence[Any] | Iterator[RecordBatch], schema: Schema | None = None + ) -> Table: ... + def to_batches(self, max_chunksize: int | None = None) -> list[RecordBatch]: ... + def to_tensor( + self, + null_to_nan: bool = False, + row_major: bool = True, + memory_pool: MemoryPool | None = None, + ) -> Tensor: ... + def to_reader(self, max_chunksize: int | None = None) -> RecordBatchReader: ... + def _to_pandas( + self, options, categories=None, ignore_metadata: bool = False, types_mapper=None + ): ... + @property + def schema(self) -> Schema: ... + def _column(self, i: int) -> ChunkedArray: ... + @property + def num_columns(self) -> int: ... + @property + def num_rows(self) -> int: ... + @property + def nbytes(self): ... + def get_total_buffer_size(self) -> int: ... + def __sizeof__(self): ... + def add_column(self, i: int, field_: str | Field, column) -> Table: ... + def remove_column(self, i: int) -> Table: ... + def set_column(self, i: int, field_: str | Field, column) -> Table: ... + def rename_columns(self, names) -> Table: ... + def drop(self, columns: str | list[str]) -> Table: ... + def group_by( + self, keys: str | list[str], use_threads: bool = True + ) -> TableGroupBy: ... + def join( + self, + right_table: Table, + keys: str | list[str], + right_keys: str | list[str] | None = None, + join_type: str = "left outer", + left_suffix: str | None = None, + right_suffix: str | None = None, + coalesce_keys: bool = True, + use_threads: bool = True, + filter_expression: pyarrow.compute.Expression | None = None, + ) -> Table: ... + def join_asof( + self, + right_table: Table, + on: str, + by: str | list[str], + tolerance: int, + right_on: str | list[str] | None = None, + right_by: str | list[str] | None = None, + ) -> Table: ... + def __arrow_c_stream__(self, requested_schema=None): ... + @property + def is_cpu(self): ... - def __or__(self, other: object): - ... +class TableGroupBy: + def __init__(self, table, keys, use_threads: bool = True): ... + def aggregate(self, aggregations) -> Table: ... - def __ror__(self, other: object): - ... +class Tensor(_Weakrefable): + type: DataType + _ssize_t_shape: bytes + _ssize_t_strides: bytes - def __xor__(self, other: object): - ... + def __init__(self): ... + def _make_shape_or_strides_buffer(self, values): ... + @staticmethod + def from_numpy(obj, dim_names: list | None = None) -> Tensor: ... + def to_numpy(self): ... + def equals(self, other: Tensor): ... + def __eq__(self, other): ... + def dim_name(self, i: int) -> str: ... + @property + def dim_names(self) -> list: ... + @property + def is_mutable(self): ... + @property + def is_contiguous(self): ... + @property + def ndim(self): ... + @property + def size(self): ... + @property + def shape(self) -> tuple: ... + @property + def strides(self) -> tuple: ... + @staticmethod + def from_dlpack( + x: object, /, *, device=None, copy: bool | None = None + ) -> Tensor: ... + def __dlpack__( + self, + *, + stream: int | None = None, + max_version=None, + dl_device=None, + copy: bool | None = None, + ): ... + def __dlpack_device__(self) -> tuple: ... - def __rxor__(self, other: object): - ... - - def __lshift__(self, other: object): - ... - - def __rlshift__(self, other: object): - ... - - def __rshift__(self, other: object): - ... - - def __rrshift__(self, other: object): - ... - -class NullScalar(Scalar): - - def __init__(self): - ... - - def as_py(self, *, maps_as_pydicts: str | None=None) -> None: - ... - -class BooleanScalar(Scalar): - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - - def __bool__(self): - ... - -class UInt8Scalar(Scalar): - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - - def __index__(self): - ... - -class Int8Scalar(Scalar): - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - - def __index__(self): - ... - -class UInt16Scalar(Scalar): - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - - def __index__(self): - ... - -class Int16Scalar(Scalar): - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - - def __index__(self): - ... - -class UInt32Scalar(Scalar): - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - - def __index__(self): - ... - -class Int32Scalar(Scalar): - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - - def __index__(self): - ... - -class UInt64Scalar(Scalar): - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - - def __index__(self): - ... - -class Int64Scalar(Scalar): - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - - def __index__(self): - ... - -class HalfFloatScalar(Scalar): - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - - def __float__(self): - ... - - def __int__(self) -> int: - ... - -class FloatScalar(Scalar): - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - - def __float__(self): - ... - - def __int__(self) -> int: - ... - -class DoubleScalar(Scalar): - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - - def __float__(self): - ... - - def __int__(self) -> int: - ... - -class Decimal32Scalar(Scalar): - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - -class Decimal64Scalar(Scalar): - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - -class Decimal128Scalar(Scalar): - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - -class Decimal256Scalar(Scalar): - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - -class Date32Scalar(Scalar): - - @property - def value(self): - ... - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - -class Date64Scalar(Scalar): - - @property - def value(self): - ... - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - -class Time32Scalar(Scalar): - - @property - def value(self): - ... - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - -class Time64Scalar(Scalar): - - @property - def value(self): - ... - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - -class TimestampScalar(Scalar): - - @property - def value(self): - ... - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - - def __repr__(self) -> str: - ... - -class DurationScalar(Scalar): - - @property - def value(self): - ... - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - -class MonthDayNanoIntervalScalar(Scalar): - - @property - def value(self): - ... - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - -class BinaryScalar(Scalar): - - def as_buffer(self) -> Buffer | None: - ... - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - - def __bytes__(self): - ... - -class LargeBinaryScalar(BinaryScalar): - ... - -class FixedSizeBinaryScalar(BinaryScalar): - ... - -class StringScalar(BinaryScalar): - - def as_py(self, *, maps_as_pydicts: str | None=None) -> None | str: - ... - -class LargeStringScalar(StringScalar): - ... - -class BinaryViewScalar(BinaryScalar): - ... - -class StringViewScalar(StringScalar): - ... - -class ListScalar(Scalar, Sequence): - - @property - def values(self) -> Array | None: - ... - - def __len__(self) -> int: - ... - - def __getitem__(self, i): - ... - - def __iter__(self): - ... - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - -class FixedSizeListScalar(ListScalar): - ... - -class LargeListScalar(ListScalar): - ... - -class ListViewScalar(ListScalar): - ... - -class LargeListViewScalar(ListScalar): - ... - -class StructScalar(Scalar, Mapping): - - def __len__(self): - ... - - def __iter__(self) -> Iterator[str]: - ... - - def items(self): - ... - - def __contains__(self, key) -> bool: - ... - - def __getitem__(self, key) -> Scalar: - ... - - def as_py(self, *, maps_as_pydicts: str | None=None) -> dict | None: - ... - - def _as_py_tuple(self) -> list | None: - ... - - def __repr__(self) -> str: - ... - - def __str__(self) -> str: - ... - -class MapScalar(ListScalar, Mapping): - - def __getitem__(self, i): - ... - - def __iter__(self) -> Iterator[tuple]: - ... - - def as_py(self, *, maps_as_pydicts: str | None=None) -> None | list | dict: - ... - - def keys(self) -> list: - ... - -class DictionaryScalar(Scalar): - - @staticmethod - def _reconstruct(type, is_valid, index, dictionary) -> Scalar: - ... - - def __reduce__(self) -> tuple: - ... - - @property - def index(self) -> Scalar: - ... - - @property - def value(self) -> Scalar: - ... - - @property - def dictionary(self) -> Array: - ... - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - -class RunEndEncodedScalar(Scalar): - - @property - def value(self) -> Scalar: - ... - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - -class UnionScalar(Scalar): - - @property - def value(self) -> Scalar | None: - ... - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - - @property - def type_code(self): - ... - -class ExtensionScalar(Scalar): - - @property - def value(self) -> Scalar | None: - ... - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - - @staticmethod - def from_storage(typ: BaseExtensionType, value: object) -> Scalar: - ... - -class JsonScalar(ExtensionScalar): - ... - -class UuidScalar(ExtensionScalar): - - def as_py(self, *, maps_as_pydicts: str | None=None): - ... - -class FixedShapeTensorScalar(ExtensionScalar): - - def to_numpy(self): - ... - - def to_tensor(self) -> Tensor: - ... - - def __dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None): - ... - - def __dlpack_device__(self) -> tuple: - ... - -class OpaqueScalar(ExtensionScalar): - ... - -class Bool8Scalar(ExtensionScalar): - - def as_py(self, *, maps_as_pydicts: str | None=None) -> None | bool: - ... - -class ArrayStatistics(_Weakrefable): - - def __init__(self): - ... - - def __repr__(self) -> str: - ... - - @property - def null_count(self): - ... - - @property - def is_null_count_exact(self): - ... - - @property - def distinct_count(self): - ... - - @property - def is_distinct_count_exact(self): - ... - - @property - def min(self): - ... - - @property - def is_min_exact(self): - ... - - @property - def max(self): - ... - - @property - def is_max_exact(self): - ... - -class _PandasConvertible(_Weakrefable): - - def to_pandas(self, memory_pool: MemoryPool | None=None, categories: list | None=None, strings_to_categorical: bool=False, zero_copy_only: bool=False, integer_object_nulls: bool=False, date_as_object: bool=True, timestamp_as_object: bool=False, use_threads: bool=True, deduplicate_objects: bool=True, ignore_metadata: bool=False, safe: bool=True, split_blocks: bool=False, self_destruct: bool=False, maps_as_pydicts: str | None=None, types_mapper: Callable[..., Any] | None=None, coerce_temporal_nanoseconds: bool=False): - ... - -class Array(_PandasConvertible): - type: DataType - _name: object - - def __init__(self): - ... - - def _debug_print(self) -> None: - ... - - def diff(self, other: Array) -> str: - ... - - def cast(self, target_type: object | None=None, safe: bool | None=None, options: pyarrow.compute.CastOptions | None=None, memory_pool: MemoryPool | None=None) -> Array: - ... - - def view(self, target_type: object) -> Array: - ... - - def sum(self, **kwargs) -> Scalar: - ... - - def unique(self) -> Array: - ... - - def dictionary_encode(self, null_encoding: str='mask') -> DictionaryArray: - ... - - def value_counts(self) -> StructArray: - ... - - @staticmethod - def from_pandas(obj, mask=None, type: DataType | None=None, safe: bool=True, memory_pool: MemoryPool | None=None) -> Array | ChunkedArray: - ... - - def __reduce__(self) -> tuple: - ... - - @staticmethod - def from_buffers(type: DataType, length: int, buffers, null_count: int=-1, offset: int=0, children: list[Array] | None=None) -> Array: - ... - - @property - def null_count(self): - ... - - @property - def nbytes(self): - ... - - def get_total_buffer_size(self) -> int: - ... - - def __sizeof__(self): - ... - - def __iter__(self) -> Iterator[Any]: - ... - - def __repr__(self) -> str: - ... - - def to_string(self, *, indent: int=2, top_level_indent: int=0, window: int=10, container_window: int=2, skip_new_lines: bool=False, element_size_limit: int=100) -> str: - ... - - def __str__(self): - ... - - def __eq__(self, other): - ... - - def equals(self: Array, other: Array) -> bool: - ... - - def __len__(self): - ... - - def is_null(self, *, nan_is_null: bool=False): - ... - - def is_nan(self): - ... - - def is_valid(self): - ... - - def fill_null(self, fill_value: Any) -> Array: - ... - - def __getitem__(self, key) -> Scalar | Array: - ... - - def slice(self, offset: int | Scalar=0, length: int | Scalar | None=None) -> Array: - ... - - def take(self, indices: object) -> Array: - ... - - def drop_null(self): - ... - - def filter(self, mask: object, *, null_selection_behavior: str='drop') -> Array: - ... - - def index(self, value: Scalar | object, start: int | None=None, end: int | None=None, *, memory_pool: MemoryPool | None=None) -> Int64Scalar: - ... - - def sort(self, order: str='ascending', null_placement: str='at_end', **kwargs) -> Array: - ... - - def _to_pandas(self, options, types_mapper=None, **kwargs): - ... - - def __array__(self, dtype=None, copy=None): - ... - - def to_numpy(self, zero_copy_only: bool=True, writable: bool=False): - ... - - def to_tensor(self, *, allow_nulls: bool=False) -> Tensor: - ... - - def to_pylist(self, *, maps_as_pydicts: str | None=None) -> list: - ... - - def tolist(self): - ... - - def validate(self, *, full: bool=False) -> None: - ... - - @property - def offset(self): - ... - - def buffers(self) -> list: - ... - - def copy_to(self, destination: MemoryManager | Device) -> Array: - ... - - def _export_to_c(self, out_ptr, out_schema_ptr: int=0) -> None: - ... - - @staticmethod - def _import_from_c(in_ptr, type) -> Array: - ... - - def __arrow_c_array__(self, requested_schema=None) -> tuple: - ... - - @staticmethod - def _import_from_c_capsule(schema_capsule, array_capsule) -> Array: - ... - - def _export_to_c_device(self, out_ptr, out_schema_ptr: int=0) -> None: - ... - - @staticmethod - def _import_from_c_device(in_ptr, type) -> Array: - ... - - def __arrow_c_device_array__(self, requested_schema=None, **kwargs) -> tuple: - ... - - @staticmethod - def _import_from_c_device_capsule(schema_capsule, array_capsule) -> Array: - ... - - @staticmethod - def from_dlpack(x: object, /, *, device=None, copy: bool | None=None) -> Array: - ... - - def __dlpack__(self, *, stream: int | None=None, max_version=None, dl_device=None, copy: bool | None=None): - ... - - def __dlpack_device__(self) -> tuple: - ... - - @property - def device_type(self) -> DeviceAllocationType: - ... - - @property - def is_cpu(self) -> bool: - ... - - @property - def statistics(self) -> None | ArrayStatistics: - ... - - def __abs__(self): - ... - - def __add__(self, other: object): - ... - - def __radd__(self, other: object): - ... - - def __truediv__(self, other: object): - ... - - def __rtruediv__(self, other: object): - ... - - def __mul__(self, other: object): - ... - - def __rmul__(self, other: object): - ... - - def __neg__(self): - ... - - def __pow__(self, other: object, mod: None=None): - ... - - def __rpow__(self, other: object, mod: None=None): - ... - - def __sub__(self, other: object): - ... - - def __rsub__(self, other: object): - ... - - def __and__(self, other: object): - ... - - def __rand__(self, other: object): - ... - - def __or__(self, other: object): - ... - - def __ror__(self, other: object): - ... - - def __xor__(self, other: object): - ... - - def __rxor__(self, other: object): - ... - - def __lshift__(self, other: object): - ... - - def __rlshift__(self, other: object): - ... - - def __rshift__(self, other: object): - ... - - def __rrshift__(self, other: object): - ... - -class NullArray(Array): - ... - -class BooleanArray(Array): - - @property - def false_count(self): - ... - - @property - def true_count(self): - ... - -class NumericArray(Array): - ... - -class IntegerArray(NumericArray): - ... - -class FloatingPointArray(NumericArray): - ... - -class Int8Array(IntegerArray): - ... - -class UInt8Array(IntegerArray): - ... - -class Int16Array(IntegerArray): - ... - -class UInt16Array(IntegerArray): - ... - -class Int32Array(IntegerArray): - ... - -class UInt32Array(IntegerArray): - ... - -class Int64Array(IntegerArray): - ... - -class UInt64Array(IntegerArray): - ... - -class Date32Array(NumericArray): - ... - -class Date64Array(NumericArray): - ... - -class TimestampArray(NumericArray): - ... - -class Time32Array(NumericArray): - ... - -class Time64Array(NumericArray): - ... - -class DurationArray(NumericArray): - ... - -class MonthDayNanoIntervalArray(Array): - - def to_pylist(self, *, maps_as_pydicts: str | None=None) -> list: - ... - -class HalfFloatArray(FloatingPointArray): - ... - -class FloatArray(FloatingPointArray): - ... - -class DoubleArray(FloatingPointArray): - ... - -class FixedSizeBinaryArray(Array): - ... - -class Decima32Array(FixedSizeBinaryArray): - ... - -class Decimal64Array(FixedSizeBinaryArray): - ... - -class Decimal128Array(FixedSizeBinaryArray): - ... - -class Decimal256Array(FixedSizeBinaryArray): - ... - -class BaseListArray(Array): - - def flatten(self, recursive: bool=False) -> Array: - ... - - def value_parent_indices(self): - ... - - def value_lengths(self): - ... - -class ListArray(BaseListArray): - - @staticmethod - def from_arrays(offsets: Array, values: Array, type: DataType | None=None, pool: MemoryPool | None=None, mask: Array | None=None) -> Array: - ... - - @property - def values(self) -> Array: - ... - - @property - def offsets(self) -> Array: - ... - -class LargeListArray(BaseListArray): - - @staticmethod - def from_arrays(offsets: Array, values: Array, type: DataType | None=None, pool: MemoryPool | None=None, mask: Array | None=None) -> Array: - ... - - @property - def values(self) -> Array: - ... - - @property - def offsets(self) -> Array: - ... - -class ListViewArray(BaseListArray): - - @staticmethod - def from_arrays(offsets: Array, sizes: Array, values: Array, type: DataType | None=None, pool: MemoryPool | None=None, mask: Array | None=None) -> Array: - ... - - @property - def values(self) -> Array: - ... - - @property - def offsets(self) -> Array: - ... - - @property - def sizes(self) -> Array: - ... - -class LargeListViewArray(BaseListArray): - - @staticmethod - def from_arrays(offsets: Array, sizes: Array, values: Array, type: DataType | None=None, pool: MemoryPool | None=None, mask: Array | None=None) -> Array: - ... - - @property - def values(self) -> Array: - ... - - @property - def offsets(self) -> Array: - ... - - @property - def sizes(self) -> Array: - ... - -class MapArray(ListArray): - - @staticmethod - def from_arrays(offsets, keys, items, type: DataType | None=None, pool: MemoryPool | None=None, mask: Array | None=None) -> Array: - ... - - @property - def keys(self) -> Array: - ... - - @property - def items(self) -> Array: - ... - -class FixedSizeListArray(BaseListArray): - - @staticmethod - def from_arrays(values: Array, list_size: int | None=None, type: DataType | None=None, mask: Array | None=None) -> Array: - ... - - @property - def values(self) -> Array: - ... - -class UnionArray(Array): - - def child(self, pos: int) -> Field: - ... - - def field(self, pos: int) -> Array: - ... - - @property - def type_codes(self) -> Array: - ... - - @property - def offsets(self) -> Array: - ... - - @staticmethod - def from_dense(types: Array, value_offsets: Array, children: list[Any], field_names: list[Any] | None=None, type_codes: list[Any] | None=None) -> Array: - ... - - @staticmethod - def from_sparse(types: Array, children: list[Any], field_names: list[Any] | None=None, type_codes: list[Any] | None=None) -> Array: - ... - -class StringArray(Array): - - @staticmethod - def from_buffers(length: int, value_offsets: Buffer, data: Buffer, null_bitmap: Buffer | None=None, null_count: int=-1, offset: int=0) -> Array: - ... - -class LargeStringArray(Array): - - @staticmethod - def from_buffers(length: int, value_offsets: Buffer, data: Buffer, null_bitmap: Buffer | None=None, null_count: int=-1, offset: int=0) -> Array: - ... - -class StringViewArray(Array): - ... - -class BinaryArray(Array): - - @property - def total_values_length(self): - ... - -class LargeBinaryArray(Array): - - @property - def total_values_length(self): - ... - -class BinaryViewArray(Array): - ... - -class DictionaryArray(Array): - - def dictionary_encode(self) -> DictionaryArray: - ... - - def dictionary_decode(self): - ... - - @property - def dictionary(self): - ... - - @property - def indices(self): - ... - - @staticmethod - def from_buffers(type: DataType, length: int, buffers, dictionary: Array, null_count: int=-1, offset: int=0) -> Array: - ... - - @staticmethod - def from_arrays(indices, dictionary, mask=None, ordered: bool=False, from_pandas: bool=False, safe: bool=True, memory_pool: MemoryPool | None=None) -> Array: - ... - -class StructArray(Array): - - def field(self, index) -> Array: - ... - - def _flattened_field(self, index, memory_pool: MemoryPool | None=None) -> Array: - ... - - def flatten(self, memory_pool: MemoryPool | None=None) -> list: - ... - - @staticmethod - def from_arrays(arrays: Sequence[Array], names: list[str] | None=None, fields: list[Field] | None=None, mask=None, memory_pool: MemoryPool | None=None, type: StructType | None=None) -> StructArray: - ... - - def sort(self, order: str='ascending', null_placement: str='at_end', by: str | None=None, **kwargs) -> StructArray: - ... - -class RunEndEncodedArray(Array): - - @staticmethod - def _from_arrays(type, allow_none_for_type, logical_length, run_ends, values, logical_offset) -> Array: - ... - - @staticmethod - def from_arrays(run_ends, values: Array, type: DataType | None=None) -> RunEndEncodedArray: - ... - - @staticmethod - def from_buffers(type: DataType, length: int, buffers: list[Buffer], null_count: int=-1, offset: int=0, children: list[Array] | None=None) -> RunEndEncodedArray: - ... - - @property - def run_ends(self) -> Array: - ... - - @property - def values(self) -> Array: - ... - - def find_physical_offset(self): - ... - - def find_physical_length(self): - ... - -class ExtensionArray(Array): - - @property - def storage(self) -> Array: - ... - - @staticmethod - def from_storage(typ: BaseExtensionType, storage: Array) -> Array: - ... - -class JsonArray(ExtensionArray): - ... - -class UuidArray(ExtensionArray): - ... - -class FixedShapeTensorArray(ExtensionArray): - - def to_numpy_ndarray(self): - ... - - @staticmethod - def from_tensor(tensor: Tensor) -> Array: - ... - - @staticmethod - def from_numpy_ndarray(obj, dim_names: tuple | list[str] | None=None) -> ExtensionArray: - ... - - @staticmethod - def from_dlpack(x: object, /, *, device=None, copy: bool | None=None) -> FixedShapeTensorArray: - ... - - def __dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None): - ... - -class OpaqueArray(ExtensionArray): - ... - -class Bool8Array(ExtensionArray): - - def to_numpy(self, zero_copy_only: bool=True, writable: bool=False): - ... - - @staticmethod - def from_storage(storage: Int8Array) -> ExtensionArray: - ... - - @staticmethod - def from_numpy(obj) -> Bool8Array: - ... - -class StringBuilder(_Weakrefable): - - def __init__(self, memory_pool: MemoryPool | None=None): - ... - - def append(self, value) -> None: - ... - - def append_values(self, values) -> None: - ... - - def finish(self) -> Array: - ... - - @property - def null_count(self): - ... - - def __len__(self): - ... - -class StringViewBuilder(_Weakrefable): - - def __init__(self, memory_pool: MemoryPool | None=None): - ... - - def append(self, value) -> None: - ... - - def append_values(self, values) -> None: - ... - - def finish(self) -> Array: - ... - - @property - def null_count(self): - ... - - def __len__(self): - ... - -class ChunkedArray(_PandasConvertible): - _name: object - - def __init__(self): - ... - - def __reduce__(self) -> tuple: - ... - - @property - def data(self) -> ChunkedArray: - ... - - @property - def type(self) -> DataType: - ... - - def length(self): - ... - - def __len__(self): - ... - - def __repr__(self) -> str: - ... - - def to_string(self, *, indent: int=0, window: int=5, container_window: int=2, skip_new_lines: bool=False, element_size_limit: int=100) -> str: - ... - - def format(self, **kwargs) -> str: - ... - - def __str__(self): - ... - - def validate(self, *, full: bool=False) -> None: - ... - - @property - def null_count(self) -> int: - ... - - @property - def nbytes(self): - ... - - def get_total_buffer_size(self) -> int: - ... - - def __sizeof__(self): - ... - - def __iter__(self) -> Iterator[Any]: - ... - - def __getitem__(self, key) -> Scalar | ChunkedArray: - ... - - def is_null(self, *, nan_is_null: bool=False): - ... - - def is_nan(self): - ... - - def is_valid(self): - ... - - def __eq__(self, other): - ... - - def fill_null(self, fill_value: Any) -> Array | ChunkedArray: - ... - - def equals(self, other: ChunkedArray) -> bool: - ... - - def _to_pandas(self, options, types_mapper=None, **kwargs): - ... - - def to_numpy(self, zero_copy_only: bool=False): - ... - - def __array__(self, dtype=None, copy=None): - ... - - def cast(self, target_type: object | None=None, safe: bool | None=None, options: pyarrow.compute.CastOptions | None=None) -> Array | ChunkedArray: - ... - - def dictionary_encode(self, null_encoding: str='mask') -> ChunkedArray: - ... - - def flatten(self, memory_pool: MemoryPool | None=None) -> list: - ... - - def combine_chunks(self, memory_pool: MemoryPool | None=None) -> Array: - ... - - def unique(self) -> Array: - ... - - def value_counts(self): - ... - - def slice(self, offset: int | Scalar=0, length: int | Scalar | None=None) -> ChunkedArray: - ... - - def filter(self, mask, null_selection_behavior: object='drop') -> Array | ChunkedArray: - ... - - def index(self, value: Scalar | object, start: int | None=None, end: int | None=None, *, memory_pool: MemoryPool | None=None) -> Int64Scalar: - ... - - def take(self, indices: object) -> Array | ChunkedArray: - ... - - def drop_null(self): - ... - - def sort(self, order: str='ascending', null_placement: str='at_end', **kwargs) -> ChunkedArray: - ... - - def unify_dictionaries(self, memory_pool: MemoryPool | None=None) -> ChunkedArray: - ... - - @property - def num_chunks(self) -> int: - ... - - def chunk(self, i: int) -> Array: - ... - - @property - def chunks(self) -> list: - ... - - def iterchunks(self) -> Iterator[Any]: - ... - - def to_pylist(self, *, maps_as_pydicts: str | None=None) -> list: - ... - - def __arrow_c_stream__(self, requested_schema=None): - ... - - @staticmethod - def _import_from_c_capsule(stream) -> ChunkedArray: - ... - - @property - def is_cpu(self): - ... - - def _assert_cpu(self) -> None: - ... - -class _Tabular(_PandasConvertible): - - def __init__(self): - ... - - def __array__(self, dtype=None, copy=None): - ... - - def __dataframe__(self, nan_as_null: bool=False, allow_copy: bool=True): - ... - - def __eq__(self, other): - ... - - def __getitem__(self, key): - ... - - def __len__(self): - ... - - def __repr__(self): - ... - - def _column(self, i: int): - ... - - def _ensure_integer_index(self, i): - ... - - def _is_initialized(self): - ... - - def column(self, i: int | str) -> Array | ChunkedArray: - ... - - @property - def column_names(self) -> list: - ... - - @property - def columns(self) -> list: - ... - - def drop_null(self) -> Table | RecordBatch: - ... - - def field(self, i: int | str) -> Field: - ... - - @classmethod - def from_pydict(cls, mapping: dict | Mapping[Any, Any], schema: Schema | None=None, metadata: dict | Mapping[Any, Any] | None=None) -> Table | RecordBatch: - ... - - @classmethod - def from_pylist(cls, mapping, schema: Schema | None=None, metadata: dict | Mapping[Any, Any] | None=None) -> Table | RecordBatch: - ... - - def itercolumns(self) -> Iterator[Any]: - ... - - @property - def num_columns(self): - ... - - @property - def num_rows(self): - ... - - @property - def shape(self) -> tuple: - ... - - @property - def schema(self): - ... - - def sort_by(self, sorting, **kwargs) -> Table | RecordBatch: - ... - - def take(self, indices: object) -> Table | RecordBatch: - ... - - def filter(self, mask, null_selection_behavior: object='drop') -> Table | RecordBatch: - ... - - def to_pydict(self, *, maps_as_pydicts: str | None=None) -> dict: - ... - - def to_pylist(self, *, maps_as_pydicts: str | None=None) -> list: - ... - - def to_string(self, *, show_metadata: bool=False, preview_cols: int=0) -> str: - ... - - def remove_column(self, i: int): - ... - - def drop_columns(self, columns: str | list[str]) -> _Tabular: - ... - - def add_column(self, i: int, field_, column): - ... - - def append_column(self, field_: str | Field, column) -> Table | RecordBatch: - ... - -class RecordBatch(_Tabular): - - def __init__(self): - ... - - def _is_initialized(self) -> bool: - ... - - def __reduce__(self) -> tuple: - ... - - def validate(self, *, full: bool=False) -> None: - ... - - def replace_schema_metadata(self, metadata: dict | None=None) -> RecordBatch: - ... - - @property - def num_columns(self) -> int: - ... - - @property - def num_rows(self) -> int: - ... - - @property - def schema(self) -> Schema: - ... - - def _column(self, i: int) -> Array: - ... - - @property - def nbytes(self): - ... - - def get_total_buffer_size(self) -> int: - ... - - def __sizeof__(self): - ... - - def add_column(self, i: int, field_: str | Field, column) -> RecordBatch: - ... - - def remove_column(self, i: int) -> RecordBatch: - ... - - def set_column(self, i: int, field_: str | Field, column) -> RecordBatch: - ... - - def rename_columns(self, names) -> RecordBatch: - ... - - def serialize(self, memory_pool: MemoryPool | None=None) -> Buffer: - ... - - def slice(self, offset: int | Scalar=0, length: int | Scalar | None=None) -> RecordBatch: - ... - - def equals(self, other: object, check_metadata: bool=False) -> bool: - ... - - def select(self, columns: object) -> RecordBatch: - ... - - def cast(self, target_schema: Schema, safe: bool | None=None, options: pyarrow.compute.CastOptions | None=None) -> RecordBatch: - ... - - def _to_pandas(self, options, **kwargs): - ... - - @classmethod - def from_pandas(cls, df, schema: Schema | None=None, preserve_index: bool | None=None, nthreads: int | None=None, columns: list | None=None) -> RecordBatch | Self: - ... - - @staticmethod - def from_arrays(arrays: list[Any], names: list[str] | None=None, schema: Schema | None=None, metadata: dict | Mapping[Any, Any] | None=None) -> RecordBatch: - ... - - @staticmethod - def from_struct_array(struct_array: StructArray) -> RecordBatch: - ... - - def to_struct_array(self) -> Array: - ... - - def to_tensor(self, null_to_nan: bool=False, row_major: bool=True, memory_pool: MemoryPool | None=None) -> Tensor: - ... - - def copy_to(self, destination: MemoryManager | Device) -> RecordBatch: - ... - - def _export_to_c(self, out_ptr, out_schema_ptr: int=0) -> None: - ... - - @staticmethod - def _import_from_c(in_ptr, schema) -> RecordBatch: - ... - - def __arrow_c_array__(self, requested_schema=None) -> tuple: - ... - - def __arrow_c_stream__(self, requested_schema=None): - ... - - @staticmethod - def _import_from_c_capsule(schema_capsule, array_capsule) -> RecordBatch: - ... - - def _export_to_c_device(self, out_ptr, out_schema_ptr: int=0) -> None: - ... - - @staticmethod - def _import_from_c_device(in_ptr, schema) -> RecordBatch: - ... - - def __arrow_c_device_array__(self, requested_schema=None, **kwargs) -> tuple: - ... - - @staticmethod - def _import_from_c_device_capsule(schema_capsule, array_capsule) -> RecordBatch: - ... - - @property - def device_type(self) -> DeviceAllocationType: - ... - - @property - def is_cpu(self) -> bool: - ... - -class Table(_Tabular): - - def __init__(self): - ... - - def _is_initialized(self) -> bool: - ... - - def validate(self, *, full: bool=False) -> None: - ... - - def __reduce__(self) -> tuple: - ... - - def slice(self, offset: int | Scalar=0, length: int | Scalar | None=None) -> Table: - ... - - def select(self, columns: object) -> Table: - ... - - def replace_schema_metadata(self, metadata: dict | None=None) -> Table: - ... - - def flatten(self, memory_pool: MemoryPool | None=None) -> Table: - ... - - def combine_chunks(self, memory_pool: MemoryPool | None=None) -> Table: - ... - - def unify_dictionaries(self, memory_pool: MemoryPool | None=None) -> Table: - ... - - def equals(self, other: Table, check_metadata: bool=False) -> bool: - ... - - def cast(self, target_schema: Schema, safe: bool | None=None, options: pyarrow.compute.CastOptions | None=None) -> Table: - ... - - @classmethod - def from_pandas(cls, df, schema: Schema | None=None, preserve_index: bool | None=None, nthreads: int | None=None, columns: list | None=None, safe: bool=True) -> Table | Self: - ... - - @staticmethod - def from_arrays(arrays: list[Array] | ChunkedArray, names: list[str] | None=None, schema: Schema | None=None, metadata: dict | Mapping[Any, Any] | None=None) -> Table: - ... - - @staticmethod - def from_struct_array(struct_array: StructArray | ChunkedArray) -> Table: - ... - - def to_struct_array(self, max_chunksize: int | None=None) -> ChunkedArray: - ... - - @staticmethod - def from_batches(batches: Sequence[Any] | Iterator[RecordBatch], schema: Schema | None=None) -> Table: - ... - - def to_batches(self, max_chunksize: int | None=None) -> list[RecordBatch]: - ... - - def to_tensor(self, null_to_nan: bool=False, row_major: bool=True, memory_pool: MemoryPool | None=None) -> Tensor: - ... - - def to_reader(self, max_chunksize: int | None=None) -> RecordBatchReader: - ... - - def _to_pandas(self, options, categories=None, ignore_metadata: bool=False, types_mapper=None): - ... - - @property - def schema(self) -> Schema: - ... - - def _column(self, i: int) -> ChunkedArray: - ... - - @property - def num_columns(self) -> int: - ... - - @property - def num_rows(self) -> int: - ... - - @property - def nbytes(self): - ... - - def get_total_buffer_size(self) -> int: - ... - - def __sizeof__(self): - ... - - def add_column(self, i: int, field_: str | Field, column) -> Table: - ... - - def remove_column(self, i: int) -> Table: - ... - - def set_column(self, i: int, field_: str | Field, column) -> Table: - ... - - def rename_columns(self, names) -> Table: - ... - - def drop(self, columns: str | list[str]) -> Table: - ... - - def group_by(self, keys: str | list[str], use_threads: bool=True) -> TableGroupBy: - ... - - def join(self, right_table: Table, keys: str | list[str], right_keys: str | list[str] | None=None, join_type: str='left outer', left_suffix: str | None=None, right_suffix: str | None=None, coalesce_keys: bool=True, use_threads: bool=True, filter_expression: pyarrow.compute.Expression | None=None) -> Table: - ... - - def join_asof(self, right_table: Table, on: str, by: str | list[str], tolerance: int, right_on: str | list[str] | None=None, right_by: str | list[str] | None=None) -> Table: - ... - - def __arrow_c_stream__(self, requested_schema=None): - ... - - @property - def is_cpu(self): - ... - -class TableGroupBy: - - def __init__(self, table, keys, use_threads: bool=True): - ... - - def aggregate(self, aggregations) -> Table: - ... - -class Tensor(_Weakrefable): - type: DataType - _ssize_t_shape: bytes - _ssize_t_strides: bytes - - def __init__(self): - ... - - def _make_shape_or_strides_buffer(self, values): - ... - - def __repr__(self) -> str: - ... - - @staticmethod - def from_numpy(obj, dim_names: list | None=None) -> Tensor: - ... - - def to_numpy(self): - ... - - def equals(self, other: Tensor): - ... - - def __eq__(self, other): - ... - - def dim_name(self, i: int) -> str: - ... - - @property - def dim_names(self) -> list: - ... - - @property - def is_mutable(self): - ... - - @property - def is_contiguous(self): - ... - - @property - def ndim(self): - ... - - @property - def size(self): - ... - - @property - def shape(self) -> tuple: - ... - - @property - def strides(self) -> tuple: - ... - - @staticmethod - def from_dlpack(x: object, /, *, device=None, copy: bool | None=None) -> Tensor: - ... - - def __dlpack__(self, *, stream: int | None=None, max_version=None, dl_device=None, copy: bool | None=None): - ... - - def __dlpack_device__(self) -> tuple: - ... - -class SparseCOOTensor(_Weakrefable): - type: DataType - - def __init__(self): - ... - - def __repr__(self) -> str: - ... +class SparseCOOTensor(_Weakrefable): + type: DataType + def __init__(self): ... @classmethod - def from_dense_numpy(cls, obj, dim_names: list[str] | None=None) -> Self: - ... - - @staticmethod - def from_numpy(data, coords, shape: tuple, dim_names: list | None=None) -> SparseCOOTensor: - ... - - @staticmethod - def from_scipy(obj, dim_names: list | None=None) -> SparseCOOTensor: - ... - - @staticmethod - def from_pydata_sparse(obj, dim_names: list | None=None) -> SparseCOOTensor: - ... - + def from_dense_numpy(cls, obj, dim_names: list[str] | None = None) -> Self: ... @staticmethod - def from_tensor(obj: Tensor) -> SparseCOOTensor: - ... - - def to_numpy(self) -> tuple: - ... - - def to_scipy(self): - ... - - def to_pydata_sparse(self): - ... - - def to_tensor(self) -> Tensor: - ... - - def equals(self, other: SparseCOOTensor): - ... - - def __eq__(self, other): - ... - + def from_numpy( + data, coords, shape: tuple, dim_names: list | None = None + ) -> SparseCOOTensor: ... + @staticmethod + def from_scipy(obj, dim_names: list | None = None) -> SparseCOOTensor: ... + @staticmethod + def from_pydata_sparse(obj, dim_names: list | None = None) -> SparseCOOTensor: ... + @staticmethod + def from_tensor(obj: Tensor) -> SparseCOOTensor: ... + def to_numpy(self) -> tuple: ... + def to_scipy(self): ... + def to_pydata_sparse(self): ... + def to_tensor(self) -> Tensor: ... + def equals(self, other: SparseCOOTensor): ... + def __eq__(self, other): ... @property - def is_mutable(self): - ... - + def is_mutable(self): ... @property - def ndim(self): - ... - + def ndim(self): ... @property - def shape(self) -> tuple: - ... - + def shape(self) -> tuple: ... @property - def size(self): - ... - - def dim_name(self, i: int) -> str: - ... - + def size(self): ... + def dim_name(self, i: int) -> str: ... @property - def dim_names(self) -> tuple: - ... - + def dim_names(self) -> tuple: ... @property - def non_zero_length(self): - ... - + def non_zero_length(self): ... @property - def has_canonical_format(self): - ... + def has_canonical_format(self): ... class SparseCSRMatrix(_Weakrefable): type: DataType - def __init__(self): - ... - - def __repr__(self) -> str: - ... - + def __init__(self): ... @classmethod - def from_dense_numpy(cls, obj, dim_names: list | None=None) -> Self: - ... - + def from_dense_numpy(cls, obj, dim_names: list | None = None) -> Self: ... @staticmethod - def from_numpy(data, indptr, indices, shape: tuple, dim_names: list | None=None) -> SparseCSRMatrix: - ... - + def from_numpy( + data, indptr, indices, shape: tuple, dim_names: list | None = None + ) -> SparseCSRMatrix: ... @staticmethod - def from_scipy(obj, dim_names: list | None=None) -> SparseCSRMatrix: - ... - + def from_scipy(obj, dim_names: list | None = None) -> SparseCSRMatrix: ... @staticmethod - def from_tensor(obj: Tensor) -> SparseCSRMatrix: - ... - - def to_numpy(self) -> tuple: - ... - - def to_scipy(self): - ... - - def to_tensor(self) -> Tensor: - ... - - def equals(self, other: SparseCSRMatrix): - ... - - def __eq__(self, other): - ... - + def from_tensor(obj: Tensor) -> SparseCSRMatrix: ... + def to_numpy(self) -> tuple: ... + def to_scipy(self): ... + def to_tensor(self) -> Tensor: ... + def equals(self, other: SparseCSRMatrix): ... + def __eq__(self, other): ... @property - def is_mutable(self): - ... - + def is_mutable(self): ... @property - def ndim(self): - ... - + def ndim(self): ... @property - def shape(self) -> tuple: - ... - + def shape(self) -> tuple: ... @property - def size(self): - ... - - def dim_name(self, i: int) -> str: - ... - + def size(self): ... + def dim_name(self, i: int) -> str: ... @property - def dim_names(self) -> tuple: - ... - + def dim_names(self) -> tuple: ... @property - def non_zero_length(self): - ... + def non_zero_length(self): ... class SparseCSCMatrix(_Weakrefable): type: DataType - def __init__(self): - ... - - def __repr__(self) -> str: - ... - + def __init__(self): ... @classmethod - def from_dense_numpy(cls, obj, dim_names: list[str] | None=None) -> Self: - ... - + def from_dense_numpy(cls, obj, dim_names: list[str] | None = None) -> Self: ... @staticmethod - def from_numpy(data, indptr, indices, shape: tuple, dim_names: list | None=None) -> SparseCSCMatrix: - ... - + def from_numpy( + data, indptr, indices, shape: tuple, dim_names: list | None = None + ) -> SparseCSCMatrix: ... @staticmethod - def from_scipy(obj, dim_names: list | None=None) -> SparseCSCMatrix: - ... - + def from_scipy(obj, dim_names: list | None = None) -> SparseCSCMatrix: ... @staticmethod - def from_tensor(obj: Tensor) -> SparseCSCMatrix: - ... - - def to_numpy(self) -> tuple: - ... - - def to_scipy(self): - ... - - def to_tensor(self) -> Tensor: - ... - - def equals(self, other: SparseCSCMatrix): - ... - - def __eq__(self, other): - ... - + def from_tensor(obj: Tensor) -> SparseCSCMatrix: ... + def to_numpy(self) -> tuple: ... + def to_scipy(self): ... + def to_tensor(self) -> Tensor: ... + def equals(self, other: SparseCSCMatrix): ... + def __eq__(self, other): ... @property - def is_mutable(self): - ... - + def is_mutable(self): ... @property - def ndim(self): - ... - + def ndim(self): ... @property - def shape(self) -> tuple: - ... - + def shape(self) -> tuple: ... @property - def size(self): - ... - - def dim_name(self, i: int) -> str: - ... - + def size(self): ... + def dim_name(self, i: int) -> str: ... @property - def dim_names(self) -> tuple: - ... - + def dim_names(self) -> tuple: ... @property - def non_zero_length(self): - ... + def non_zero_length(self): ... class SparseCSFTensor(_Weakrefable): type: DataType - def __init__(self): - ... - - def __repr__(self) -> str: - ... - + def __init__(self): ... @classmethod - def from_dense_numpy(cls, obj, dim_names: list[str] | None=None) -> Self: - ... - + def from_dense_numpy(cls, obj, dim_names: list[str] | None = None) -> Self: ... @staticmethod - def from_numpy(data, indptr, indices, shape: tuple, axis_order: list | None=None, dim_names: list | None=None) -> SparseCSFTensor: - ... - + def from_numpy( + data, + indptr, + indices, + shape: tuple, + axis_order: list | None = None, + dim_names: list | None = None, + ) -> SparseCSFTensor: ... @staticmethod - def from_tensor(obj: Tensor) -> SparseCSFTensor: - ... - - def to_numpy(self) -> tuple: - ... - - def to_tensor(self) -> Tensor: - ... - - def equals(self, other: SparseCSFTensor): - ... - - def __eq__(self, other): - ... - + def from_tensor(obj: Tensor) -> SparseCSFTensor: ... + def to_numpy(self) -> tuple: ... + def to_tensor(self) -> Tensor: ... + def equals(self, other: SparseCSFTensor): ... + def __eq__(self, other): ... @property - def is_mutable(self): - ... - + def is_mutable(self): ... @property - def ndim(self): - ... - + def ndim(self): ... @property - def shape(self) -> tuple: - ... - + def shape(self) -> tuple: ... @property - def size(self): - ... - - def dim_name(self, i: int) -> str: - ... - + def size(self): ... + def dim_name(self, i: int) -> str: ... @property - def dim_names(self) -> tuple: - ... - + def dim_names(self) -> tuple: ... @property - def non_zero_length(self): - ... + def non_zero_length(self): ... class NativeFile(_Weakrefable): _default_chunk_size: int - def __init__(self): - ... - - def __enter__(self) -> Self: - ... - - def __exit__(self, exc_type, exc_value, tb) -> None: - ... - - def __repr__(self) -> str: - ... - - @property - def mode(self) -> str: - ... - - def readable(self): - ... - - def writable(self): - ... - - def seekable(self): - ... - - def isatty(self) -> bool: - ... - - def fileno(self): - ... - - @property - def closed(self): - ... - - def close(self) -> None: - ... - - def _assert_open(self) -> None: - ... - - def _assert_readable(self) -> None: - ... - - def _assert_writable(self) -> None: - ... - - def _assert_seekable(self) -> None: - ... - - def size(self) -> int: - ... - - def metadata(self) -> dict: - ... - - def tell(self) -> int: - ... - - def seek(self, position: int, whence: int=0) -> int: - ... - - def flush(self) -> None: - ... - - def write(self, data) -> int: - ... - - def read(self, nbytes: int | None=None) -> bytes: - ... - - def get_stream(self, file_offset: int, nbytes: int) -> NativeFile: - ... - - def read_at(self, nbytes: int, offset: int) -> bytes: - ... - - def read1(self, nbytes: int | None=None) -> bytes: - ... - - def readall(self): - ... - - def readinto(self, b) -> int: - ... - - def readline(self, size: int | None=None): - ... - - def readlines(self, hint: int | None=None): - ... - - def __iter__(self) -> Self: - ... - - def __next__(self): - ... - - def read_buffer(self, nbytes: int | None=None): - ... - - def truncate(self): - ... - - def writelines(self, lines: Iterable[Any]) -> None: - ... - - def download(self, stream_or_path: str | IO[Any], buffer_size: int | None=None): - ... - - def _download_nothreads(self, stream_or_path, buffer_size=None) -> None: - ... - - def upload(self, stream: IO[Any], buffer_size: int | None=None): - ... - - def _upload_nothreads(self, stream, buffer_size=None) -> None: - ... + def __init__(self): ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type, exc_value, tb) -> None: ... + @property + def mode(self) -> str: ... + def readable(self): ... + def writable(self): ... + def seekable(self): ... + def isatty(self) -> bool: ... + def fileno(self): ... + @property + def closed(self): ... + def close(self) -> None: ... + def _assert_open(self) -> None: ... + def _assert_readable(self) -> None: ... + def _assert_writable(self) -> None: ... + def _assert_seekable(self) -> None: ... + def size(self) -> int: ... + def metadata(self) -> dict: ... + def tell(self) -> int: ... + def seek(self, position: int, whence: int = 0) -> int: ... + def flush(self) -> None: ... + def write(self, data) -> int: ... + def read(self, nbytes: int | None = None) -> bytes: ... + def get_stream(self, file_offset: int, nbytes: int) -> NativeFile: ... + def read_at(self, nbytes: int, offset: int) -> bytes: ... + def read1(self, nbytes: int | None = None) -> bytes: ... + def readall(self): ... + def readinto(self, b) -> int: ... + def readline(self, size: int | None = None): ... + def readlines(self, hint: int | None = None): ... + def __iter__(self) -> Self: ... + def __next__(self): ... + def read_buffer(self, nbytes: int | None = None): ... + def truncate(self): ... + def writelines(self, lines: Iterable[Any]) -> None: ... + def download( + self, stream_or_path: str | IO[Any], buffer_size: int | None = None + ): ... + def _download_nothreads(self, stream_or_path, buffer_size=None) -> None: ... + def upload(self, stream: IO[Any], buffer_size: int | None = None): ... + def _upload_nothreads(self, stream, buffer_size=None) -> None: ... class PythonFile(NativeFile): - - def __init__(self, handle, mode=None): - ... - - def truncate(self, pos: int | None=None) -> None: - ... - - def readline(self, size: int | None=None): - ... - - def readlines(self, hint: int | None=None): - ... + def __init__(self, handle, mode=None): ... + def truncate(self, pos: int | None = None) -> None: ... + def readline(self, size: int | None = None): ... + def readlines(self, hint: int | None = None): ... class MemoryMappedFile(NativeFile): - @staticmethod - def create(path: str, size: int) -> MemoryMappedFile: - ... - - def _open(self, path, mode: str='r') -> None: - ... - - def resize(self, new_size) -> None: - ... - - def fileno(self): - ... + def create(path: str, size: int) -> MemoryMappedFile: ... + def _open(self, path, mode: str = "r") -> None: ... + def resize(self, new_size) -> None: ... + def fileno(self): ... class OSFile(NativeFile): - - def __init__(self, path, mode: str='r', memory_pool: MemoryPool | None=None): - ... - - def fileno(self): - ... + def __init__( + self, path, mode: str = "r", memory_pool: MemoryPool | None = None + ): ... + def fileno(self): ... class FixedSizeBufferWriter(NativeFile): - - def __init__(self, buffer: Buffer): - ... - - def set_memcopy_threads(self, num_threads: int) -> None: - ... - - def set_memcopy_blocksize(self, blocksize: int) -> None: - ... - - def set_memcopy_threshold(self, threshold: int) -> None: - ... + def __init__(self, buffer: Buffer): ... + def set_memcopy_threads(self, num_threads: int) -> None: ... + def set_memcopy_blocksize(self, blocksize: int) -> None: ... + def set_memcopy_threshold(self, threshold: int) -> None: ... class Buffer(_Weakrefable): - - def __init__(self): - ... - - def __len__(self): - ... - - def __repr__(self) -> str: - ... - - def _assert_cpu(self) -> None: - ... - + def __init__(self): ... + def __len__(self): ... + def _assert_cpu(self) -> None: ... @property - def size(self): - ... - + def size(self): ... @property - def address(self): - ... - - def hex(self): - ... - + def address(self): ... + def hex(self): ... @property - def is_mutable(self): - ... - + def is_mutable(self): ... @property - def is_cpu(self): - ... - + def is_cpu(self): ... @property - def device(self) -> Device: - ... - + def device(self) -> Device: ... @property - def memory_manager(self) -> MemoryManager: - ... - + def memory_manager(self) -> MemoryManager: ... @property - def device_type(self) -> DeviceAllocationType: - ... - + def device_type(self) -> DeviceAllocationType: ... @property - def parent(self) -> None | Buffer: - ... - - def __getitem__(self, key): - ... - - def slice(self, offset: int=0, length: int | None=None) -> Buffer: - ... - - def equals(self, other: Buffer) -> bool: - ... - - def __eq__(self, other): - ... - - def __reduce_ex__(self, protocol) -> tuple: - ... - - def to_pybytes(self): - ... + def parent(self) -> None | Buffer: ... + def __getitem__(self, key): ... + def slice(self, offset: int = 0, length: int | None = None) -> Buffer: ... + def equals(self, other: Buffer) -> bool: ... + def __eq__(self, other): ... + def __reduce_ex__(self, protocol) -> tuple: ... + def to_pybytes(self): ... class ResizableBuffer(Buffer): - - def resize(self, new_size: int, shrink_to_fit: bool=False) -> None: - ... + def resize(self, new_size: int, shrink_to_fit: bool = False) -> None: ... class BufferOutputStream(NativeFile): - - def __init__(self, memory_pool: MemoryPool | None=None): - ... - - def getvalue(self) -> Buffer: - ... + def __init__(self, memory_pool: MemoryPool | None = None): ... + def getvalue(self) -> Buffer: ... class MockOutputStream(NativeFile): - - def __init__(self): - ... - - def size(self): - ... + def __init__(self): ... + def size(self): ... class BufferReader(NativeFile): - - def __init__(self, obj): - ... + def __init__(self, obj): ... class CompressedInputStream(NativeFile): - - def __init__(self, stream: object, compression: str): - ... + def __init__(self, stream: object, compression: str): ... class CompressedOutputStream(NativeFile): - - def __init__(self, stream: object, compression: str): - ... + def __init__(self, stream: object, compression: str): ... class BufferedInputStream(NativeFile): - - def __init__(self, stream: NativeFile, buffer_size: int, memory_pool: MemoryPool | None=None): - ... - - def detach(self) -> NativeFile: - ... + def __init__( + self, + stream: NativeFile, + buffer_size: int, + memory_pool: MemoryPool | None = None, + ): ... + def detach(self) -> NativeFile: ... class BufferedOutputStream(NativeFile): - - def __init__(self, stream: NativeFile, buffer_size: int, memory_pool: MemoryPool | None=None): - ... - - def detach(self) -> NativeFile: - ... + def __init__( + self, + stream: NativeFile, + buffer_size: int, + memory_pool: MemoryPool | None = None, + ): ... + def detach(self) -> NativeFile: ... class TransformInputStream(NativeFile): - - def __init__(self, stream: NativeFile, transform_func): - ... + def __init__(self, stream: NativeFile, transform_func): ... class Transcoder: - - def __init__(self, decoder, encoder): - ... - - def __call__(self, buf): - ... + def __init__(self, decoder, encoder): ... + def __call__(self, buf): ... class CacheOptions(_Weakrefable): - - def __init__(self, *, hole_size_limit=None, range_size_limit=None, lazy=None, prefetch_limit=None): - ... - - @property - def hole_size_limit(self) -> None: - ... - + def __init__( + self, + *, + hole_size_limit=None, + range_size_limit=None, + lazy=None, + prefetch_limit=None, + ): ... + @property + def hole_size_limit(self) -> None: ... @hole_size_limit.setter - def hole_size_limit(self, hole_size_limit) -> None: - ... - + def hole_size_limit(self, hole_size_limit) -> None: ... @property - def range_size_limit(self) -> None: - ... - + def range_size_limit(self) -> None: ... @range_size_limit.setter - def range_size_limit(self, range_size_limit) -> None: - ... - + def range_size_limit(self, range_size_limit) -> None: ... @property - def lazy(self) -> None: - ... - + def lazy(self) -> None: ... @lazy.setter - def lazy(self, lazy) -> None: - ... - + def lazy(self, lazy) -> None: ... @property - def prefetch_limit(self) -> None: - ... - + def prefetch_limit(self) -> None: ... @prefetch_limit.setter - def prefetch_limit(self, prefetch_limit) -> None: - ... - - def __eq__(self, other: CacheOptions): - ... - + def prefetch_limit(self, prefetch_limit) -> None: ... + def __eq__(self, other: CacheOptions): ... @staticmethod - def from_network_metrics(time_to_first_byte_millis: int, transfer_bandwidth_mib_per_sec: int, ideal_bandwidth_utilization_frac: int | float=0.9, max_ideal_request_size_mib: int=64) -> CacheOptions: - ... - + def from_network_metrics( + time_to_first_byte_millis: int, + transfer_bandwidth_mib_per_sec: int, + ideal_bandwidth_utilization_frac: float = 0.9, + max_ideal_request_size_mib: int = 64, + ) -> CacheOptions: ... @staticmethod - def _reconstruct(kwargs) -> CacheOptions: - ... - - def __reduce__(self) -> tuple: - ... + def _reconstruct(kwargs) -> CacheOptions: ... + def __reduce__(self) -> tuple: ... class Codec(_Weakrefable): - - def __init__(self, compression: str, compression_level=None): - ... - + def __init__(self, compression: str, compression_level=None): ... @staticmethod - def detect(path: str | os.PathLike[str]) -> Codec: - ... - + def detect(path: str | os.PathLike[str]) -> Codec: ... @staticmethod - def is_available(compression: str) -> bool: - ... - + def is_available(compression: str) -> bool: ... @staticmethod - def supports_compression_level(compression: str): - ... - + def supports_compression_level(compression: str): ... @staticmethod - def default_compression_level(compression: str): - ... - + def default_compression_level(compression: str): ... @staticmethod - def minimum_compression_level(compression: str): - ... - + def minimum_compression_level(compression: str): ... @staticmethod - def maximum_compression_level(compression: str): - ... - + def maximum_compression_level(compression: str): ... @property - def name(self) -> str: - ... - + def name(self) -> str: ... @property - def compression_level(self): - ... - - def compress(self, buf: object, asbytes: bool=False, memory_pool: MemoryPool | None=None) -> Buffer | bytes: - ... - - def decompress(self, buf: object, decompressed_size: int | None=None, asbytes: bool=False, memory_pool: MemoryPool | None=None) -> Buffer | bytes: - ... - - def __repr__(self) -> str: - ... + def compression_level(self): ... + def compress( + self, buf: object, asbytes: bool = False, memory_pool: MemoryPool | None = None + ) -> Buffer | bytes: ... + def decompress( + self, + buf: object, + decompressed_size: int | None = None, + asbytes: bool = False, + memory_pool: MemoryPool | None = None, + ) -> Buffer | bytes: ... class WriteStats(_WriteStats): __slots__ = () @@ -4060,283 +2311,172 @@ class ReadStats(_ReadStats): class IpcReadOptions(_Weakrefable): __slots__ = () - def __init__(self, *, ensure_native_endian: bool=True, ensure_alignment: Alignment=Alignment.Any, use_threads: bool=True, included_fields: list[Any] | None=None): - ... - + def __init__( + self, + *, + ensure_native_endian: bool = True, + ensure_alignment: Alignment = ..., + use_threads: bool = True, + included_fields: list[Any] | None = None, + ): ... @property - def ensure_native_endian(self) -> None: - ... - + def ensure_native_endian(self) -> None: ... @ensure_native_endian.setter - def ensure_native_endian(self, value: bool) -> None: - ... - + def ensure_native_endian(self, value: bool) -> None: ... @property - def ensure_alignment(self) -> None: - ... - + def ensure_alignment(self) -> None: ... @ensure_alignment.setter - def ensure_alignment(self, value: Alignment) -> None: - ... - + def ensure_alignment(self, value: Alignment) -> None: ... @property - def use_threads(self) -> None: - ... - + def use_threads(self) -> None: ... @use_threads.setter - def use_threads(self, value: bool) -> None: - ... - + def use_threads(self, value: bool) -> None: ... @property - def included_fields(self) -> None: - ... - + def included_fields(self) -> None: ... @included_fields.setter - def included_fields(self, value: list[Any]) -> None: - ... - - def __repr__(self) -> str: - ... + def included_fields(self, value: list[Any]) -> None: ... class IpcWriteOptions(_Weakrefable): __slots__ = () - def __init__(self, *, metadata_version=MetadataVersion.V5, allow_64bit: bool=False, use_legacy_format: bool=False, compression=None, use_threads: bool=True, emit_dictionary_deltas: bool=False, unify_dictionaries: bool=False): - ... - - @property - def allow_64bit(self) -> None: - ... - + def __init__( + self, + *, + metadata_version=..., + allow_64bit: bool = False, + use_legacy_format: bool = False, + compression=None, + use_threads: bool = True, + emit_dictionary_deltas: bool = False, + unify_dictionaries: bool = False, + ): ... + @property + def allow_64bit(self) -> None: ... @allow_64bit.setter - def allow_64bit(self, value: bool) -> None: - ... - + def allow_64bit(self, value: bool) -> None: ... @property - def use_legacy_format(self) -> None: - ... - + def use_legacy_format(self) -> None: ... @use_legacy_format.setter - def use_legacy_format(self, value: bool) -> None: - ... - + def use_legacy_format(self, value: bool) -> None: ... @property - def metadata_version(self) -> None: - ... - + def metadata_version(self) -> None: ... @metadata_version.setter - def metadata_version(self, value) -> None: - ... - + def metadata_version(self, value) -> None: ... @property - def compression(self) -> None: - ... - + def compression(self) -> None: ... @compression.setter - def compression(self, value) -> None: - ... - + def compression(self, value) -> None: ... @property - def use_threads(self) -> None: - ... - + def use_threads(self) -> None: ... @use_threads.setter - def use_threads(self, value: bool) -> None: - ... - + def use_threads(self, value: bool) -> None: ... @property - def emit_dictionary_deltas(self) -> None: - ... - + def emit_dictionary_deltas(self) -> None: ... @emit_dictionary_deltas.setter - def emit_dictionary_deltas(self, value: bool) -> None: - ... - + def emit_dictionary_deltas(self, value: bool) -> None: ... @property - def unify_dictionaries(self) -> None: - ... - + def unify_dictionaries(self) -> None: ... @unify_dictionaries.setter - def unify_dictionaries(self, value: bool) -> None: - ... - - def __repr__(self) -> str: - ... + def unify_dictionaries(self, value: bool) -> None: ... class Message(_Weakrefable): - - def __init__(self): - ... - + def __init__(self): ... @property - def type(self) -> str: - ... - + def type(self) -> str: ... @property - def metadata(self) -> Buffer: - ... - + def metadata(self) -> Buffer: ... @property - def metadata_version(self): - ... - + def metadata_version(self): ... @property - def body(self) -> None | Buffer: - ... - - def equals(self, other: Message) -> bool: - ... - - def serialize_to(self, sink: NativeFile, alignment: int=8, memory_pool: MemoryPool | None=None) -> None: - ... - - def serialize(self, alignment: int=8, memory_pool: MemoryPool | None=None) -> Buffer: - ... - - def __repr__(self) -> str: - ... + def body(self) -> None | Buffer: ... + def equals(self, other: Message) -> bool: ... + def serialize_to( + self, + sink: NativeFile, + alignment: int = 8, + memory_pool: MemoryPool | None = None, + ) -> None: ... + def serialize( + self, alignment: int = 8, memory_pool: MemoryPool | None = None + ) -> Buffer: ... class MessageReader(_Weakrefable): - - def __init__(self): - ... - + def __init__(self): ... @staticmethod - def open_stream(source) -> MessageReader: - ... - - def __iter__(self) -> Self: - ... - - def __next__(self): - ... - - def read_next_message(self) -> Message: - ... + def open_stream(source) -> MessageReader: ... + def __iter__(self) -> Self: ... + def __next__(self): ... + def read_next_message(self) -> Message: ... class _CRecordBatchWriter(_Weakrefable): - - def write(self, table_or_batch) -> None: - ... - - def write_batch(self, batch: RecordBatch, custom_metadata: Mapping[Any, Any] | KeyValueMetadata | None=None) -> None: - ... - - def write_table(self, table: Table, max_chunksize: int | None=None) -> None: - ... - - def close(self) -> None: - ... - - def __enter__(self) -> Self: - ... - - def __exit__(self, exc_type, exc_val, exc_tb) -> None: - ... - - @property - def stats(self): - ... + def write(self, table_or_batch) -> None: ... + def write_batch( + self, + batch: RecordBatch, + custom_metadata: Mapping[Any, Any] | KeyValueMetadata | None = None, + ) -> None: ... + def write_table(self, table: Table, max_chunksize: int | None = None) -> None: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type, exc_val, exc_tb) -> None: ... + @property + def stats(self): ... class _RecordBatchStreamWriter(_CRecordBatchWriter): - - def __init__(self): - ... - + def __init__(self): ... @property - def _use_legacy_format(self): - ... - + def _use_legacy_format(self): ... @property - def _metadata_version(self): - ... - - def _open(self, sink, schema: Schema, options: IpcWriteOptions=IpcWriteOptions()) -> None: - ... + def _metadata_version(self): ... + def _open(self, sink, schema: Schema, options: IpcWriteOptions = ...) -> None: ... class _ReadPandasMixin: - - def read_pandas(self, **options): - ... + def read_pandas(self, **options): ... class RecordBatchReader(_Weakrefable): read_pandas = _ReadPandasMixin.read_pandas - def __init__(self): - ... - - def __iter__(self) -> Self: - ... - - def __next__(self): - ... - + def __init__(self): ... + def __iter__(self) -> Self: ... + def __next__(self): ... @property - def schema(self) -> Schema: - ... - - def read_next_batch(self) -> RecordBatch: - ... - - def read_next_batch_with_custom_metadata(self): - ... - - def iter_batches_with_custom_metadata(self) -> Iterator[Any]: - ... - - def read_all(self) -> Table: - ... - - def close(self) -> None: - ... - - def __enter__(self) -> Self: - ... - - def __exit__(self, exc_type, exc_val, exc_tb) -> None: - ... - - def cast(self, target_schema: Schema) -> RecordBatchReader: - ... - - def _export_to_c(self, out_ptr) -> None: - ... - + def schema(self) -> Schema: ... + def read_next_batch(self) -> RecordBatch: ... + def read_next_batch_with_custom_metadata(self): ... + def iter_batches_with_custom_metadata(self) -> Iterator[Any]: ... + def read_all(self) -> Table: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type, exc_val, exc_tb) -> None: ... + def cast(self, target_schema: Schema) -> RecordBatchReader: ... + def _export_to_c(self, out_ptr) -> None: ... @staticmethod - def _import_from_c(in_ptr) -> RecordBatchReader: - ... - - def __arrow_c_stream__(self, requested_schema=None): - ... - + def _import_from_c(in_ptr) -> RecordBatchReader: ... + def __arrow_c_stream__(self, requested_schema=None): ... @staticmethod - def _import_from_c_capsule(stream) -> RecordBatchReader: - ... - + def _import_from_c_capsule(stream) -> RecordBatchReader: ... @staticmethod - def from_stream(data, schema: Schema | None=None) -> RecordBatchReader: - ... - + def from_stream(data, schema: Schema | None = None) -> RecordBatchReader: ... @staticmethod - def from_batches(schema: Schema, batches: Iterable[RecordBatch]) -> RecordBatchReader: - ... + def from_batches( + schema: Schema, batches: Iterable[RecordBatch] + ) -> RecordBatchReader: ... class _RecordBatchStreamReader(RecordBatchReader): - - def __init__(self): - ... - - def _open(self, source, options: IpcReadOptions=IpcReadOptions(), memory_pool: MemoryPool | None=None) -> None: - ... - + def __init__(self): ... + def _open( + self, + source, + options: IpcReadOptions = ..., + memory_pool: MemoryPool | None = None, + ) -> None: ... @property - def stats(self): - ... + def stats(self): ... class _RecordBatchFileWriter(_RecordBatchStreamWriter): - - def _open(self, sink, schema: Schema, options: IpcWriteOptions=IpcWriteOptions(), metadata=None) -> None: - ... + def _open( + self, sink, schema: Schema, options: IpcWriteOptions = ..., metadata=None + ) -> None: ... class RecordBatchWithMetadata(_RecordBatchWithMetadata): __slots__ = () @@ -4345,42 +2485,27 @@ class _RecordBatchFileReader(_Weakrefable): read_pandas = _ReadPandasMixin.read_pandas schema: Schema - def __init__(self): - ... - - def _open(self, source, footer_offset=None, options: IpcReadOptions=IpcReadOptions(), memory_pool: MemoryPool | None=None) -> None: - ... - - @property - def num_record_batches(self): - ... - - def count_rows(self) -> int: - ... - - def get_batch(self, i: int) -> RecordBatch: - ... + def __init__(self): ... + def _open( + self, + source, + footer_offset=None, + options: IpcReadOptions = ..., + memory_pool: MemoryPool | None = None, + ) -> None: ... + @property + def num_record_batches(self): ... + def count_rows(self) -> int: ... + def get_batch(self, i: int) -> RecordBatch: ... get_record_batch = get_batch - def get_batch_with_custom_metadata(self, i: int): - ... - - def read_all(self) -> Table: - ... - - def __enter__(self) -> Self: - ... - - def __exit__(self, exc_type, exc_value, traceback) -> None: - ... - + def get_batch_with_custom_metadata(self, i: int): ... + def read_all(self) -> Table: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type, exc_value, traceback) -> None: ... @property - def stats(self): - ... - + def stats(self): ... @property - def metadata(self): - ... + def metadata(self): ... -class Decimal32Array(FixedSizeBinaryArray): - ... +class Decimal32Array(FixedSizeBinaryArray): ... diff --git a/python/pyarrow-stubs/pyarrow/orc.pyi b/python/pyarrow-stubs/pyarrow/orc.pyi index fcf77076b182..77df7e448d52 100644 --- a/python/pyarrow-stubs/pyarrow/orc.pyi +++ b/python/pyarrow-stubs/pyarrow/orc.pyi @@ -17,129 +17,110 @@ from io import BytesIO from pathlib import Path -from pyarrow._fs import LocalFileSystem -from pyarrow.lib import BufferOutputStream, BufferReader, NativeFile, RecordBatch, Schema -from typing import List, Optional, Union -import os -import re +from pyarrow.lib import BufferReader, NativeFile, Schema import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from typing import Any, IO import pyarrow.fs from pyarrow.lib import __doc__ as __doc__ -import types from _typeshed import Incomplete from pyarrow.fs import _resolve_filesystem_and_path as _resolve_filesystem_and_path from pyarrow.lib import Table as Table -_orc_writer_args_docs: str -def read_table(source: str | pyarrow.NativeFile | IO[Any], columns: list | None=None, filesystem: pyarrow.fs.FileSystem | None=None) -> Table: - ... +_orc_writer_args_docs: str -def write_table(table: Table, where, *, file_version: str='0.12', batch_size: int=1024, stripe_size: int=..., compression: str='uncompressed', compression_block_size: int=65536, compression_strategy: str='speed', row_index_stride: int=10000, padding_tolerance: float=0.0, dictionary_key_size_threshold: float=0.0, bloom_filter_columns=None, bloom_filter_fpp: float=0.05) -> None: - ... +def read_table( + source: str | pyarrow.NativeFile | IO[Any], + columns: list | None = None, + filesystem: pyarrow.fs.FileSystem | None = None, +) -> Table: ... +def write_table( + table: Table, + where, + *, + file_version: str = "0.12", + batch_size: int = 1024, + stripe_size: int = ..., + compression: str = "uncompressed", + compression_block_size: int = 65536, + compression_strategy: str = "speed", + row_index_stride: int = 10000, + padding_tolerance: float = 0.0, + dictionary_key_size_threshold: float = 0.0, + bloom_filter_columns=None, + bloom_filter_fpp: float = 0.05, +) -> None: ... class ORCFile: reader: Incomplete - def __init__(self, source: Path | BufferReader | str | BytesIO | NativeFile) -> None: - ... - + def __init__( + self, source: Path | BufferReader | str | BytesIO | NativeFile + ) -> None: ... @property - def metadata(self): - ... - + def metadata(self): ... @property - def schema(self) -> Schema: - ... - + def schema(self) -> Schema: ... @property - def nrows(self) -> int: - ... - + def nrows(self) -> int: ... @property - def nstripes(self) -> int: - ... - + def nstripes(self) -> int: ... @property - def file_version(self) -> str: - ... - + def file_version(self) -> str: ... @property - def software_version(self): - ... - + def software_version(self): ... @property - def compression(self) -> str: - ... - + def compression(self) -> str: ... @property - def compression_size(self) -> int: - ... - + def compression_size(self) -> int: ... @property - def writer(self): - ... - + def writer(self): ... @property - def writer_version(self): - ... - + def writer_version(self): ... @property - def row_index_stride(self) -> int: - ... - + def row_index_stride(self) -> int: ... @property - def nstripe_statistics(self): - ... - + def nstripe_statistics(self): ... @property - def content_length(self): - ... - + def content_length(self): ... @property - def stripe_statistics_length(self): - ... - + def stripe_statistics_length(self): ... @property - def file_footer_length(self): - ... - + def file_footer_length(self): ... @property - def file_postscript_length(self): - ... - + def file_postscript_length(self): ... @property - def file_length(self): - ... - - def _select_names(self, columns: list[int] | list[str] | None=None) -> list[str]: - ... - - def read_stripe(self, n: int, columns: list | None=None) -> pyarrow.RecordBatch: - ... - - def read(self, columns: list | None=None) -> Table: - ... + def file_length(self): ... + def _select_names( + self, columns: list[int] | list[str] | None = None + ) -> list[str]: ... + def read_stripe( + self, n: int, columns: list | None = None + ) -> pyarrow.RecordBatch: ... + def read(self, columns: list | None = None) -> Table: ... class ORCWriter: __doc__: Incomplete is_open: bool writer: Incomplete - def __init__(self, where, *, file_version: str='0.12', batch_size: int=1024, stripe_size=..., compression: str='uncompressed', compression_block_size: int=65536, compression_strategy: str='speed', row_index_stride: int=10000, padding_tolerance: float=0.0, dictionary_key_size_threshold: float=0.0, bloom_filter_columns=None, bloom_filter_fpp: float=0.05) -> None: - ... - - def __del__(self) -> None: - ... - - def __enter__(self): - ... - - def __exit__(self, *args, **kwargs) -> None: - ... - - def write(self, table: Table) -> None: - ... - - def close(self) -> None: - ... + def __init__( + self, + where, + *, + file_version: str = "0.12", + batch_size: int = 1024, + stripe_size=..., + compression: str = "uncompressed", + compression_block_size: int = 65536, + compression_strategy: str = "speed", + row_index_stride: int = 10000, + padding_tolerance: float = 0.0, + dictionary_key_size_threshold: float = 0.0, + bloom_filter_columns=None, + bloom_filter_fpp: float = 0.05, + ) -> None: ... + def __del__(self) -> None: ... + def __enter__(self): ... + def __exit__(self, *args, **kwargs) -> None: ... + def write(self, table: Table) -> None: ... + def close(self) -> None: ... diff --git a/python/pyarrow-stubs/pyarrow/pandas_compat.pyi b/python/pyarrow-stubs/pyarrow/pandas_compat.pyi index b629a0496282..1b4b91c148a6 100644 --- a/python/pyarrow-stubs/pyarrow/pandas_compat.pyi +++ b/python/pyarrow-stubs/pyarrow/pandas_compat.pyi @@ -15,121 +15,705 @@ # specific language governing permissions and limitations # under the License. -from pyarrow.lib import BinaryArray, BooleanArray, ChunkedArray, DataType, Date32Array, Decimal128Array, Decimal128Type, DictionaryArray, DictionaryType, DoubleArray, DurationArray, DurationType, ExtensionArray, Field, FixedSizeBinaryType, FixedSizeListType, FloatArray, HalfFloatArray, Int16Array, Int32Array, Int64Array, Int8Array, LargeListType, LargeListViewType, LargeStringArray, ListArray, ListType, ListViewType, MapArray, MapType, NullArray, RunEndEncodedType, Schema, StringArray, StringViewArray, StructArray, StructType, Table, Time32Type, Time64Type, TimestampArray, TimestampType, UInt16Array, UInt32Array, UInt64Array, UInt8Array, UuidArray, UuidType -from pyarrow.tests.test_pandas import DummyExtensionType -from typing import Dict, List, Optional, Tuple, Type, Union -import os -import re +from pyarrow.lib import ( + BinaryArray, + BooleanArray, + ChunkedArray, + DataType, + Date32Array, + Decimal128Array, + Decimal128Type, + DictionaryArray, + DictionaryType, + DoubleArray, + DurationArray, + DurationType, + ExtensionArray, + FixedSizeBinaryType, + FixedSizeListType, + FloatArray, + HalfFloatArray, + Int16Array, + Int32Array, + Int64Array, + Int8Array, + LargeListType, + LargeListViewType, + LargeStringArray, + ListArray, + ListType, + ListViewType, + MapArray, + MapType, + NullArray, + RunEndEncodedType, + Schema, + StringArray, + StringViewArray, + StructArray, + StructType, + Table, + Time32Type, + Time64Type, + TimestampArray, + TimestampType, + UInt16Array, + UInt32Array, + UInt64Array, + UInt8Array, + UuidArray, +) import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from typing import Any from _typeshed import Incomplete -from pyarrow.lib import _pandas_api as _pandas_api, frombytes as frombytes, is_threading_enabled as is_threading_enabled +from pyarrow.lib import ( + _pandas_api as _pandas_api, + frombytes as frombytes, + is_threading_enabled as is_threading_enabled, +) + _logical_type_map: Incomplete _numpy_logical_type_map: Incomplete _pandas_logical_type_map: Incomplete -def get_logical_type_map() -> dict[int, str]: - ... - -def get_logical_type(arrow_type) -> str: - ... - -def get_numpy_logical_type_map(): - ... - -def get_logical_type_from_numpy(pandas_collection) -> str: - ... - -def get_extension_dtype_info(column) -> tuple[str, dict[str, str]] | tuple[str, dict[str, int | bool]] | tuple[str, None]: - ... - -def get_column_metadata(column, name: str, arrow_type: pyarrow.DataType, field_name: str) -> dict: - ... - -def construct_metadata(columns_to_convert, df, column_names: list[Any | str], index_levels, index_descriptors: list[dict], preserve_index: bool, types: list[pyarrow.DataType], column_field_names: list[Any | str] | None=None) -> dict: - ... - -def _get_simple_index_descriptor(level, name: str | None) -> dict[str, str | dict[str, str] | None] | dict[str, str | dict[str, str]] | dict[str, str | None]: - ... - -def _column_name_to_strings(name: str | tuple) -> str | tuple: - ... - -def _index_level_name(index, i: int, column_names: list[Any | str]) -> str: - ... - -def _get_columns_to_convert(df, schema: Schema | None, preserve_index: bool | None, columns: list[str] | None): - ... - -def _get_columns_to_convert_given_schema(df, schema: Schema, preserve_index: bool | None): - ... - -def _get_index_level(df, name: str): - ... - -def _level_name(name) -> int | str | None: - ... - -def _get_range_index_descriptor(level) -> dict[str, str | int] | dict[str, str | int | None]: - ... - -def _get_index_level_values(index): - ... - -def _resolve_columns_of_interest(df, schema: Schema | None, columns: list[str] | None): - ... +def get_logical_type_map() -> dict[int, str]: ... +def get_logical_type(arrow_type) -> str: ... +def get_numpy_logical_type_map(): ... +def get_logical_type_from_numpy(pandas_collection) -> str: ... +def get_extension_dtype_info( + column, +) -> ( + tuple[str, dict[str, str]] | tuple[str, dict[str, int | bool]] | tuple[str, None] +): ... +def get_column_metadata( + column, name: str, arrow_type: pyarrow.DataType, field_name: str +) -> dict: ... +def construct_metadata( + columns_to_convert, + df, + column_names: list[Any | str], + index_levels, + index_descriptors: list[dict], + preserve_index: bool, + types: list[pyarrow.DataType], + column_field_names: list[Any | str] | None = None, +) -> dict: ... +def _get_simple_index_descriptor( + level, name: str | None +) -> ( + dict[str, str | dict[str, str] | None] + | dict[str, str | dict[str, str]] + | dict[str, str | None] +): ... +def _column_name_to_strings(name: str | tuple) -> str | tuple: ... +def _index_level_name(index, i: int, column_names: list[Any | str]) -> str: ... +def _get_columns_to_convert( + df, schema: Schema | None, preserve_index: bool | None, columns: list[str] | None +): ... +def _get_columns_to_convert_given_schema( + df, schema: Schema, preserve_index: bool | None +): ... +def _get_index_level(df, name: str): ... +def _level_name(name) -> int | str | None: ... +def _get_range_index_descriptor( + level, +) -> dict[str, str | int] | dict[str, str | int | None]: ... +def _get_index_level_values(index): ... +def _resolve_columns_of_interest( + df, schema: Schema | None, columns: list[str] | None +): ... +def dataframe_to_types(df, preserve_index: bool | None, columns=None): ... +def dataframe_to_arrays( + df, + schema: Schema | None, + preserve_index: bool | None, + nthreads: int = 1, + columns: list[str] | None = None, + safe: bool = True, +) -> ( + tuple[list[MapArray | StringArray], Schema, None] + | tuple[ + list[Int64Array | FloatArray | DoubleArray | BooleanArray | LargeStringArray], + Schema, + None, + ] + | tuple[list[LargeStringArray | Int64Array | TimestampArray], Schema, None] + | tuple[list[DoubleArray], Schema, None] + | tuple[list[DoubleArray | LargeStringArray], Schema, None] + | tuple[ + list[ + UInt8Array + | UInt16Array + | UInt32Array + | UInt64Array + | Int16Array + | Int32Array + | Int64Array + | HalfFloatArray + | FloatArray + | DoubleArray + | BooleanArray + | TimestampArray + | DurationArray + | LargeStringArray + | NullArray + | ListArray + ], + Schema, + None, + ] + | tuple[ + list[ + TimestampArray + | DictionaryArray + | LargeStringArray + | BooleanArray + | DoubleArray + | Int64Array + | ExtensionArray + ], + Schema, + None, + ] + | tuple[list[Date32Array], Schema, None] + | tuple[list[StringArray | Int64Array | DoubleArray], Schema, None] + | tuple[list[UInt32Array], Schema, None] + | tuple[list[Int16Array], Schema, None] + | tuple[list[Int64Array | LargeStringArray], Schema, None] + | tuple[list[Date32Array | Int64Array | DoubleArray | StringArray], Schema, None] + | tuple[list[Any], Schema, None] + | tuple[list[ExtensionArray], Schema, None] + | tuple[list[ListArray], Schema, None] + | tuple[list[StructArray], Schema, None] + | tuple[ + list[ + UInt8Array + | UInt32Array + | Int16Array + | Int32Array + | HalfFloatArray + | DoubleArray + | TimestampArray + | LargeStringArray + | ListArray + ], + Schema, + None, + ] + | tuple[list[Int64Array | DoubleArray], Schema, None] + | tuple[list[TimestampArray], Schema, None] + | tuple[ + list[Int64Array | UInt32Array | DoubleArray | LargeStringArray | BooleanArray], + Schema, + None, + ] + | tuple[ + list[ + UInt8Array + | UInt16Array + | UInt32Array + | UInt64Array + | Int8Array + | Int16Array + | Int32Array + | Int64Array + | FloatArray + | DoubleArray + | BooleanArray + ], + Schema, + None, + ] + | tuple[list[StructArray | LargeStringArray], Schema, None] + | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array], Schema, None] + | tuple[list[DurationArray], Schema, None] + | tuple[ + list[ + UInt8Array + | UInt16Array + | UInt32Array + | UInt64Array + | Int16Array + | Int32Array + | Int64Array + | FloatArray + | DoubleArray + | BooleanArray + | TimestampArray + | DurationArray + | LargeStringArray + ], + Schema, + None, + ] + | tuple[list[LargeStringArray | ListArray], Schema, None] + | tuple[list[UInt64Array], Schema, None] + | tuple[list[DoubleArray | ListArray | Int32Array], Schema, None] + | tuple[list[LargeStringArray | Decimal128Array], Schema, None] + | tuple[list[StringViewArray], Schema, None] + | tuple[list[Int64Array | StructArray], Schema, None] + | tuple[list[DoubleArray | LargeStringArray | Int64Array], Schema, None] + | tuple[list[UInt8Array], Schema, None] + | tuple[ + list[ + UInt8Array + | UInt16Array + | UInt32Array + | UInt64Array + | Int8Array + | Int16Array + | Int32Array + | Int64Array + | FloatArray + | DoubleArray + | BooleanArray + | LargeStringArray + | NullArray + ], + Schema, + None, + ] + | tuple[ + list[UInt32Array | DoubleArray | BooleanArray | LargeStringArray], Schema, None + ] + | tuple[list[FloatArray | DoubleArray], Schema, None] + | tuple[list[Int16Array | FloatArray | StringArray], Schema, None] + | tuple[ + list[ + UInt8Array + | UInt16Array + | Int64Array + | UInt64Array + | Int8Array + | Int16Array + | Int32Array + | FloatArray + | DoubleArray + | BooleanArray + | LargeStringArray + | NullArray + ], + Schema, + None, + ] + | tuple[list[Int64Array | BooleanArray], Schema, None] + | tuple[ + list[ + LargeStringArray + | Int64Array + | UInt8Array + | DoubleArray + | BooleanArray + | DictionaryArray + | TimestampArray + ], + Schema, + None, + ] + | tuple[list[Int64Array | ListArray | DoubleArray], Schema, None] + | tuple[ + list[ + UInt8Array + | UInt16Array + | UInt32Array + | UInt64Array + | Int16Array + | Int32Array + | Int64Array + | FloatArray + | DoubleArray + | BooleanArray + ], + Schema, + None, + ] + | tuple[list[LargeStringArray], Schema, None] + | tuple[list[NullArray], Schema, None] + | tuple[list[DictionaryArray], Schema, None] + | tuple[list[Int32Array | StringArray], Schema, None] + | tuple[ + list[DictionaryArray | Int64Array | LargeStringArray | BinaryArray], + Schema, + None, + ] + | tuple[list[FloatArray | TimestampArray], Schema, None] + | tuple[list[StringArray | Int64Array | Int32Array | TimestampArray], Schema, None] + | tuple[list[BooleanArray], Schema, None] + | tuple[ + list[LargeStringArray | Int64Array | DoubleArray | TimestampArray], Schema, None + ] + | tuple[list[Int64Array | ListArray], Schema, None] + | tuple[list[DictionaryArray | Int64Array], Schema, None] + | tuple[ + list[ + UInt8Array + | UInt16Array + | UInt32Array + | UInt64Array + | Int16Array + | Int32Array + | Int64Array + | FloatArray + | DoubleArray + | BooleanArray + | LargeStringArray + ], + Schema, + None, + ] + | tuple[list[StringArray], Schema, None] + | tuple[list[Any], Schema, int] + | tuple[list[Int8Array], Schema, None] + | tuple[list[Decimal128Array], Schema, None] + | tuple[list[NullArray | DoubleArray], Schema, None] + | tuple[list[UInt16Array], Schema, None] + | tuple[list[UuidArray], Schema, None] + | tuple[list[BinaryArray], Schema, None] + | tuple[list[LargeStringArray | DictionaryArray], Schema, None] + | tuple[ + list[ + UInt8Array + | UInt16Array + | UInt32Array + | UInt64Array + | Int16Array + | Int32Array + | Int64Array + | HalfFloatArray + | FloatArray + | DoubleArray + | BooleanArray + | TimestampArray + | DurationArray + | LargeStringArray + | NullArray + | ListArray + | DictionaryArray + ], + Schema, + None, + ] + | tuple[list[Int32Array], Schema, None] + | tuple[list[DoubleArray | DictionaryArray], Schema, None] + | tuple[list[StringArray | FloatArray], Schema, None] + | tuple[ + list[ + Int8Array + | Int16Array + | Int32Array + | Int64Array + | UInt8Array + | UInt16Array + | UInt32Array + | UInt64Array + ], + Schema, + None, + ] + | tuple[list[FloatArray], Schema, None] + | tuple[list[Int64Array], Schema, None] + | tuple[list[ChunkedArray | StringArray], Schema, None] + | tuple[list[TimestampArray | Int64Array], Schema, None] +): ... +def get_datetimetz_type( + values, + dtype, + type_: Decimal128Type + | LargeListViewType + | StructType + | LargeListType + | MapType + | RunEndEncodedType + | DictionaryType + | ListType + | TimestampType + | FixedSizeListType + | DataType + | FixedSizeBinaryType + | DurationType + | Time64Type + | Time32Type + | ListViewType + | None, +): ... +def _reconstruct_block( + item: dict, + columns: list[str] | None = None, + extension_columns: dict | None = None, + return_block: bool = True, +): ... +def make_datetimetz(unit: str, tz: str): ... +def table_to_dataframe( + options: dict[str, bool | None], + table: Table, + categories=None, + ignore_metadata: bool = False, + types_mapper=None, +): ... -def dataframe_to_types(df, preserve_index: bool | None, columns=None): - ... - -def dataframe_to_arrays(df, schema: Schema | None, preserve_index: bool | None, nthreads: int=1, columns: list[str] | None=None, safe: bool=True) -> tuple[list[MapArray | StringArray], Schema, None] | tuple[list[Int64Array | FloatArray | DoubleArray | BooleanArray | LargeStringArray], Schema, None] | tuple[list[LargeStringArray | Int64Array | TimestampArray], Schema, None] | tuple[list[DoubleArray], Schema, None] | tuple[list[DoubleArray | LargeStringArray], Schema, None] | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array | Int16Array | Int32Array | Int64Array | HalfFloatArray | FloatArray | DoubleArray | BooleanArray | TimestampArray | DurationArray | LargeStringArray | NullArray | ListArray], Schema, None] | tuple[list[TimestampArray | DictionaryArray | LargeStringArray | BooleanArray | DoubleArray | Int64Array | ExtensionArray], Schema, None] | tuple[list[Date32Array], Schema, None] | tuple[list[StringArray | Int64Array | DoubleArray], Schema, None] | tuple[list[UInt32Array], Schema, None] | tuple[list[Int16Array], Schema, None] | tuple[list[Int64Array | LargeStringArray], Schema, None] | tuple[list[Date32Array | Int64Array | DoubleArray | StringArray], Schema, None] | tuple[list[Any], Schema, None] | tuple[list[ExtensionArray], Schema, None] | tuple[list[ListArray], Schema, None] | tuple[list[StructArray], Schema, None] | tuple[list[UInt8Array | UInt32Array | Int16Array | Int32Array | HalfFloatArray | DoubleArray | TimestampArray | LargeStringArray | ListArray], Schema, None] | tuple[list[Int64Array | DoubleArray], Schema, None] | tuple[list[TimestampArray], Schema, None] | tuple[list[Int64Array | UInt32Array | DoubleArray | LargeStringArray | BooleanArray], Schema, None] | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array | Int8Array | Int16Array | Int32Array | Int64Array | FloatArray | DoubleArray | BooleanArray], Schema, None] | tuple[list[StructArray | LargeStringArray], Schema, None] | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array], Schema, None] | tuple[list[DurationArray], Schema, None] | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array | Int16Array | Int32Array | Int64Array | FloatArray | DoubleArray | BooleanArray | TimestampArray | DurationArray | LargeStringArray], Schema, None] | tuple[list[LargeStringArray | ListArray], Schema, None] | tuple[list[UInt64Array], Schema, None] | tuple[list[DoubleArray | ListArray | Int32Array], Schema, None] | tuple[list[LargeStringArray | Decimal128Array], Schema, None] | tuple[list[StringViewArray], Schema, None] | tuple[list[Int64Array | StructArray], Schema, None] | tuple[list[DoubleArray | LargeStringArray | Int64Array], Schema, None] | tuple[list[UInt8Array], Schema, None] | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array | Int8Array | Int16Array | Int32Array | Int64Array | FloatArray | DoubleArray | BooleanArray | LargeStringArray | NullArray], Schema, None] | tuple[list[UInt32Array | DoubleArray | BooleanArray | LargeStringArray], Schema, None] | tuple[list[FloatArray | DoubleArray], Schema, None] | tuple[list[Int16Array | FloatArray | StringArray], Schema, None] | tuple[list[UInt8Array | UInt16Array | Int64Array | UInt64Array | Int8Array | Int16Array | Int32Array | FloatArray | DoubleArray | BooleanArray | LargeStringArray | NullArray], Schema, None] | tuple[list[Int64Array | BooleanArray], Schema, None] | tuple[list[LargeStringArray | Int64Array | UInt8Array | DoubleArray | BooleanArray | DictionaryArray | TimestampArray], Schema, None] | tuple[list[Int64Array | ListArray | DoubleArray], Schema, None] | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array | Int16Array | Int32Array | Int64Array | FloatArray | DoubleArray | BooleanArray], Schema, None] | tuple[list[LargeStringArray], Schema, None] | tuple[list[NullArray], Schema, None] | tuple[list[DictionaryArray], Schema, None] | tuple[list[Int32Array | StringArray], Schema, None] | tuple[list[DictionaryArray | Int64Array | LargeStringArray | BinaryArray], Schema, None] | tuple[list[FloatArray | TimestampArray], Schema, None] | tuple[list[StringArray | Int64Array | Int32Array | TimestampArray], Schema, None] | tuple[list[BooleanArray], Schema, None] | tuple[list[LargeStringArray | Int64Array | DoubleArray | TimestampArray], Schema, None] | tuple[list[Int64Array | ListArray], Schema, None] | tuple[list[DictionaryArray | Int64Array], Schema, None] | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array | Int16Array | Int32Array | Int64Array | FloatArray | DoubleArray | BooleanArray | LargeStringArray], Schema, None] | tuple[list[StringArray], Schema, None] | tuple[list[Any], Schema, int] | tuple[list[Int8Array], Schema, None] | tuple[list[Decimal128Array], Schema, None] | tuple[list[NullArray | DoubleArray], Schema, None] | tuple[list[UInt16Array], Schema, None] | tuple[list[UuidArray], Schema, None] | tuple[list[BinaryArray], Schema, None] | tuple[list[LargeStringArray | DictionaryArray], Schema, None] | tuple[list[UInt8Array | UInt16Array | UInt32Array | UInt64Array | Int16Array | Int32Array | Int64Array | HalfFloatArray | FloatArray | DoubleArray | BooleanArray | TimestampArray | DurationArray | LargeStringArray | NullArray | ListArray | DictionaryArray], Schema, None] | tuple[list[Int32Array], Schema, None] | tuple[list[DoubleArray | DictionaryArray], Schema, None] | tuple[list[StringArray | FloatArray], Schema, None] | tuple[list[Int8Array | Int16Array | Int32Array | Int64Array | UInt8Array | UInt16Array | UInt32Array | UInt64Array], Schema, None] | tuple[list[FloatArray], Schema, None] | tuple[list[Int64Array], Schema, None] | tuple[list[ChunkedArray | StringArray], Schema, None] | tuple[list[TimestampArray | Int64Array], Schema, None]: - ... - -def get_datetimetz_type(values, dtype, type_: Decimal128Type | LargeListViewType | StructType | LargeListType | MapType | RunEndEncodedType | DictionaryType | ListType | TimestampType | FixedSizeListType | DataType | FixedSizeBinaryType | DurationType | Time64Type | Time32Type | ListViewType | None): - ... - -def _reconstruct_block(item: dict, columns: list[str] | None=None, extension_columns: dict | None=None, return_block: bool=True): - ... - -def make_datetimetz(unit: str, tz: str): - ... - -def table_to_dataframe(options: dict[str, bool | None], table: Table, categories=None, ignore_metadata: bool=False, types_mapper=None): - ... _pandas_supported_numpy_types: Incomplete -def _get_extension_dtypes(table: Table, columns_metadata: list[dict[str, str | dict[str, int]] | dict[str, str | None] | dict[str, str | dict[str, int | bool]] | Any | dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None]], types_mapper, options: dict[str, bool | None], categories): - ... - -def _check_data_column_metadata_consistency(all_columns: list[dict[str, str | dict[str, int]] | dict[str, str | None] | dict[str, str | dict[str, int | bool]] | Any | dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None]]) -> None: - ... - -def _deserialize_column_index(block_table: Table, all_columns: list[dict[str, str | dict[str, int]] | dict[str, str | None] | dict[str, str | dict[str, int | bool]] | Any | dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None]], column_indexes: list[dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None] | dict[str, str | None] | Any]): - ... - -def _reconstruct_index(table: Table, index_descriptors: list[Any | dict[str, str | int] | dict[str, str | int | None] | str], all_columns: list[dict[str, str | dict[str, int]] | dict[str, str | None] | dict[str, str | dict[str, int | bool]] | Any | dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None]], types_mapper=None): - ... - -def _extract_index_level(table: Table, result_table: Table, field_name: str, field_name_to_metadata: dict[str, dict[str, str | None] | dict[str, str | dict[str, int | bool]]] | dict[str, dict[str, str | None] | dict[str, str | dict[str, str]]] | dict[str, dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None]] | dict[str, dict[str, str | None]], types_mapper=None): - ... - -def _backwards_compatible_index_name(raw_name: str, logical_name: str) -> str: - ... - -def _is_generated_index_name(name: str) -> bool: - ... - -def get_pandas_logical_type_map(): - ... - -def _pandas_type_to_numpy_type(pandas_type: str): - ... - -def _reconstruct_columns_from_metadata(columns, column_indexes: list[dict[str, str | dict[str, str]] | dict[str, str | None] | dict[str, str | dict[str, str] | None]]): - ... - -def _add_any_metadata(table: Table, pandas_metadata: dict[str, list[str] | str | list[dict[str, str | None] | dict[str, str | dict[str, str]]]] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | None] | dict[str, str | dict[str, str]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | dict[str, int]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[Any] | list[dict[str, str | None] | dict[str, str | dict[str, int | bool]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[Any] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[str] | list[dict[str, str | None]] | str] | dict[str, list[str] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | None]] | list[dict[str, str | int]] | str] | dict[str, list[str] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | None]] | str] | dict[str, list[Any] | list[dict[str, str | dict[str, int | bool]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | dict[str, str]]] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[str] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | None] | dict[str, str | dict[str, int | bool]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[Any] | list[dict[str, str | None] | dict[str, str | dict[str, int]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[str] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[str] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | dict[str, str]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | None] | dict[str, str | dict[str, int | bool]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[Any] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | None] | dict[str, str | dict[str, str]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | dict[str, str] | None]] | list[dict[str, str | dict[str, int | bool]]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[dict[str, str | int | None]] | list[dict[str, str | None]] | list[Any] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[Any] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str] | dict[str, list[Any] | list[dict[str, str | dict[str, str]]] | dict[Any, Any] | dict[str, str] | str]) -> Table: - ... - -def make_tz_aware(series, tz: str): - ... +def _get_extension_dtypes( + table: Table, + columns_metadata: list[ + dict[str, str | dict[str, int]] + | dict[str, str | None] + | dict[str, str | dict[str, int | bool]] + | Any + | dict[str, str | dict[str, str]] + | dict[str, str | dict[str, str] | None] + ], + types_mapper, + options: dict[str, bool | None], + categories, +): ... +def _check_data_column_metadata_consistency( + all_columns: list[ + dict[str, str | dict[str, int]] + | dict[str, str | None] + | dict[str, str | dict[str, int | bool]] + | Any + | dict[str, str | dict[str, str]] + | dict[str, str | dict[str, str] | None] + ], +) -> None: ... +def _deserialize_column_index( + block_table: Table, + all_columns: list[ + dict[str, str | dict[str, int]] + | dict[str, str | None] + | dict[str, str | dict[str, int | bool]] + | Any + | dict[str, str | dict[str, str]] + | dict[str, str | dict[str, str] | None] + ], + column_indexes: list[ + dict[str, str | dict[str, str]] + | dict[str, str | dict[str, str] | None] + | dict[str, str | None] + | Any + ], +): ... +def _reconstruct_index( + table: Table, + index_descriptors: list[ + Any | dict[str, str | int] | dict[str, str | int | None] | str + ], + all_columns: list[ + dict[str, str | dict[str, int]] + | dict[str, str | None] + | dict[str, str | dict[str, int | bool]] + | Any + | dict[str, str | dict[str, str]] + | dict[str, str | dict[str, str] | None] + ], + types_mapper=None, +): ... +def _extract_index_level( + table: Table, + result_table: Table, + field_name: str, + field_name_to_metadata: dict[ + str, dict[str, str | None] | dict[str, str | dict[str, int | bool]] + ] + | dict[str, dict[str, str | None] | dict[str, str | dict[str, str]]] + | dict[ + str, dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None] + ] + | dict[str, dict[str, str | None]], + types_mapper=None, +): ... +def _backwards_compatible_index_name(raw_name: str, logical_name: str) -> str: ... +def _is_generated_index_name(name: str) -> bool: ... +def get_pandas_logical_type_map(): ... +def _pandas_type_to_numpy_type(pandas_type: str): ... +def _reconstruct_columns_from_metadata( + columns, + column_indexes: list[ + dict[str, str | dict[str, str]] + | dict[str, str | None] + | dict[str, str | dict[str, str] | None] + ], +): ... +def _add_any_metadata( + table: Table, + pandas_metadata: dict[ + str, + list[str] | str | list[dict[str, str | None] | dict[str, str | dict[str, str]]], + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | None] | dict[str, str | dict[str, str]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | dict[str, int]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[Any] + | list[dict[str, str | None] | dict[str, str | dict[str, int | bool]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[Any] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | None]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[str, list[str] | list[dict[str, str | None]] | str] + | dict[ + str, + list[str] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | None]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[str, list[dict[str, str | None]] | list[dict[str, str | int]] | str] + | dict[ + str, + list[str] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | None]] + | str, + ] + | dict[ + str, + list[Any] + | list[dict[str, str | dict[str, int | bool]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | dict[str, str]]] + | list[dict[str, str | None]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[str] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | None] | dict[str, str | dict[str, int | bool]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[Any] + | list[dict[str, str | None] | dict[str, str | dict[str, int]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | None]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[str] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | dict[str, str]] | dict[str, str | dict[str, str] | None]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[str] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str, + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | dict[str, str]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | None]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | None] | dict[str, str | dict[str, int | bool]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[Any] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | None] | dict[str, str | dict[str, str]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | dict[str, str] | None]] + | list[dict[str, str | dict[str, int | bool]]] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[dict[str, str | int | None]] + | list[dict[str, str | None]] + | list[Any] + | dict[Any, Any] + | dict[str, str] + | str, + ] + | dict[ + str, + list[Any] | list[dict[str, str | None]] | dict[Any, Any] | dict[str, str] | str, + ] + | dict[ + str, + list[Any] + | list[dict[str, str | dict[str, str]]] + | dict[Any, Any] + | dict[str, str] + | str, + ], +) -> Table: ... +def make_tz_aware(series, tz: str): ... diff --git a/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi b/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi index a6655e8ae5de..14d5874f5c46 100644 --- a/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi +++ b/python/pyarrow-stubs/pyarrow/parquet/__init__.pyi @@ -15,9 +15,4 @@ # specific language governing permissions and limitations # under the License. -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete from .core import * diff --git a/python/pyarrow-stubs/pyarrow/parquet/core.pyi b/python/pyarrow-stubs/pyarrow/parquet/core.pyi index c8a9d7f8f817..d86961927fc8 100644 --- a/python/pyarrow-stubs/pyarrow/parquet/core.pyi +++ b/python/pyarrow-stubs/pyarrow/parquet/core.pyi @@ -15,51 +15,163 @@ # specific language governing permissions and limitations # under the License. -from io import BufferedReader, BufferedWriter, BytesIO -from datetime import datetime from pathlib import Path -from pyarrow._compute import Expression from pyarrow._fs import LocalFileSystem as LocalFileSystem, SubTreeFileSystem -from pyarrow.lib import Buffer, BufferOutputStream, BufferReader, NativeFile, OSFile, PythonFile, RecordBatch, Schema, Table -from pyarrow.tests.util import FSProtocolClass -from typing import DefaultDict, Dict, List, Optional, Set, Tuple, Union -import os -import re +from pyarrow.lib import RecordBatch, Schema, Table import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence +from typing import Any, IO, Literal +from collections.abc import Callable, Sequence import pyarrow.compute import pyarrow.dataset import pyarrow.fs from pyarrow.lib import __doc__ as __doc__ -import types from _typeshed import Incomplete -from pyarrow._parquet import ColumnChunkMetaData as ColumnChunkMetaData, ColumnSchema as ColumnSchema, FileDecryptionProperties as FileDecryptionProperties, FileEncryptionProperties as FileEncryptionProperties, FileMetaData as FileMetaData, ParquetLogicalType as ParquetLogicalType, ParquetReader as ParquetReader, ParquetSchema as ParquetSchema, RowGroupMetaData as RowGroupMetaData, SortingColumn as SortingColumn, Statistics as Statistics -__all__ = ['ParquetReader', 'Statistics', 'FileMetaData', 'RowGroupMetaData', 'ColumnChunkMetaData', 'ParquetSchema', 'ColumnSchema', 'ParquetLogicalType', 'FileEncryptionProperties', 'FileDecryptionProperties', 'SortingColumn', 'filters_to_expression', '_filters_to_expression', 'ParquetFile', 'ParquetWriter', 'ParquetDataset', 'read_table', 'read_pandas', 'write_table', 'write_to_dataset', 'write_metadata', 'read_metadata', 'read_schema'] +from pyarrow._parquet import ( + ColumnChunkMetaData as ColumnChunkMetaData, + ColumnSchema as ColumnSchema, + FileDecryptionProperties as FileDecryptionProperties, + FileEncryptionProperties as FileEncryptionProperties, + FileMetaData as FileMetaData, + ParquetLogicalType as ParquetLogicalType, + ParquetReader as ParquetReader, + ParquetSchema as ParquetSchema, + RowGroupMetaData as RowGroupMetaData, + SortingColumn as SortingColumn, + Statistics as Statistics, +) +from collections import defaultdict + +__all__ = [ + "ParquetReader", + "Statistics", + "FileMetaData", + "RowGroupMetaData", + "ColumnChunkMetaData", + "ParquetSchema", + "ColumnSchema", + "ParquetLogicalType", + "FileEncryptionProperties", + "FileDecryptionProperties", + "SortingColumn", + "filters_to_expression", + "_filters_to_expression", + "ParquetFile", + "ParquetWriter", + "ParquetDataset", + "read_table", + "read_pandas", + "write_table", + "write_to_dataset", + "write_metadata", + "read_metadata", + "read_schema", +] + +def filters_to_expression( + filters: list[tuple] | list[list[tuple]], +) -> pyarrow.compute.Expression: ... -def filters_to_expression(filters: list[tuple] | list[list[tuple]]) -> pyarrow.compute.Expression: - ... _filters_to_expression: Incomplete -def read_table(source: str | list[str] | pyarrow.NativeFile | IO[Any], *, columns: list | None=None, use_threads: bool=True, schema: pyarrow.Schema | None=None, use_pandas_metadata: bool=False, read_dictionary: list | None=None, binary_type: pyarrow.DataType | None=None, list_type=None, memory_map: bool=False, buffer_size: int=0, partitioning: str='hive', filesystem: pyarrow.fs.FileSystem | None=None, filters: pyarrow.compute.Expression | list[tuple] | list[list[tuple]] | None=None, ignore_prefixes: list | None=None, pre_buffer: bool=True, coerce_int96_timestamp_unit: str | None=None, decryption_properties: FileDecryptionProperties | None=None, thrift_string_size_limit: int | None=None, thrift_container_size_limit: int | None=None, schema_depth_limit: int | None=None, page_checksum_verification: bool=False, arrow_extensions_enabled: bool=True) -> pyarrow.Table: - ... +def read_table( + source: str | list[str] | pyarrow.NativeFile | IO[Any], + *, + columns: list | None = None, + use_threads: bool = True, + schema: pyarrow.Schema | None = None, + use_pandas_metadata: bool = False, + read_dictionary: list | None = None, + binary_type: pyarrow.DataType | None = None, + list_type=None, + memory_map: bool = False, + buffer_size: int = 0, + partitioning: str = "hive", + filesystem: pyarrow.fs.FileSystem | None = None, + filters: pyarrow.compute.Expression | list[tuple] | list[list[tuple]] | None = None, + ignore_prefixes: list | None = None, + pre_buffer: bool = True, + coerce_int96_timestamp_unit: str | None = None, + decryption_properties: FileDecryptionProperties | None = None, + thrift_string_size_limit: int | None = None, + thrift_container_size_limit: int | None = None, + schema_depth_limit: int | None = None, + page_checksum_verification: bool = False, + arrow_extensions_enabled: bool = True, +) -> pyarrow.Table: ... +def read_pandas( + source: str | list[str] | pyarrow.NativeFile | IO[Any], + columns: list | None = None, + **kwargs, +) -> pyarrow.Table: ... +def write_table( + table: pyarrow.Table, + where: str | pyarrow.NativeFile, + row_group_size: int | None = None, + version: str = "2.6", + use_dictionary: bool = True, + compression: str = "snappy", + write_statistics: bool = True, + use_deprecated_int96_timestamps: bool | None = None, + coerce_timestamps: str | None = None, + allow_truncated_timestamps: bool = False, + data_page_size: int | None = None, + flavor: Literal["spark"] | None = None, + filesystem: pyarrow.fs.FileSystem | None = None, + compression_level: int | dict | None = None, + use_byte_stream_split: bool = False, + column_encoding: str | dict | None = None, + data_page_version: str = "1.0", + use_compliant_nested_type: bool = True, + encryption_properties: FileEncryptionProperties | None = None, + write_batch_size: int | None = None, + dictionary_pagesize_limit: int | None = None, + store_schema: bool = True, + write_page_index: bool = False, + write_page_checksum: bool = False, + sorting_columns: Sequence[SortingColumn] | None = None, + store_decimal_as_integer: bool = False, + write_time_adjusted_to_utc: bool = False, + max_rows_per_page: int | None = None, + bloom_filter_options: dict | None = None, + use_content_defined_chunking: bool = False, + **kwargs, +) -> None: ... +def write_to_dataset( + table: pyarrow.Table, + root_path: Path | str, + partition_cols: list | None = None, + filesystem: pyarrow.fs.FileSystem | None = None, + schema: pyarrow.Schema | None = None, + partitioning: pyarrow.dataset.Partitioning | list[str] | None = None, + basename_template: str | None = None, + use_threads: bool | None = None, + file_visitor: Callable[..., Any] | None = None, + existing_data_behavior: Literal["overwrite_or_ignore", "error", "delete_matching"] + | None = None, + **kwargs, +) -> None: ... +def write_metadata( + schema: pyarrow.Schema, + where: str | pyarrow.NativeFile, + metadata_collector: list | None = None, + filesystem: pyarrow.fs.FileSystem | None = None, + **kwargs, +) -> None: ... +def read_metadata( + where: str | IO[Any], + memory_map: bool = False, + decryption_properties: FileDecryptionProperties | None = None, + filesystem: pyarrow.fs.FileSystem | None = None, + arrow_extensions_enabled: bool = True, +) -> FileMetaData: ... +def read_schema( + where: str | IO[Any], + memory_map: bool = False, + decryption_properties: FileDecryptionProperties | None = None, + filesystem: pyarrow.fs.FileSystem | None = None, + arrow_extensions_enabled: bool = True, +) -> pyarrow.Schema: ... -def read_pandas(source: str | list[str] | pyarrow.NativeFile | IO[Any], columns: list | None=None, **kwargs) -> pyarrow.Table: - ... - -def write_table(table: pyarrow.Table, where: str | pyarrow.NativeFile, row_group_size: int | None=None, version: str='2.6', use_dictionary: bool=True, compression: str='snappy', write_statistics: bool=True, use_deprecated_int96_timestamps: bool | None=None, coerce_timestamps: str | None=None, allow_truncated_timestamps: bool=False, data_page_size: int | None=None, flavor: Literal['spark'] | None=None, filesystem: pyarrow.fs.FileSystem | None=None, compression_level: int | dict | None=None, use_byte_stream_split: bool=False, column_encoding: str | dict | None=None, data_page_version: str='1.0', use_compliant_nested_type: bool=True, encryption_properties: FileEncryptionProperties | None=None, write_batch_size: int | None=None, dictionary_pagesize_limit: int | None=None, store_schema: bool=True, write_page_index: bool=False, write_page_checksum: bool=False, sorting_columns: Sequence[SortingColumn] | None=None, store_decimal_as_integer: bool=False, write_time_adjusted_to_utc: bool=False, max_rows_per_page: int | None=None, bloom_filter_options: dict | None=None, use_content_defined_chunking: bool=False, **kwargs) -> None: - ... - -def write_to_dataset(table: pyarrow.Table, root_path: Path | str, partition_cols: list | None=None, filesystem: pyarrow.fs.FileSystem | None=None, schema: pyarrow.Schema | None=None, partitioning: pyarrow.dataset.Partitioning | list[str] | None=None, basename_template: str | None=None, use_threads: bool | None=None, file_visitor: Callable[..., Any] | None=None, existing_data_behavior: Literal['overwrite_or_ignore'] | Literal['error'] | Literal['delete_matching'] | None=None, **kwargs) -> None: - ... - -def write_metadata(schema: pyarrow.Schema, where: str | pyarrow.NativeFile, metadata_collector: list | None=None, filesystem: pyarrow.fs.FileSystem | None=None, **kwargs) -> None: - ... - -def read_metadata(where: str | IO[Any], memory_map: bool=False, decryption_properties: FileDecryptionProperties | None=None, filesystem: pyarrow.fs.FileSystem | None=None, arrow_extensions_enabled: bool=True) -> FileMetaData: - ... - -def read_schema(where: str | IO[Any], memory_map: bool=False, decryption_properties: FileDecryptionProperties | None=None, filesystem: pyarrow.fs.FileSystem | None=None, arrow_extensions_enabled: bool=True) -> pyarrow.Schema: - ... EXCLUDED_PARQUET_PATHS: set[str] class ParquetFile: @@ -68,58 +180,77 @@ class ParquetFile: common_metadata: Incomplete _nested_paths_by_prefix: Incomplete - def __init__(self, source, *, metadata=None, common_metadata=None, read_dictionary=None, binary_type=None, list_type=None, memory_map: bool=False, buffer_size: int=0, pre_buffer: bool=True, coerce_int96_timestamp_unit=None, decryption_properties=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, filesystem=None, page_checksum_verification: bool=False, arrow_extensions_enabled: bool=True) -> None: - ... - - def __enter__(self): - ... - - def __exit__(self, *args, **kwargs) -> None: - ... - - def _build_nested_paths(self) -> DefaultDict[str, list[int]] | DefaultDict[Any, Any]: - ... - + def __init__( + self, + source, + *, + metadata=None, + common_metadata=None, + read_dictionary=None, + binary_type=None, + list_type=None, + memory_map: bool = False, + buffer_size: int = 0, + pre_buffer: bool = True, + coerce_int96_timestamp_unit=None, + decryption_properties=None, + thrift_string_size_limit=None, + thrift_container_size_limit=None, + schema_depth_limit=None, + filesystem=None, + page_checksum_verification: bool = False, + arrow_extensions_enabled: bool = True, + ) -> None: ... + def __enter__(self): ... + def __exit__(self, *args, **kwargs) -> None: ... + def _build_nested_paths( + self, + ) -> defaultdict[str, list[int]] | defaultdict[Any, Any]: ... @property - def metadata(self) -> FileMetaData: - ... - + def metadata(self) -> FileMetaData: ... @property - def schema(self) -> ParquetSchema: - ... - + def schema(self) -> ParquetSchema: ... @property - def schema_arrow(self) -> Schema: - ... - + def schema_arrow(self) -> Schema: ... @property - def num_row_groups(self) -> int: - ... - - def close(self, force: bool=False): - ... - + def num_row_groups(self) -> int: ... + def close(self, force: bool = False): ... @property - def closed(self) -> bool: - ... - - def read_row_group(self, i: int, columns: list | None=None, use_threads: bool=True, use_pandas_metadata: bool=False) -> Table: - ... - - def read_row_groups(self, row_groups: list, columns: list | None=None, use_threads: bool=True, use_pandas_metadata: bool=False) -> Table: - ... - - def iter_batches(self, batch_size: int=65536, row_groups: list | None=None, columns: list | None=None, use_threads: bool=True, use_pandas_metadata: bool=False): - ... - - def read(self, columns: list | None=None, use_threads: bool=True, use_pandas_metadata: bool=False) -> Table: - ... - - def scan_contents(self, columns: list[int] | None=None, batch_size: int=65536) -> int: - ... - - def _get_column_indices(self, column_names, use_pandas_metadata: bool=False) -> list[int]: - ... + def closed(self) -> bool: ... + def read_row_group( + self, + i: int, + columns: list | None = None, + use_threads: bool = True, + use_pandas_metadata: bool = False, + ) -> Table: ... + def read_row_groups( + self, + row_groups: list, + columns: list | None = None, + use_threads: bool = True, + use_pandas_metadata: bool = False, + ) -> Table: ... + def iter_batches( + self, + batch_size: int = 65536, + row_groups: list | None = None, + columns: list | None = None, + use_threads: bool = True, + use_pandas_metadata: bool = False, + ): ... + def read( + self, + columns: list | None = None, + use_threads: bool = True, + use_pandas_metadata: bool = False, + ) -> Table: ... + def scan_contents( + self, columns: list[int] | None = None, batch_size: int = 65536 + ) -> int: ... + def _get_column_indices( + self, column_names, use_pandas_metadata: bool = False + ) -> list[int]: ... class ParquetWriter: __doc__: Incomplete @@ -132,32 +263,59 @@ class ParquetWriter: writer: Incomplete is_open: bool - def __init__(self, where, schema: Schema | None, filesystem: LocalFileSystem | None=None, flavor: str | None=None, version: str='2.6', use_dictionary: bool=True, compression: str='snappy', write_statistics: bool=True, use_deprecated_int96_timestamps: bool | None=None, compression_level: int | dict[str, int] | None=None, use_byte_stream_split: bool=False, column_encoding: dict[str, str] | bool | str | None=None, writer_engine_version=None, data_page_version: str='1.0', use_compliant_nested_type: bool=True, encryption_properties: FileEncryptionProperties | None=None, write_batch_size: int | None=None, dictionary_pagesize_limit: int | str | None=None, store_schema: bool=True, write_page_index: bool=False, write_page_checksum: bool=False, sorting_columns: tuple[SortingColumn, SortingColumn] | None=None, store_decimal_as_integer: bool=False, write_time_adjusted_to_utc: bool=False, max_rows_per_page: int | None=None, bloom_filter_options: bool | dict[str, dict[str, str | int]] | dict[str, dict[Any, Any]] | dict[str, dict[str, float]] | dict[str, dict[str, str | float]] | dict[str, dict[str, int]] | dict[str, bool] | dict[str, dict[str, int | float]] | None=None, use_content_defined_chunking: bool=False, **options) -> None: - ... - - def __del__(self) -> None: - ... - - def __enter__(self): - ... - - def __exit__(self, *args, **kwargs): - ... - - def write(self, table_or_batch: Table | RecordBatch, row_group_size: int | None=None) -> None: - ... - - def write_batch(self, batch: pyarrow.RecordBatch, row_group_size: int | None=None) -> None: - ... - - def write_table(self, table: pyarrow.Table, row_group_size: int | None=None) -> None: - ... - - def close(self) -> None: - ... - - def add_key_value_metadata(self, key_value_metadata: dict) -> None: - ... + def __init__( + self, + where, + schema: Schema | None, + filesystem: LocalFileSystem | None = None, + flavor: str | None = None, + version: str = "2.6", + use_dictionary: bool = True, + compression: str = "snappy", + write_statistics: bool = True, + use_deprecated_int96_timestamps: bool | None = None, + compression_level: int | dict[str, int] | None = None, + use_byte_stream_split: bool = False, + column_encoding: dict[str, str] | bool | str | None = None, + writer_engine_version=None, + data_page_version: str = "1.0", + use_compliant_nested_type: bool = True, + encryption_properties: FileEncryptionProperties | None = None, + write_batch_size: int | None = None, + dictionary_pagesize_limit: int | str | None = None, + store_schema: bool = True, + write_page_index: bool = False, + write_page_checksum: bool = False, + sorting_columns: tuple[SortingColumn, SortingColumn] | None = None, + store_decimal_as_integer: bool = False, + write_time_adjusted_to_utc: bool = False, + max_rows_per_page: int | None = None, + bloom_filter_options: bool + | dict[str, dict[str, str | int]] + | dict[str, dict[Any, Any]] + | dict[str, dict[str, float]] + | dict[str, dict[str, str | float]] + | dict[str, dict[str, int]] + | dict[str, bool] + | dict[str, dict[str, int | float]] + | None = None, + use_content_defined_chunking: bool = False, + **options, + ) -> None: ... + def __del__(self) -> None: ... + def __enter__(self): ... + def __exit__(self, *args, **kwargs): ... + def write( + self, table_or_batch: Table | RecordBatch, row_group_size: int | None = None + ) -> None: ... + def write_batch( + self, batch: pyarrow.RecordBatch, row_group_size: int | None = None + ) -> None: ... + def write_table( + self, table: pyarrow.Table, row_group_size: int | None = None + ) -> None: ... + def close(self) -> None: ... + def add_key_value_metadata(self, key_value_metadata: dict) -> None: ... class ParquetDataset: __doc__: Incomplete @@ -165,40 +323,46 @@ class ParquetDataset: _base_dir: Incomplete _dataset: Incomplete - def __init__(self, path_or_paths, filesystem: SubTreeFileSystem | LocalFileSystem | str | None=None, schema: Schema | None=None, *, filters=None, read_dictionary=None, binary_type=None, list_type=None, memory_map: bool=False, buffer_size=None, partitioning: str='hive', ignore_prefixes=None, pre_buffer: bool=True, coerce_int96_timestamp_unit=None, decryption_properties=None, thrift_string_size_limit=None, thrift_container_size_limit=None, schema_depth_limit=None, page_checksum_verification: bool=False, arrow_extensions_enabled: bool=True) -> None: - ... - - def equals(self, other) -> bool: - ... - - def __eq__(self, other) -> bool: - ... - + def __init__( + self, + path_or_paths, + filesystem: SubTreeFileSystem | LocalFileSystem | str | None = None, + schema: Schema | None = None, + *, + filters=None, + read_dictionary=None, + binary_type=None, + list_type=None, + memory_map: bool = False, + buffer_size=None, + partitioning: str = "hive", + ignore_prefixes=None, + pre_buffer: bool = True, + coerce_int96_timestamp_unit=None, + decryption_properties=None, + thrift_string_size_limit=None, + thrift_container_size_limit=None, + schema_depth_limit=None, + page_checksum_verification: bool = False, + arrow_extensions_enabled: bool = True, + ) -> None: ... + def equals(self, other) -> bool: ... + def __eq__(self, other) -> bool: ... @property - def schema(self) -> Schema: - ... - - def read(self, columns: list[str] | None=None, use_threads: bool=True, use_pandas_metadata: bool=False) -> pyarrow.Table: - ... - - def _get_common_pandas_metadata(self) -> dict[bytes, bytes]: - ... - - def read_pandas(self, **kwargs) -> Table: - ... - + def schema(self) -> Schema: ... + def read( + self, + columns: list[str] | None = None, + use_threads: bool = True, + use_pandas_metadata: bool = False, + ) -> pyarrow.Table: ... + def _get_common_pandas_metadata(self) -> dict[bytes, bytes]: ... + def read_pandas(self, **kwargs) -> Table: ... @property - def fragments(self): - ... - + def fragments(self): ... @property - def files(self) -> list[str]: - ... - + def files(self) -> list[str]: ... @property - def filesystem(self) -> LocalFileSystem: - ... - + def filesystem(self) -> LocalFileSystem: ... @property - def partitioning(self): - ... + def partitioning(self): ... diff --git a/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi b/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi index 09df26e59b21..fceed598f204 100644 --- a/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi +++ b/python/pyarrow-stubs/pyarrow/parquet/encryption.pyi @@ -15,9 +15,12 @@ # specific language governing permissions and limitations # under the License. -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete -from pyarrow._parquet_encryption import CryptoFactory as CryptoFactory, DecryptionConfiguration as DecryptionConfiguration, EncryptionConfiguration as EncryptionConfiguration, KmsClient as KmsClient, KmsConnectionConfig as KmsConnectionConfig, create_decryption_properties as create_decryption_properties, create_encryption_properties as create_encryption_properties +from pyarrow._parquet_encryption import ( + CryptoFactory as CryptoFactory, + DecryptionConfiguration as DecryptionConfiguration, + EncryptionConfiguration as EncryptionConfiguration, + KmsClient as KmsClient, + KmsConnectionConfig as KmsConnectionConfig, + create_decryption_properties as create_decryption_properties, + create_encryption_properties as create_encryption_properties, +) diff --git a/python/pyarrow-stubs/pyarrow/substrait.pyi b/python/pyarrow-stubs/pyarrow/substrait.pyi index 095f981d3b97..58bf52066092 100644 --- a/python/pyarrow-stubs/pyarrow/substrait.pyi +++ b/python/pyarrow-stubs/pyarrow/substrait.pyi @@ -15,9 +15,13 @@ # specific language governing permissions and limitations # under the License. -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete -from pyarrow._substrait import BoundExpressions as BoundExpressions, SubstraitSchema as SubstraitSchema, deserialize_expressions as deserialize_expressions, deserialize_schema as deserialize_schema, get_supported_functions as get_supported_functions, run_query as run_query, serialize_expressions as serialize_expressions, serialize_schema as serialize_schema +from pyarrow._substrait import ( + BoundExpressions as BoundExpressions, + SubstraitSchema as SubstraitSchema, + deserialize_expressions as deserialize_expressions, + deserialize_schema as deserialize_schema, + get_supported_functions as get_supported_functions, + run_query as run_query, + serialize_expressions as serialize_expressions, + serialize_schema as serialize_schema, +) diff --git a/python/pyarrow-stubs/pyarrow/types.pyi b/python/pyarrow-stubs/pyarrow/types.pyi index 28de0bc22f86..864bbb851717 100644 --- a/python/pyarrow-stubs/pyarrow/types.pyi +++ b/python/pyarrow-stubs/pyarrow/types.pyi @@ -15,18 +15,16 @@ # specific language governing permissions and limitations # under the License. -from pyarrow.lib import DataType, Decimal128Type, Decimal256Type, Decimal32Type, Decimal64Type, DenseUnionType, DictionaryType, DurationType, FixedSizeBinaryType, FixedSizeListType, LargeListType, LargeListViewType, ListType, ListViewType, MapType, RunEndEncodedType, SparseUnionType, StructType, Time32Type, Time64Type, TimestampType, UuidType -from pyarrow.tests.test_extension_type import PeriodType, PeriodTypeWithClass, PeriodTypeWithToPandasDtype -from pyarrow.tests.test_pandas import DummyExtensionType, MyCustomIntegerType -from typing import Union -import os -import re import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence from _typeshed import Incomplete from enum import IntEnum -from pyarrow.lib import is_boolean_value as is_boolean_value, is_float_value as is_float_value, is_integer_value as is_integer_value +from pyarrow.lib import ( + is_boolean_value as is_boolean_value, + is_float_value as is_float_value, + is_integer_value as is_integer_value, +) from pyarrow.util import doc as doc + _SIGNED_INTEGER_TYPES: Incomplete _UNSIGNED_INTEGER_TYPES: Incomplete _INTEGER_TYPES: set[Incomplete] @@ -38,164 +36,59 @@ _INTERVAL_TYPES: Incomplete _TEMPORAL_TYPES: Incomplete _UNION_TYPES: Incomplete -def is_null(t: pyarrow.DataType) -> bool: - ... - -def is_boolean(t: pyarrow.DataType) -> bool: - ... - -def is_integer(t: pyarrow.DataType) -> bool: - ... - -def is_signed_integer(t: pyarrow.DataType) -> bool: - ... - -def is_unsigned_integer(t: pyarrow.DataType) -> bool: - ... - -def is_int8(t: pyarrow.DataType) -> bool: - ... - -def is_int16(t: pyarrow.DataType) -> bool: - ... - -def is_int32(t: pyarrow.DataType) -> bool: - ... - -def is_int64(t: pyarrow.DataType) -> bool: - ... - -def is_uint8(t: pyarrow.DataType) -> bool: - ... - -def is_uint16(t: pyarrow.DataType) -> bool: - ... - -def is_uint32(t: pyarrow.DataType) -> bool: - ... - -def is_uint64(t: pyarrow.DataType) -> bool: - ... - -def is_floating(t: pyarrow.DataType) -> bool: - ... - -def is_float16(t: pyarrow.DataType) -> bool: - ... - -def is_float32(t: pyarrow.DataType) -> bool: - ... - -def is_float64(t: pyarrow.DataType) -> bool: - ... - -def is_list(t: pyarrow.DataType) -> bool: - ... - -def is_large_list(t: pyarrow.DataType) -> bool: - ... - -def is_fixed_size_list(t: pyarrow.DataType) -> bool: - ... - -def is_list_view(t: pyarrow.DataType) -> bool: - ... - -def is_large_list_view(t: pyarrow.DataType) -> bool: - ... - -def is_struct(t: pyarrow.DataType) -> bool: - ... - -def is_union(t: pyarrow.DataType) -> bool: - ... - -def is_nested(t: pyarrow.DataType) -> bool: - ... - -def is_run_end_encoded(t: pyarrow.DataType) -> bool: - ... - -def is_temporal(t: pyarrow.DataType) -> bool: - ... - -def is_timestamp(t: pyarrow.DataType) -> bool: - ... - -def is_duration(t: pyarrow.DataType) -> bool: - ... - -def is_time(t: pyarrow.DataType) -> bool: - ... - -def is_time32(t: pyarrow.DataType) -> bool: - ... - -def is_time64(t: pyarrow.DataType) -> bool: - ... - -def is_binary(t: pyarrow.DataType) -> bool: - ... - -def is_large_binary(t: pyarrow.DataType) -> bool: - ... - -def is_unicode(t: pyarrow.DataType) -> bool: - ... - -def is_string(t: pyarrow.DataType) -> bool: - ... - -def is_large_unicode(t: pyarrow.DataType): - ... - -def is_large_string(t: pyarrow.DataType) -> bool: - ... - -def is_fixed_size_binary(t: pyarrow.DataType) -> bool: - ... - -def is_binary_view(t: pyarrow.DataType) -> bool: - ... - -def is_string_view(t: pyarrow.DataType) -> bool: - ... - -def is_date(t: pyarrow.DataType) -> bool: - ... - -def is_date32(t: pyarrow.DataType) -> bool: - ... - -def is_date64(t: pyarrow.DataType) -> bool: - ... - -def is_map(t: pyarrow.DataType) -> bool: - ... - -def is_decimal(t: pyarrow.DataType) -> bool: - ... - -def is_decimal32(t: pyarrow.DataType) -> bool: - ... - -def is_decimal64(t: pyarrow.DataType) -> bool: - ... - -def is_decimal128(t: pyarrow.DataType) -> bool: - ... - -def is_decimal256(t: pyarrow.DataType) -> bool: - ... - -def is_dictionary(t: pyarrow.DataType) -> bool: - ... - -def is_interval(t: pyarrow.DataType) -> bool: - ... - -def is_primitive(t: pyarrow.DataType) -> bool: - ... +def is_null(t: pyarrow.DataType) -> bool: ... +def is_boolean(t: pyarrow.DataType) -> bool: ... +def is_integer(t: pyarrow.DataType) -> bool: ... +def is_signed_integer(t: pyarrow.DataType) -> bool: ... +def is_unsigned_integer(t: pyarrow.DataType) -> bool: ... +def is_int8(t: pyarrow.DataType) -> bool: ... +def is_int16(t: pyarrow.DataType) -> bool: ... +def is_int32(t: pyarrow.DataType) -> bool: ... +def is_int64(t: pyarrow.DataType) -> bool: ... +def is_uint8(t: pyarrow.DataType) -> bool: ... +def is_uint16(t: pyarrow.DataType) -> bool: ... +def is_uint32(t: pyarrow.DataType) -> bool: ... +def is_uint64(t: pyarrow.DataType) -> bool: ... +def is_floating(t: pyarrow.DataType) -> bool: ... +def is_float16(t: pyarrow.DataType) -> bool: ... +def is_float32(t: pyarrow.DataType) -> bool: ... +def is_float64(t: pyarrow.DataType) -> bool: ... +def is_list(t: pyarrow.DataType) -> bool: ... +def is_large_list(t: pyarrow.DataType) -> bool: ... +def is_fixed_size_list(t: pyarrow.DataType) -> bool: ... +def is_list_view(t: pyarrow.DataType) -> bool: ... +def is_large_list_view(t: pyarrow.DataType) -> bool: ... +def is_struct(t: pyarrow.DataType) -> bool: ... +def is_union(t: pyarrow.DataType) -> bool: ... +def is_nested(t: pyarrow.DataType) -> bool: ... +def is_run_end_encoded(t: pyarrow.DataType) -> bool: ... +def is_temporal(t: pyarrow.DataType) -> bool: ... +def is_timestamp(t: pyarrow.DataType) -> bool: ... +def is_duration(t: pyarrow.DataType) -> bool: ... +def is_time(t: pyarrow.DataType) -> bool: ... +def is_time32(t: pyarrow.DataType) -> bool: ... +def is_time64(t: pyarrow.DataType) -> bool: ... +def is_binary(t: pyarrow.DataType) -> bool: ... +def is_large_binary(t: pyarrow.DataType) -> bool: ... +def is_unicode(t: pyarrow.DataType) -> bool: ... +def is_string(t: pyarrow.DataType) -> bool: ... +def is_large_unicode(t: pyarrow.DataType): ... +def is_large_string(t: pyarrow.DataType) -> bool: ... +def is_fixed_size_binary(t: pyarrow.DataType) -> bool: ... +def is_binary_view(t: pyarrow.DataType) -> bool: ... +def is_string_view(t: pyarrow.DataType) -> bool: ... +def is_date(t: pyarrow.DataType) -> bool: ... +def is_date32(t: pyarrow.DataType) -> bool: ... +def is_date64(t: pyarrow.DataType) -> bool: ... +def is_map(t: pyarrow.DataType) -> bool: ... +def is_decimal(t: pyarrow.DataType) -> bool: ... +def is_decimal32(t: pyarrow.DataType) -> bool: ... +def is_decimal64(t: pyarrow.DataType) -> bool: ... +def is_decimal128(t: pyarrow.DataType) -> bool: ... +def is_decimal256(t: pyarrow.DataType) -> bool: ... +def is_dictionary(t: pyarrow.DataType) -> bool: ... +def is_interval(t: pyarrow.DataType) -> bool: ... +def is_primitive(t: pyarrow.DataType) -> bool: ... class TypesEnum(IntEnum): NA = ... diff --git a/python/pyarrow-stubs/pyarrow/util.pyi b/python/pyarrow-stubs/pyarrow/util.pyi index 7134014f25b3..7b2fbd29ef13 100644 --- a/python/pyarrow-stubs/pyarrow/util.pyi +++ b/python/pyarrow-stubs/pyarrow/util.pyi @@ -15,57 +15,27 @@ # specific language governing permissions and limitations # under the License. -from io import BufferedReader, BufferedWriter, BytesIO, StringIO -from pathlib import Path -from pyarrow._dataset import FileSystemDataset, FileSystemDatasetFactory, InMemoryDataset, Scanner -from pyarrow.lib import Buffer, BufferOutputStream, BufferReader, BufferedInputStream, BufferedOutputStream, CompressedInputStream, CompressedOutputStream, MockOutputStream, NativeFile, OSFile, PythonFile, RecordBatch, RecordBatchReader, Table -from pyarrow.tests.util import FSProtocolClass -from typing import List, Optional, Tuple, Type, Union -import os -import re -import pyarrow -from typing import Any, Callable, IO, Iterable, Iterator, Literal, Mapping, NamedTuple, Self, Sequence -from _typeshed import Incomplete -_DEPR_MSG: str - -def doc(*docstrings, **params) -> Callable: - ... - -def _deprecate_api(old_name: str, new_name: str, api: Callable, next_version: str, type=...) -> Callable: - ... - -def _deprecate_class(old_name, new_class, next_version, instancecheck: bool=True): - ... - -def _is_iterable(obj: FileSystemDataset | Iterator[Any] | InMemoryDataset | Scanner): - ... - -def _is_path_like(path) -> bool: - ... - -def _stringify_path(path) -> str: - ... +from pyarrow._dataset import FileSystemDataset, InMemoryDataset, Scanner +from typing import Any +from collections.abc import Callable, Iterator -def product(seq): - ... - -def get_contiguous_span(shape: tuple, strides: tuple, itemsize: int) -> int: - ... - -def find_free_port() -> int: - ... - -def guid() -> str: - ... - -def _break_traceback_cycle_from_frame(frame) -> None: - ... - -def _download_urllib(url, out_path) -> None: - ... - -def _download_requests(url, out_path) -> None: - ... +_DEPR_MSG: str -def download_tzdata_on_windows() -> None: - ... +def doc(*docstrings, **params) -> Callable: ... +def _deprecate_api( + old_name: str, new_name: str, api: Callable, next_version: str, type=... +) -> Callable: ... +def _deprecate_class(old_name, new_class, next_version, instancecheck: bool = True): ... +def _is_iterable( + obj: FileSystemDataset | Iterator[Any] | InMemoryDataset | Scanner, +): ... +def _is_path_like(path) -> bool: ... +def _stringify_path(path) -> str: ... +def product(seq): ... +def get_contiguous_span(shape: tuple, strides: tuple, itemsize: int) -> int: ... +def find_free_port() -> int: ... +def guid() -> str: ... +def _break_traceback_cycle_from_frame(frame) -> None: ... +def _download_urllib(url, out_path) -> None: ... +def _download_requests(url, out_path) -> None: ... +def download_tzdata_on_windows() -> None: ...