From df521f30163907ebce8a07dc9bf45b163b5af90b Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Sat, 23 Mar 2024 19:01:05 +0900 Subject: [PATCH 01/28] change is_nested() to use C++ type traits --- python/pyarrow/includes/libarrow.pxd | 3 +++ python/pyarrow/tests/test_types.py | 17 ++++++++++------- python/pyarrow/types.pxi | 12 ++++++++++++ python/pyarrow/types.py | 8 -------- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index f461513e8b3c..0d192f6b36d4 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -3026,3 +3026,6 @@ cdef extern from "arrow/python/udf.h" namespace "arrow::py" nogil: cdef extern from "arrow/compute/cast.h" namespace "arrow::compute": CResult[CDatum] Cast(const CDatum& value, const CCastOptions& options) + +cdef extern from "arrow/type_traits.h" namespace "arrow": + bint is_nested(Type type_id) \ No newline at end of file diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py index 4f66a6f41672..4944a795dfbc 100644 --- a/python/pyarrow/tests/test_types.py +++ b/python/pyarrow/tests/test_types.py @@ -24,6 +24,9 @@ import pytest import hypothesis as h import hypothesis.strategies as st + +from pyarrow.lib import is_nested_type + try: import hypothesis.extra.pytz as tzst except ImportError: @@ -212,13 +215,13 @@ def test_is_nested_or_struct(): assert types.is_struct(struct_ex) assert not types.is_struct(pa.list_(pa.int32())) - assert types.is_nested(struct_ex) - assert types.is_nested(pa.list_(pa.int32())) - assert types.is_nested(pa.list_(pa.int32(), 3)) - assert types.is_nested(pa.large_list(pa.int32())) - assert types.is_nested(pa.list_view(pa.int32())) - assert types.is_nested(pa.large_list_view(pa.int32())) - assert not types.is_nested(pa.int32()) + assert is_nested_type(struct_ex) + assert is_nested_type(pa.list_(pa.int32())) + assert is_nested_type(pa.list_(pa.int32(), 3)) + assert is_nested_type(pa.large_list(pa.int32())) + assert is_nested_type(pa.list_view(pa.int32())) + assert is_nested_type(pa.large_list_view(pa.int32())) + assert not is_nested_type(pa.int32()) def test_is_union(): diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index 018099ae7e65..b326a1acf238 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -5521,3 +5521,15 @@ cdef object alloc_c_stream(ArrowArrayStream** c_stream): # Ensure the capsule destructor doesn't call a random release pointer c_stream[0].release = NULL return PyCapsule_New(c_stream[0], 'arrow_array_stream', &pycapsule_stream_deleter) + +def is_nested_type(data_type): + """ + Check if the data type is a nested type. + + Parameters + ---------- + data_type : DataType + The data type to check + """ + cdef Type type_id = data_type.id + return is_nested(type_id) \ No newline at end of file diff --git a/python/pyarrow/types.py b/python/pyarrow/types.py index 66b1ec33953a..87d3d8b86864 100644 --- a/python/pyarrow/types.py +++ b/python/pyarrow/types.py @@ -40,9 +40,6 @@ lib.Type_DURATION} | _TIME_TYPES | _DATE_TYPES | _INTERVAL_TYPES) _UNION_TYPES = {lib.Type_SPARSE_UNION, lib.Type_DENSE_UNION} -_NESTED_TYPES = {lib.Type_LIST, lib.Type_FIXED_SIZE_LIST, lib.Type_LARGE_LIST, - lib.Type_LIST_VIEW, lib.Type_LARGE_LIST_VIEW, - lib.Type_STRUCT, lib.Type_MAP} | _UNION_TYPES @doc(datatype="null") @@ -172,11 +169,6 @@ def is_union(t): return t.id in _UNION_TYPES -@doc(is_null, datatype="nested type") -def is_nested(t): - return t.id in _NESTED_TYPES - - @doc(is_null, datatype="run-end encoded") def is_run_end_encoded(t): return t.id == lib.Type_RUN_END_ENCODED From 1daac63cf7f7c5973de26f2291c867823e8a6cf5 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Sat, 23 Mar 2024 19:54:01 +0900 Subject: [PATCH 02/28] apply lint --- python/pyarrow/includes/libarrow.pxd | 2 +- python/pyarrow/types.pxi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 0d192f6b36d4..9c20cbbda1ac 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -3028,4 +3028,4 @@ cdef extern from "arrow/compute/cast.h" namespace "arrow::compute": CResult[CDatum] Cast(const CDatum& value, const CCastOptions& options) cdef extern from "arrow/type_traits.h" namespace "arrow": - bint is_nested(Type type_id) \ No newline at end of file + bint is_nested(Type type_id) diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index b326a1acf238..219702c6903c 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -5532,4 +5532,4 @@ def is_nested_type(data_type): The data type to check """ cdef Type type_id = data_type.id - return is_nested(type_id) \ No newline at end of file + return is_nested(type_id) From 5c67223ba2fe4ff82af29ae886a653d183d8f4f0 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Sat, 23 Mar 2024 21:01:01 +0900 Subject: [PATCH 03/28] update to maintain compatibility --- python/pyarrow/lib.pyx | 4 ++++ python/pyarrow/tests/test_types.py | 16 +++++++-------- python/pyarrow/type_traits.pxi | 31 ++++++++++++++++++++++++++++++ python/pyarrow/types.pxi | 10 ---------- python/pyarrow/types.py | 8 +++++++- 5 files changed, 49 insertions(+), 20 deletions(-) create mode 100644 python/pyarrow/type_traits.pxi diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx index 3245e50f0fe6..61cb4e0e1374 100644 --- a/python/pyarrow/lib.pyx +++ b/python/pyarrow/lib.pyx @@ -162,6 +162,10 @@ include "pandas-shim.pxi" # Memory pools and allocation include "memory.pxi" +# TypeTraits + +include "type_traits.pxi" + # DataType, Field, Schema include "types.pxi" diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py index 4944a795dfbc..75cf54be5b1c 100644 --- a/python/pyarrow/tests/test_types.py +++ b/python/pyarrow/tests/test_types.py @@ -25,8 +25,6 @@ import hypothesis as h import hypothesis.strategies as st -from pyarrow.lib import is_nested_type - try: import hypothesis.extra.pytz as tzst except ImportError: @@ -215,13 +213,13 @@ def test_is_nested_or_struct(): assert types.is_struct(struct_ex) assert not types.is_struct(pa.list_(pa.int32())) - assert is_nested_type(struct_ex) - assert is_nested_type(pa.list_(pa.int32())) - assert is_nested_type(pa.list_(pa.int32(), 3)) - assert is_nested_type(pa.large_list(pa.int32())) - assert is_nested_type(pa.list_view(pa.int32())) - assert is_nested_type(pa.large_list_view(pa.int32())) - assert not is_nested_type(pa.int32()) + assert types.is_nested(struct_ex) + assert types.is_nested(pa.list_(pa.int32())) + assert types.is_nested(pa.list_(pa.int32(), 3)) + assert types.is_nested(pa.large_list(pa.int32())) + assert types.is_nested(pa.list_view(pa.int32())) + assert types.is_nested(pa.large_list_view(pa.int32())) + assert not types.is_nested(pa.int32()) def test_is_union(): diff --git a/python/pyarrow/type_traits.pxi b/python/pyarrow/type_traits.pxi new file mode 100644 index 000000000000..b1ebb3f45b17 --- /dev/null +++ b/python/pyarrow/type_traits.pxi @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyarrow.includes.libarrow cimport is_nested +from pyarrow.lib cimport Type + +def is_nested_type(data_type): + """ + Check if the data type is a nested type. + + Parameters + ---------- + data_type : DataType + The data type to check + """ + cdef Type type_id = data_type.id + return is_nested(type_id) diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index 219702c6903c..ff78fa563328 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -5522,14 +5522,4 @@ cdef object alloc_c_stream(ArrowArrayStream** c_stream): c_stream[0].release = NULL return PyCapsule_New(c_stream[0], 'arrow_array_stream', &pycapsule_stream_deleter) -def is_nested_type(data_type): - """ - Check if the data type is a nested type. - Parameters - ---------- - data_type : DataType - The data type to check - """ - cdef Type type_id = data_type.id - return is_nested(type_id) diff --git a/python/pyarrow/types.py b/python/pyarrow/types.py index 87d3d8b86864..775463319cfa 100644 --- a/python/pyarrow/types.py +++ b/python/pyarrow/types.py @@ -20,7 +20,8 @@ from pyarrow.lib import (is_boolean_value, # noqa is_integer_value, - is_float_value) + is_float_value, + is_nested_type) import pyarrow.lib as lib from pyarrow.util import doc @@ -169,6 +170,11 @@ def is_union(t): return t.id in _UNION_TYPES +@doc(is_null, datatype="nested type") +def is_nested(t): + return is_nested_type(t) + + @doc(is_null, datatype="run-end encoded") def is_run_end_encoded(t): return t.id == lib.Type_RUN_END_ENCODED From 0f60359fe7b2ac678a7f70d7eccc405d9800f14e Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Sat, 23 Mar 2024 22:32:51 +0900 Subject: [PATCH 04/28] update type_traits to expose all type check functions provided --- python/pyarrow/includes/libarrow.pxd | 33 +++- python/pyarrow/type_traits.pxi | 232 ++++++++++++++++++++++++++- python/pyarrow/types.py | 54 +++++-- 3 files changed, 299 insertions(+), 20 deletions(-) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 9c20cbbda1ac..983a17383c6f 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -172,9 +172,6 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: CDataTypeLayout layout() c_string ToString() - c_bool is_primitive(Type type) - c_bool is_numeric(Type type) - cdef cppclass CArrayData" arrow::ArrayData": shared_ptr[CDataType] type int64_t length @@ -3028,4 +3025,32 @@ cdef extern from "arrow/compute/cast.h" namespace "arrow::compute": CResult[CDatum] Cast(const CDatum& value, const CCastOptions& options) cdef extern from "arrow/type_traits.h" namespace "arrow": - bint is_nested(Type type_id) + c_bool is_integer(Type type_id) + c_bool is_signed_integer(Type type_id) + c_bool is_unsigned_integer(Type type_id) + c_bool is_floating(Type type_id) + c_bool is_numeric(Type type_id) + cdef c_bool is_decimal(Type type_id) + cdef c_bool is_run_end_type(Type type_id) + c_bool is_primitive(Type type_id) + c_bool is_base_binary_like(Type type_id) + c_bool is_binary_like(Type type_id) + c_bool is_large_binary_like(Type type_id) + c_bool is_binary(Type type_id) + c_bool is_string(Type type_id) + c_bool is_temporal(Type type_id) + c_bool is_time(Type type_id) + c_bool is_date(Type type_id) + c_bool is_interval(Type type_id) + c_bool is_dictionary(Type type_id) + c_bool is_fixed_size_binary(Type type_id) + c_bool is_fixed_width(Type type_id) + c_bool is_var_length_list(Type type_id) + c_bool is_list(Type type_id) + c_bool is_list_like(Type type_id) + c_bool is_var_length_list_like(Type type_id) + c_bool is_list_view(Type type_id) + c_bool is_nested(Type type_id) + c_bool is_union(Type type_id) + c_bool bit_width(Type type_id) + c_bool offset_bit_width(Type type_id) diff --git a/python/pyarrow/type_traits.pxi b/python/pyarrow/type_traits.pxi index b1ebb3f45b17..00be8ac4b089 100644 --- a/python/pyarrow/type_traits.pxi +++ b/python/pyarrow/type_traits.pxi @@ -15,12 +15,42 @@ # specific language governing permissions and limitations # under the License. -from pyarrow.includes.libarrow cimport is_nested +from pyarrow.includes.libarrow cimport (is_integer, + is_signed_integer, + is_unsigned_integer, + is_floating, + is_numeric, + is_decimal, + is_run_end_type, + is_primitive, + is_base_binary_like, + is_binary_like, + is_large_binary_like, + is_binary, + is_string, + is_temporal, + is_time, + is_date, + is_interval, + is_dictionary, + is_fixed_size_binary, + is_fixed_width, + is_var_length_list, + is_list, + is_list_like, + is_var_length_list_like, + is_list_view, + is_nested, + is_union, + bit_width, + offset_bit_width) + from pyarrow.lib cimport Type -def is_nested_type(data_type): + +def is_integer_type(data_type): """ - Check if the data type is a nested type. + Check if the data type is an integer type. Parameters ---------- @@ -28,4 +58,200 @@ def is_nested_type(data_type): The data type to check """ cdef Type type_id = data_type.id + return is_integer(type_id) + +def is_signed_integer_type(data_type): + """ + Check if the data type is a signed integer type. + """ + cdef Type type_id = data_type.id + return is_signed_integer(type_id) + +def is_unsigned_integer_type(data_type): + """ + Check if the data type is an unsigned integer type. + """ + cdef Type type_id = data_type.id + return is_unsigned_integer(type_id) + +def is_floating_type(data_type): + """ + Check if the data type is a floating type. + """ + cdef Type type_id = data_type.id + return is_floating(type_id) + +def is_numeric_type(data_type): + """ + Check if the data type is a numeric type. + """ + cdef Type type_id = data_type.id + return is_numeric(type_id) + +def is_decimal_type(data_type): + """ + Check if the data type is a decimal type. + """ + cdef Type type_id = data_type.id + return is_decimal(type_id) + +def is_run_end_type_py(data_type): + """ + Check if the data type is a run end type. + """ + cdef Type type_id = data_type.id + return is_run_end_type(type_id) + +def is_primitive_type(data_type): + """ + Check if the data type is a primitive type. + """ + cdef Type type_id = data_type.id + return is_primitive(type_id) + +def is_base_binary_like_type(data_type): + """ + Check if the data type is a base_binary_like type. + """ + cdef Type type_id = data_type.id + return is_base_binary_like(type_id) + +def is_binary_like_type(data_type): + """ + Check if the data type is a _binary_like type. + """ + cdef Type type_id = data_type.id + return is_binary_like(type_id) + +def is_large_binary_like_type(data_type): + """ + Check if the data type is a large_binary_like type. + """ + cdef Type type_id = data_type.id + return is_large_binary_like(type_id) + +def is_binary_type(data_type): + """ + Check if the data type is a binary type. + """ + cdef Type type_id = data_type.id + return is_binary(type_id) + +def is_string_type(data_type): + """ + Check if the data type is a string type. + """ + cdef Type type_id = data_type.id + return is_string(type_id) + +def is_temporal_type(data_type): + """ + Check if the data type is a temporal type. + """ + cdef Type type_id = data_type.id + return is_temporal(type_id) + +def is_time_type(data_type): + """ + Check if the data type is a time type. + """ + cdef Type type_id = data_type.id + return is_time(type_id) + +def is_date_type(data_type): + """ + Check if the data type is a date type. + """ + cdef Type type_id = data_type.id + return is_date(type_id) + +def is_interval_type(data_type): + """ + Check if the data type is a interval type. + """ + cdef Type type_id = data_type.id + return is_interval(type_id) + +def is_dictionary_type(data_type): + """ + Check if the data type is a dictionary type. + """ + cdef Type type_id = data_type.id + return is_dictionary(type_id) + +def is_fixed_size_binary_type(data_type): + """ + Check if the data type is a fixed_size_binary type. + """ + cdef Type type_id = data_type.id + return is_fixed_size_binary(type_id) + +def is_fixed_width_type(data_type): + """ + Check if the data type is a fixed_width type. + """ + cdef Type type_id = data_type.id + return is_fixed_width(type_id) + +def is_var_length_list_type(data_type): + """ + Check if the data type is a var_length_list type. + """ + cdef Type type_id = data_type.id + return is_var_length_list(type_id) + +def is_list_type(data_type): + """ + Check if the data type is a list type. + """ + cdef Type type_id = data_type.id + return is_list(type_id) + +def is_list_like_type(data_type): + """ + Check if the data type is a list_like type. + """ + cdef Type type_id = data_type.id + return is_list_like(type_id) + +def is_var_length_list_like_type(data_type): + """ + Check if the data type is a var_length_list_like type. + """ + cdef Type type_id = data_type.id + return is_var_length_list_like(type_id) + +def is_list_view_type(data_type): + """ + Check if the data type is a list_view type. + """ + cdef Type type_id = data_type.id + return is_list_view(type_id) + +def is_nested_type(data_type): + """ + Check if the data type is a nested type. + """ + cdef Type type_id = data_type.id return is_nested(type_id) + +def is_union_type(data_type): + """ + Check if the data type is a union type. + """ + cdef Type type_id = data_type.id + return is_union(type_id) + +def is_bit_width_type(data_type): + """ + Check if the data type is a bit_width type. + """ + cdef Type type_id = data_type.id + return bit_width(type_id) + +def is_offset_bit_width_type(data_type): + """ + Check if the data type is a offset_bit_width type. + """ + cdef Type type_id = data_type.id + return offset_bit_width(type_id) diff --git a/python/pyarrow/types.py b/python/pyarrow/types.py index 775463319cfa..0532d7c7707e 100644 --- a/python/pyarrow/types.py +++ b/python/pyarrow/types.py @@ -21,7 +21,35 @@ from pyarrow.lib import (is_boolean_value, # noqa is_integer_value, is_float_value, - is_nested_type) + is_integer_type, + is_signed_integer_type, + is_unsigned_integer_type, + is_floating_type, + is_numeric_type, + is_decimal_type, + is_run_end_type_py, + is_primitive_type, + is_base_binary_like_type, + is_binary_like_type, + is_large_binary_like_type, + is_binary_type, + is_string_type, + is_temporal_type, + is_time_type, + is_date_type, + is_interval_type, + is_dictionary_type, + is_fixed_size_binary_type, + is_fixed_width_type, + is_var_length_list_type, + is_list_type, + is_list_like_type, + is_var_length_list_like_type, + is_list_view_type, + is_nested_type, + is_union_type, + is_bit_width_type, + is_offset_bit_width_type) import pyarrow.lib as lib from pyarrow.util import doc @@ -62,17 +90,17 @@ def is_boolean(t): @doc(is_null, datatype="any integer") def is_integer(t): - return t.id in _INTEGER_TYPES + return is_integer_type(t) @doc(is_null, datatype="signed integer") def is_signed_integer(t): - return t.id in _SIGNED_INTEGER_TYPES + return is_signed_integer_type(t) @doc(is_null, datatype="unsigned integer") def is_unsigned_integer(t): - return t.id in _UNSIGNED_INTEGER_TYPES + return is_unsigned_integer_type(t) @doc(is_null, datatype="int8") @@ -117,7 +145,7 @@ def is_uint64(t): @doc(is_null, datatype="floating point numeric") def is_floating(t): - return t.id in _FLOATING_TYPES + return is_floating_type(t) @doc(is_null, datatype="float16 (half-precision)") @@ -167,7 +195,7 @@ def is_struct(t): @doc(is_null, datatype="union") def is_union(t): - return t.id in _UNION_TYPES + return is_union_type(t) @doc(is_null, datatype="nested type") @@ -197,7 +225,7 @@ def is_duration(t): @doc(is_null, datatype="time") def is_time(t): - return t.id in _TIME_TYPES + return is_time_type(t) @doc(is_null, datatype="time32") @@ -249,7 +277,7 @@ def is_large_string(t): @doc(is_null, datatype="fixed size binary") def is_fixed_size_binary(t): - return t.id == lib.Type_FIXED_SIZE_BINARY + return is_fixed_size_binary_type(t) @doc(is_null, datatype="variable-length binary view") @@ -264,7 +292,7 @@ def is_string_view(t): @doc(is_null, datatype="date") def is_date(t): - return t.id in _DATE_TYPES + return is_date_type(t) @doc(is_null, datatype="date32 (days)") @@ -284,7 +312,7 @@ def is_map(t): @doc(is_null, datatype="decimal") def is_decimal(t): - return t.id in _DECIMAL_TYPES + return is_decimal_type(t) @doc(is_null, datatype="decimal128") @@ -299,14 +327,14 @@ def is_decimal256(t): @doc(is_null, datatype="dictionary-encoded") def is_dictionary(t): - return t.id == lib.Type_DICTIONARY + return is_dictionary_type(t) @doc(is_null, datatype="interval") def is_interval(t): - return t.id == lib.Type_INTERVAL_MONTH_DAY_NANO + return is_interval_type(t) @doc(is_null, datatype="primitive type") def is_primitive(t): - return lib._is_primitive(t.id) + return is_primitive_type(t) From 818a7abcea2ace10e70ac8b4e271b8b01c754714 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Sat, 23 Mar 2024 22:42:31 +0900 Subject: [PATCH 05/28] update to code readability --- python/pyarrow/type_traits.pxi | 89 +++++++++++----------------------- 1 file changed, 29 insertions(+), 60 deletions(-) diff --git a/python/pyarrow/type_traits.pxi b/python/pyarrow/type_traits.pxi index 00be8ac4b089..d4f993107370 100644 --- a/python/pyarrow/type_traits.pxi +++ b/python/pyarrow/type_traits.pxi @@ -45,8 +45,6 @@ from pyarrow.includes.libarrow cimport (is_integer, bit_width, offset_bit_width) -from pyarrow.lib cimport Type - def is_integer_type(data_type): """ @@ -57,201 +55,172 @@ def is_integer_type(data_type): data_type : DataType The data type to check """ - cdef Type type_id = data_type.id - return is_integer(type_id) + return is_integer(data_type.id) def is_signed_integer_type(data_type): """ Check if the data type is a signed integer type. """ - cdef Type type_id = data_type.id - return is_signed_integer(type_id) + return is_signed_integer(data_type.id) def is_unsigned_integer_type(data_type): """ Check if the data type is an unsigned integer type. """ - cdef Type type_id = data_type.id - return is_unsigned_integer(type_id) + return is_unsigned_integer(data_type.id) def is_floating_type(data_type): """ Check if the data type is a floating type. """ - cdef Type type_id = data_type.id - return is_floating(type_id) + return is_floating(data_type.id) def is_numeric_type(data_type): """ Check if the data type is a numeric type. """ - cdef Type type_id = data_type.id - return is_numeric(type_id) + return is_numeric(data_type.id) def is_decimal_type(data_type): """ Check if the data type is a decimal type. """ - cdef Type type_id = data_type.id - return is_decimal(type_id) + return is_decimal(data_type.id) def is_run_end_type_py(data_type): """ Check if the data type is a run end type. """ - cdef Type type_id = data_type.id - return is_run_end_type(type_id) + return is_run_end_type(data_type.id) def is_primitive_type(data_type): """ Check if the data type is a primitive type. """ - cdef Type type_id = data_type.id - return is_primitive(type_id) + return is_primitive(data_type.id) def is_base_binary_like_type(data_type): """ Check if the data type is a base_binary_like type. """ - cdef Type type_id = data_type.id - return is_base_binary_like(type_id) + return is_base_binary_like(data_type.id) def is_binary_like_type(data_type): """ Check if the data type is a _binary_like type. """ - cdef Type type_id = data_type.id - return is_binary_like(type_id) + return is_binary_like(data_type.id) def is_large_binary_like_type(data_type): """ Check if the data type is a large_binary_like type. """ - cdef Type type_id = data_type.id - return is_large_binary_like(type_id) + return is_large_binary_like(data_type.id) def is_binary_type(data_type): """ Check if the data type is a binary type. """ - cdef Type type_id = data_type.id - return is_binary(type_id) + return is_binary(data_type.id) def is_string_type(data_type): """ Check if the data type is a string type. """ - cdef Type type_id = data_type.id - return is_string(type_id) + return is_string(data_type.id) def is_temporal_type(data_type): """ Check if the data type is a temporal type. """ - cdef Type type_id = data_type.id - return is_temporal(type_id) + return is_temporal(data_type.id) def is_time_type(data_type): """ Check if the data type is a time type. """ - cdef Type type_id = data_type.id - return is_time(type_id) + return is_time(data_type.id) def is_date_type(data_type): """ Check if the data type is a date type. """ - cdef Type type_id = data_type.id - return is_date(type_id) + return is_date(data_type.id) def is_interval_type(data_type): """ Check if the data type is a interval type. """ - cdef Type type_id = data_type.id - return is_interval(type_id) + return is_interval(data_type.id) def is_dictionary_type(data_type): """ Check if the data type is a dictionary type. """ - cdef Type type_id = data_type.id - return is_dictionary(type_id) + return is_dictionary(data_type.id) def is_fixed_size_binary_type(data_type): """ Check if the data type is a fixed_size_binary type. """ - cdef Type type_id = data_type.id - return is_fixed_size_binary(type_id) + return is_fixed_size_binary(data_type.id) def is_fixed_width_type(data_type): """ Check if the data type is a fixed_width type. """ - cdef Type type_id = data_type.id - return is_fixed_width(type_id) + return is_fixed_width(data_type.id) def is_var_length_list_type(data_type): """ Check if the data type is a var_length_list type. """ - cdef Type type_id = data_type.id - return is_var_length_list(type_id) + return is_var_length_list(data_type.id) def is_list_type(data_type): """ Check if the data type is a list type. """ - cdef Type type_id = data_type.id - return is_list(type_id) + return is_list(data_type.id) def is_list_like_type(data_type): """ Check if the data type is a list_like type. """ - cdef Type type_id = data_type.id - return is_list_like(type_id) + return is_list_like(data_type.id) def is_var_length_list_like_type(data_type): """ Check if the data type is a var_length_list_like type. """ - cdef Type type_id = data_type.id - return is_var_length_list_like(type_id) + return is_var_length_list_like(data_type.id) def is_list_view_type(data_type): """ Check if the data type is a list_view type. """ - cdef Type type_id = data_type.id - return is_list_view(type_id) + return is_list_view(data_type.id) def is_nested_type(data_type): """ Check if the data type is a nested type. """ - cdef Type type_id = data_type.id - return is_nested(type_id) + return is_nested(data_type.id) def is_union_type(data_type): """ Check if the data type is a union type. """ - cdef Type type_id = data_type.id - return is_union(type_id) + return is_union(data_type.id) def is_bit_width_type(data_type): """ Check if the data type is a bit_width type. """ - cdef Type type_id = data_type.id - return bit_width(type_id) + return bit_width(data_type.id) def is_offset_bit_width_type(data_type): """ Check if the data type is a offset_bit_width type. """ - cdef Type type_id = data_type.id - return offset_bit_width(type_id) + return offset_bit_width(data_type.id) From 085f67c72446e09aed0d68b93b93ed6b95790fd3 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Sun, 24 Mar 2024 18:28:13 +0900 Subject: [PATCH 06/28] update docstring --- python/pyarrow/type_traits.pxi | 140 +++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/python/pyarrow/type_traits.pxi b/python/pyarrow/type_traits.pxi index d4f993107370..13bee7c0f669 100644 --- a/python/pyarrow/type_traits.pxi +++ b/python/pyarrow/type_traits.pxi @@ -60,167 +60,307 @@ def is_integer_type(data_type): def is_signed_integer_type(data_type): """ Check if the data type is a signed integer type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_signed_integer(data_type.id) def is_unsigned_integer_type(data_type): """ Check if the data type is an unsigned integer type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_unsigned_integer(data_type.id) def is_floating_type(data_type): """ Check if the data type is a floating type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_floating(data_type.id) def is_numeric_type(data_type): """ Check if the data type is a numeric type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_numeric(data_type.id) def is_decimal_type(data_type): """ Check if the data type is a decimal type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_decimal(data_type.id) def is_run_end_type_py(data_type): """ Check if the data type is a run end type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_run_end_type(data_type.id) def is_primitive_type(data_type): """ Check if the data type is a primitive type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_primitive(data_type.id) def is_base_binary_like_type(data_type): """ Check if the data type is a base_binary_like type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_base_binary_like(data_type.id) def is_binary_like_type(data_type): """ Check if the data type is a _binary_like type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_binary_like(data_type.id) def is_large_binary_like_type(data_type): """ Check if the data type is a large_binary_like type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_large_binary_like(data_type.id) def is_binary_type(data_type): """ Check if the data type is a binary type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_binary(data_type.id) def is_string_type(data_type): """ Check if the data type is a string type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_string(data_type.id) def is_temporal_type(data_type): """ Check if the data type is a temporal type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_temporal(data_type.id) def is_time_type(data_type): """ Check if the data type is a time type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_time(data_type.id) def is_date_type(data_type): """ Check if the data type is a date type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_date(data_type.id) def is_interval_type(data_type): """ Check if the data type is a interval type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_interval(data_type.id) def is_dictionary_type(data_type): """ Check if the data type is a dictionary type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_dictionary(data_type.id) def is_fixed_size_binary_type(data_type): """ Check if the data type is a fixed_size_binary type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_fixed_size_binary(data_type.id) def is_fixed_width_type(data_type): """ Check if the data type is a fixed_width type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_fixed_width(data_type.id) def is_var_length_list_type(data_type): """ Check if the data type is a var_length_list type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_var_length_list(data_type.id) def is_list_type(data_type): """ Check if the data type is a list type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_list(data_type.id) def is_list_like_type(data_type): """ Check if the data type is a list_like type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_list_like(data_type.id) def is_var_length_list_like_type(data_type): """ Check if the data type is a var_length_list_like type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_var_length_list_like(data_type.id) def is_list_view_type(data_type): """ Check if the data type is a list_view type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_list_view(data_type.id) def is_nested_type(data_type): """ Check if the data type is a nested type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_nested(data_type.id) def is_union_type(data_type): """ Check if the data type is a union type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return is_union(data_type.id) def is_bit_width_type(data_type): """ Check if the data type is a bit_width type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return bit_width(data_type.id) def is_offset_bit_width_type(data_type): """ Check if the data type is a offset_bit_width type. + + Parameters + ---------- + data_type : DataType + The data type to check """ return offset_bit_width(data_type.id) From edc1f9bc32bbf8ed2918fd6314dae63848fab91e Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Sun, 24 Mar 2024 19:03:58 +0900 Subject: [PATCH 07/28] apply lint --- python/pyarrow/type_traits.pxi | 82 +++++++++++++++++++++++----------- python/pyarrow/types.pxi | 2 - 2 files changed, 55 insertions(+), 29 deletions(-) diff --git a/python/pyarrow/type_traits.pxi b/python/pyarrow/type_traits.pxi index 13bee7c0f669..936a6878b780 100644 --- a/python/pyarrow/type_traits.pxi +++ b/python/pyarrow/type_traits.pxi @@ -57,6 +57,7 @@ def is_integer_type(data_type): """ return is_integer(data_type.id) + def is_signed_integer_type(data_type): """ Check if the data type is a signed integer type. @@ -68,10 +69,11 @@ def is_signed_integer_type(data_type): """ return is_signed_integer(data_type.id) + def is_unsigned_integer_type(data_type): """ Check if the data type is an unsigned integer type. - + Parameters ---------- data_type : DataType @@ -79,10 +81,11 @@ def is_unsigned_integer_type(data_type): """ return is_unsigned_integer(data_type.id) + def is_floating_type(data_type): """ Check if the data type is a floating type. - + Parameters ---------- data_type : DataType @@ -90,10 +93,11 @@ def is_floating_type(data_type): """ return is_floating(data_type.id) + def is_numeric_type(data_type): """ Check if the data type is a numeric type. - + Parameters ---------- data_type : DataType @@ -101,10 +105,11 @@ def is_numeric_type(data_type): """ return is_numeric(data_type.id) + def is_decimal_type(data_type): """ Check if the data type is a decimal type. - + Parameters ---------- data_type : DataType @@ -112,10 +117,11 @@ def is_decimal_type(data_type): """ return is_decimal(data_type.id) + def is_run_end_type_py(data_type): """ Check if the data type is a run end type. - + Parameters ---------- data_type : DataType @@ -123,10 +129,11 @@ def is_run_end_type_py(data_type): """ return is_run_end_type(data_type.id) + def is_primitive_type(data_type): """ Check if the data type is a primitive type. - + Parameters ---------- data_type : DataType @@ -134,10 +141,11 @@ def is_primitive_type(data_type): """ return is_primitive(data_type.id) + def is_base_binary_like_type(data_type): """ Check if the data type is a base_binary_like type. - + Parameters ---------- data_type : DataType @@ -145,10 +153,11 @@ def is_base_binary_like_type(data_type): """ return is_base_binary_like(data_type.id) + def is_binary_like_type(data_type): """ Check if the data type is a _binary_like type. - + Parameters ---------- data_type : DataType @@ -156,10 +165,11 @@ def is_binary_like_type(data_type): """ return is_binary_like(data_type.id) + def is_large_binary_like_type(data_type): """ Check if the data type is a large_binary_like type. - + Parameters ---------- data_type : DataType @@ -167,10 +177,11 @@ def is_large_binary_like_type(data_type): """ return is_large_binary_like(data_type.id) + def is_binary_type(data_type): """ Check if the data type is a binary type. - + Parameters ---------- data_type : DataType @@ -178,10 +189,11 @@ def is_binary_type(data_type): """ return is_binary(data_type.id) + def is_string_type(data_type): """ Check if the data type is a string type. - + Parameters ---------- data_type : DataType @@ -189,10 +201,11 @@ def is_string_type(data_type): """ return is_string(data_type.id) + def is_temporal_type(data_type): """ Check if the data type is a temporal type. - + Parameters ---------- data_type : DataType @@ -200,10 +213,11 @@ def is_temporal_type(data_type): """ return is_temporal(data_type.id) + def is_time_type(data_type): """ Check if the data type is a time type. - + Parameters ---------- data_type : DataType @@ -211,10 +225,11 @@ def is_time_type(data_type): """ return is_time(data_type.id) + def is_date_type(data_type): """ Check if the data type is a date type. - + Parameters ---------- data_type : DataType @@ -222,10 +237,11 @@ def is_date_type(data_type): """ return is_date(data_type.id) + def is_interval_type(data_type): """ Check if the data type is a interval type. - + Parameters ---------- data_type : DataType @@ -233,10 +249,11 @@ def is_interval_type(data_type): """ return is_interval(data_type.id) + def is_dictionary_type(data_type): """ Check if the data type is a dictionary type. - + Parameters ---------- data_type : DataType @@ -244,10 +261,11 @@ def is_dictionary_type(data_type): """ return is_dictionary(data_type.id) + def is_fixed_size_binary_type(data_type): """ Check if the data type is a fixed_size_binary type. - + Parameters ---------- data_type : DataType @@ -255,10 +273,11 @@ def is_fixed_size_binary_type(data_type): """ return is_fixed_size_binary(data_type.id) + def is_fixed_width_type(data_type): """ Check if the data type is a fixed_width type. - + Parameters ---------- data_type : DataType @@ -266,10 +285,11 @@ def is_fixed_width_type(data_type): """ return is_fixed_width(data_type.id) + def is_var_length_list_type(data_type): """ Check if the data type is a var_length_list type. - + Parameters ---------- data_type : DataType @@ -277,10 +297,11 @@ def is_var_length_list_type(data_type): """ return is_var_length_list(data_type.id) + def is_list_type(data_type): """ Check if the data type is a list type. - + Parameters ---------- data_type : DataType @@ -288,10 +309,11 @@ def is_list_type(data_type): """ return is_list(data_type.id) + def is_list_like_type(data_type): """ Check if the data type is a list_like type. - + Parameters ---------- data_type : DataType @@ -299,10 +321,11 @@ def is_list_like_type(data_type): """ return is_list_like(data_type.id) + def is_var_length_list_like_type(data_type): """ Check if the data type is a var_length_list_like type. - + Parameters ---------- data_type : DataType @@ -310,10 +333,11 @@ def is_var_length_list_like_type(data_type): """ return is_var_length_list_like(data_type.id) + def is_list_view_type(data_type): """ Check if the data type is a list_view type. - + Parameters ---------- data_type : DataType @@ -321,10 +345,11 @@ def is_list_view_type(data_type): """ return is_list_view(data_type.id) + def is_nested_type(data_type): """ Check if the data type is a nested type. - + Parameters ---------- data_type : DataType @@ -332,10 +357,11 @@ def is_nested_type(data_type): """ return is_nested(data_type.id) + def is_union_type(data_type): """ Check if the data type is a union type. - + Parameters ---------- data_type : DataType @@ -343,10 +369,11 @@ def is_union_type(data_type): """ return is_union(data_type.id) + def is_bit_width_type(data_type): """ Check if the data type is a bit_width type. - + Parameters ---------- data_type : DataType @@ -354,10 +381,11 @@ def is_bit_width_type(data_type): """ return bit_width(data_type.id) + def is_offset_bit_width_type(data_type): """ Check if the data type is a offset_bit_width type. - + Parameters ---------- data_type : DataType diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index ff78fa563328..018099ae7e65 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -5521,5 +5521,3 @@ cdef object alloc_c_stream(ArrowArrayStream** c_stream): # Ensure the capsule destructor doesn't call a random release pointer c_stream[0].release = NULL return PyCapsule_New(c_stream[0], 'arrow_array_stream', &pycapsule_stream_deleter) - - From 38750b7fbc5049466d899ad68041a3b5da6e15ae Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Sun, 24 Mar 2024 20:19:07 +0900 Subject: [PATCH 08/28] delete not used types --- python/pyarrow/types.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/python/pyarrow/types.py b/python/pyarrow/types.py index 0532d7c7707e..4d174ed44a10 100644 --- a/python/pyarrow/types.py +++ b/python/pyarrow/types.py @@ -55,20 +55,12 @@ from pyarrow.util import doc -_SIGNED_INTEGER_TYPES = {lib.Type_INT8, lib.Type_INT16, lib.Type_INT32, - lib.Type_INT64} -_UNSIGNED_INTEGER_TYPES = {lib.Type_UINT8, lib.Type_UINT16, lib.Type_UINT32, - lib.Type_UINT64} -_INTEGER_TYPES = _SIGNED_INTEGER_TYPES | _UNSIGNED_INTEGER_TYPES -_FLOATING_TYPES = {lib.Type_HALF_FLOAT, lib.Type_FLOAT, lib.Type_DOUBLE} -_DECIMAL_TYPES = {lib.Type_DECIMAL128, lib.Type_DECIMAL256} _DATE_TYPES = {lib.Type_DATE32, lib.Type_DATE64} _TIME_TYPES = {lib.Type_TIME32, lib.Type_TIME64} _INTERVAL_TYPES = {lib.Type_INTERVAL_MONTH_DAY_NANO} _TEMPORAL_TYPES = ({lib.Type_TIMESTAMP, lib.Type_DURATION} | _TIME_TYPES | _DATE_TYPES | _INTERVAL_TYPES) -_UNION_TYPES = {lib.Type_SPARSE_UNION, lib.Type_DENSE_UNION} @doc(datatype="null") From cbaada007f8e2a43ccdad6f2735691e9a0811b8e Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Sun, 24 Mar 2024 20:56:55 +0900 Subject: [PATCH 09/28] update by self-review --- python/pyarrow/includes/libarrow.pxd | 4 ++-- python/pyarrow/lib.pyx | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 983a17383c6f..3072bf1d47e4 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -3030,8 +3030,8 @@ cdef extern from "arrow/type_traits.h" namespace "arrow": c_bool is_unsigned_integer(Type type_id) c_bool is_floating(Type type_id) c_bool is_numeric(Type type_id) - cdef c_bool is_decimal(Type type_id) - cdef c_bool is_run_end_type(Type type_id) + c_bool is_decimal(Type type_id) + c_bool is_run_end_type(Type type_id) c_bool is_primitive(Type type_id) c_bool is_base_binary_like(Type type_id) c_bool is_binary_like(Type type_id) diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx index 61cb4e0e1374..277d5df4cf62 100644 --- a/python/pyarrow/lib.pyx +++ b/python/pyarrow/lib.pyx @@ -163,7 +163,6 @@ include "pandas-shim.pxi" include "memory.pxi" # TypeTraits - include "type_traits.pxi" # DataType, Field, Schema From 4d18b7e153efe4cb8171b30a9eb07503a1e03099 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Thu, 28 Mar 2024 15:28:47 +0900 Subject: [PATCH 10/28] enhance docstring to help users understand how to use the function --- python/pyarrow/type_traits.pxi | 337 ++++++++++++++++++++++++++++++++- 1 file changed, 332 insertions(+), 5 deletions(-) diff --git a/python/pyarrow/type_traits.pxi b/python/pyarrow/type_traits.pxi index 936a6878b780..9c283302850f 100644 --- a/python/pyarrow/type_traits.pxi +++ b/python/pyarrow/type_traits.pxi @@ -50,10 +50,21 @@ def is_integer_type(data_type): """ Check if the data type is an integer type. + This function checks whether the `data_type` is an integer type, which + includes signed and unsigned integers of various bit widths (8, 16, 32, 64 bits). + + Useful for ensuring that a given data type is an integer type before performing + operations that are only valid for integers. + Parameters ---------- data_type : DataType - The data type to check + The data type to check against the set of supported integer types. + + Returns + ------- + bool + True if `data_type` is an integer type, False otherwise. """ return is_integer(data_type.id) @@ -62,10 +73,21 @@ def is_signed_integer_type(data_type): """ Check if the data type is a signed integer type. + This function checks whether the `data_type` is a signed integer type, + which includes signed integers of various bit widths (8, 16, 32, 64 bits). + + Useful for ensuring that a given data type is a signed integer type before + performing operations that are only valid for signed integers. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a signed integer type, False otherwise. """ return is_signed_integer(data_type.id) @@ -74,10 +96,21 @@ def is_unsigned_integer_type(data_type): """ Check if the data type is an unsigned integer type. + This function checks whether the `data_type` is an unsigned integer type, + which includes unsigned integers of various bit widths (8, 16, 32, 64 bits). + + Useful for ensuring that a given data type is an unsigned integer type before + performing operations that are only valid for unsigned integers. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is an unsigned integer type, False otherwise. """ return is_unsigned_integer(data_type.id) @@ -86,10 +119,21 @@ def is_floating_type(data_type): """ Check if the data type is a floating type. + This function checks whether the `data_type` is a floating type, which includes + floating point numbers of various bit widths (16, 32, 64 bits). + + Useful for ensuring that a given data type is a floating type before performing + operations that are only valid for floating point numbers. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a floating type, False otherwise. """ return is_floating(data_type.id) @@ -98,10 +142,24 @@ def is_numeric_type(data_type): """ Check if the data type is a numeric type. + This function checks whether the `data_type` is a numeric type, which includes + integers and floating point numbers with specific bit widths. Integer types + include signed and unsigned integers of various bit widths (8, 16, 32, 64 bits), + while floating point types include floating point numbers of various bit widths + (16, 32, 64 bits). + + Useful for ensuring that a given data type is a numeric type before performing + operations that are only valid for numeric types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a numeric type, False otherwise. """ return is_numeric(data_type.id) @@ -110,10 +168,21 @@ def is_decimal_type(data_type): """ Check if the data type is a decimal type. + This function checks whether the `data_type` is a decimal type, which includes + fixed-point decimal numbers with specific precision and scale. + + Useful for ensuring that a given data type is a decimal type before performing + operations that are only valid for decimal numbers. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a decimal type, False otherwise. """ return is_decimal(data_type.id) @@ -122,10 +191,21 @@ def is_run_end_type_py(data_type): """ Check if the data type is a run end type. + This function checks whether the `data_type` is a run end type, which includes + integers of various bit widths (16, 32, 64 bits). + + Useful for ensuring that a given data type is a run end type before performing + operations that are only valid for run end types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a run end type, False otherwise. """ return is_run_end_type(data_type.id) @@ -134,10 +214,25 @@ def is_primitive_type(data_type): """ Check if the data type is a primitive type. + This function checks whether the `data_type` is a primitive type, which includes + integers, floating point numbers, dates (days since the UNIX epoch and milliseconds + since the UNIX epoch), times (seconds and milliseconds since midnight), + timestamp (milliseconds since the UNIX epoch), and duration (elapsed time in + seconds, milliseconds, microseconds, and nanoseconds), and intervals (months, + days plus nanoseconds, and day-time intervals) types. + + Useful for ensuring that a given data type is a primitive type before performing + operations that are only valid for primitive types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a primitive type, False otherwise. """ return is_primitive(data_type.id) @@ -146,10 +241,21 @@ def is_base_binary_like_type(data_type): """ Check if the data type is a base_binary_like type. + This function checks whether the `data_type` is a base binary-like type, which + includes binary, string, large binary, and large string types. + + Useful for ensuring that a given data type is a base binary-like type before + performing operations that are only valid for base binary-like types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a base binary-like type, False otherwise. """ return is_base_binary_like(data_type.id) @@ -158,10 +264,21 @@ def is_binary_like_type(data_type): """ Check if the data type is a _binary_like type. + This function checks whether the `data_type` is a binary-like type, which includes + binary and string types. + + Useful for ensuring that a given data type is a binary-like type before performing + operations that are only valid for binary-like types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a binary-like type, False otherwise. """ return is_binary_like(data_type.id) @@ -170,10 +287,21 @@ def is_large_binary_like_type(data_type): """ Check if the data type is a large_binary_like type. + This function checks whether the `data_type` is a large binary-like type, which + includes large binary and large string types. + + Useful for ensuring that a given data type is a large binary-like type before + performing operations that are only valid for large binary-like types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a large binary-like type, False otherwise. """ return is_large_binary_like(data_type.id) @@ -182,10 +310,21 @@ def is_binary_type(data_type): """ Check if the data type is a binary type. + This function checks whether the `data_type` is a binary type, which includes + binary and large binary types. + + Useful for ensuring that a given data type is a binary type before performing + operations that are only valid for binary types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a binary type, False otherwise. """ return is_binary(data_type.id) @@ -194,10 +333,21 @@ def is_string_type(data_type): """ Check if the data type is a string type. + This function checks whether the `data_type` is a string type, which includes + string and large string types. + + Useful for ensuring that a given data type is a string type before performing + operations that are only valid for string types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a string type, False otherwise. """ return is_string(data_type.id) @@ -206,10 +356,23 @@ def is_temporal_type(data_type): """ Check if the data type is a temporal type. + This function checks whether the `data_type` is a temporal type, which includes + dates (days since the UNIX epoch and milliseconds since the UNIX epoch), times + (seconds and milliseconds since midnight), timestamp (milliseconds since the UNIX + epoch) types. + + Useful for ensuring that a given data type is a temporal type before performing + operations that are only valid for temporal types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a temporal type, False otherwise. """ return is_temporal(data_type.id) @@ -218,10 +381,21 @@ def is_time_type(data_type): """ Check if the data type is a time type. + This function checks whether the `data_type` is a time type, which includes + times (seconds and milliseconds since midnight) types. + + Useful for ensuring that a given data type is a time type before performing + operations that are only valid for time types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a time type, False otherwise. """ return is_time(data_type.id) @@ -230,10 +404,21 @@ def is_date_type(data_type): """ Check if the data type is a date type. + This function checks whether the `data_type` is a date type, which includes + dates (days since the UNIX epoch and milliseconds since the UNIX epoch) types. + + Useful for ensuring that a given data type is a date type before performing + operations that are only valid for date types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a date type, False otherwise. """ return is_date(data_type.id) @@ -242,10 +427,21 @@ def is_interval_type(data_type): """ Check if the data type is a interval type. + This function checks whether the `data_type` is a interval type, which includes + intervals (months, days plus nanoseconds, and day-time intervals) types. + + Useful for ensuring that a given data type is a interval type before performing + operations that are only valid for interval types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a interval type, False otherwise. """ return is_interval(data_type.id) @@ -254,10 +450,20 @@ def is_dictionary_type(data_type): """ Check if the data type is a dictionary type. + This function checks whether the `data_type` is a dictionary type. + + Useful for ensuring that a given data type is a dictionary type before performing + operations that are only valid for dictionary types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a dictionary type, False otherwise. """ return is_dictionary(data_type.id) @@ -266,10 +472,21 @@ def is_fixed_size_binary_type(data_type): """ Check if the data type is a fixed_size_binary type. + This function checks whether the `data_type` is a fixed size binary type + which includes decimal and fixed size binary types. + + Useful for ensuring that a given data type is a fixed size binary type before + performing operations that are only valid for fixed size binary types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a fixed size binary type, False otherwise. """ return is_fixed_size_binary(data_type.id) @@ -278,10 +495,21 @@ def is_fixed_width_type(data_type): """ Check if the data type is a fixed_width type. + This function checks whether the `data_type` is a fixed width type which includes + is_primitive, is_dictionary, is_fixed_size_binary types. + + Useful for ensuring that a given data type is a fixed width type before + performing operations that are only valid for fixed width types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a fixed width type, False otherwise. """ return is_fixed_width(data_type.id) @@ -290,10 +518,21 @@ def is_var_length_list_type(data_type): """ Check if the data type is a var_length_list type. + This function checks whether the `data_type` is a variable length list type + which includes list, large list, and map types. + + Useful for ensuring that a given data type is a variable length list type before + performing operations that are only valid for variable length list types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a variable length list type, False otherwise. """ return is_var_length_list(data_type.id) @@ -302,10 +541,21 @@ def is_list_type(data_type): """ Check if the data type is a list type. + This function checks whether the `data_type` is a list type which includes + list, large list, fixed size list types. + + Useful for ensuring that a given data type is a list type before performing + operations that are only valid for list types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a list type, False otherwise. """ return is_list(data_type.id) @@ -314,10 +564,21 @@ def is_list_like_type(data_type): """ Check if the data type is a list_like type. + This function checks whether the `data_type` is a list_like type which includes + list, large list, fixed size list, and map types. + + Useful for ensuring that a given data type is a list_like type before performing + operations that are only valid for list_like types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a list_like type, False otherwise. """ return is_list_like(data_type.id) @@ -326,10 +587,21 @@ def is_var_length_list_like_type(data_type): """ Check if the data type is a var_length_list_like type. + This function checks whether the `data_type` is a variable length list like + type which includes list, large list, list view, large list view, and map types. + + Useful for ensuring that a given data type is a variable length list like type + before performing operations that are only valid for variable length list like types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a variable length list like type, False otherwise. """ return is_var_length_list_like(data_type.id) @@ -338,10 +610,21 @@ def is_list_view_type(data_type): """ Check if the data type is a list_view type. + This function checks whether the `data_type` is a list view type which includes + list view and large list view types. + + Useful for ensuring that a given data type is a list view type before performing + operations that are only valid for list view types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a list view type, False otherwise. """ return is_list_view(data_type.id) @@ -350,10 +633,22 @@ def is_nested_type(data_type): """ Check if the data type is a nested type. + This function checks whether the `data_type` is a nested type which includes + list, large list, list view, large list view, fixed size list, map, struct, + sparse union, dense union, and run end encoded types. + + Useful for ensuring that a given data type is a nested type before performing + operations that are only valid for nested types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a nested type, False otherwise. """ return is_nested(data_type.id) @@ -362,33 +657,65 @@ def is_union_type(data_type): """ Check if the data type is a union type. + This function checks whether the `data_type` is a union type which includes + sparse union and dense union types. + + Useful for ensuring that a given data type is a union type before performing + operations that are only valid for union types. + Parameters ---------- data_type : DataType The data type to check + + Returns + ------- + bool + True if `data_type` is a union type, False otherwise. """ return is_union(data_type.id) def is_bit_width_type(data_type): """ - Check if the data type is a bit_width type. + Determine the bit width of the data type. + + This function evaluates the `data_type` and returns its bit width, which is + essential for understanding the storage size and precision of data types. + It supports a wide range of data types, including boolean, integers, floating + point numbers, date, time, timestamp, duration, intervals, and decimal types. Parameters ---------- data_type : DataType - The data type to check + The data type for which to determine the bit width. + + Returns + ------- + int + The bit width of the `data_type`, or 0 if the bit width is not applicable. """ return bit_width(data_type.id) def is_offset_bit_width_type(data_type): """ - Check if the data type is a offset_bit_width type. + Determine the offset bit width of the data type. + + This function evaluates the `data_type` and returns its offset bit width, which is + essential for understanding the storage size and precision of data types. + It supports a wide range of data types, including string, binary, list, list + view, map, dense union, large string, large binary, large list, and large list view + types. Parameters ---------- data_type : DataType - The data type to check + The data type for which to determine the offset bit width. + + Returns + ------- + int + The offset bit width of the `data_type`, or 0 if the offset bit width is not applicable. """ return offset_bit_width(data_type.id) From 3d38c7eb68598d4b7be3960f52e3f3180fc44413 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Thu, 28 Mar 2024 15:42:13 +0900 Subject: [PATCH 11/28] update is_integer to check bit width --- python/pyarrow/type_traits.pxi | 4 ++-- python/pyarrow/types.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/pyarrow/type_traits.pxi b/python/pyarrow/type_traits.pxi index 9c283302850f..18d05b2fdfc7 100644 --- a/python/pyarrow/type_traits.pxi +++ b/python/pyarrow/type_traits.pxi @@ -676,7 +676,7 @@ def is_union_type(data_type): return is_union(data_type.id) -def is_bit_width_type(data_type): +def bit_width_type(data_type): """ Determine the bit width of the data type. @@ -698,7 +698,7 @@ def is_bit_width_type(data_type): return bit_width(data_type.id) -def is_offset_bit_width_type(data_type): +def offset_bit_width_type(data_type): """ Determine the offset bit width of the data type. diff --git a/python/pyarrow/types.py b/python/pyarrow/types.py index 4d174ed44a10..735331f2fbe0 100644 --- a/python/pyarrow/types.py +++ b/python/pyarrow/types.py @@ -48,8 +48,8 @@ is_list_view_type, is_nested_type, is_union_type, - is_bit_width_type, - is_offset_bit_width_type) + bit_width_type, + offset_bit_width_type) import pyarrow.lib as lib from pyarrow.util import doc @@ -82,7 +82,7 @@ def is_boolean(t): @doc(is_null, datatype="any integer") def is_integer(t): - return is_integer_type(t) + return is_integer_type(t) and bit_width_type(t) @doc(is_null, datatype="signed integer") From 837e153d7c0b69dc3167f286ac44abbed352474d Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Thu, 28 Mar 2024 16:05:32 +0900 Subject: [PATCH 12/28] fix return type for bit_width and offset_bit_width --- python/pyarrow/includes/libarrow.pxd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 3072bf1d47e4..a77a530de5b8 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -3052,5 +3052,5 @@ cdef extern from "arrow/type_traits.h" namespace "arrow": c_bool is_list_view(Type type_id) c_bool is_nested(Type type_id) c_bool is_union(Type type_id) - c_bool bit_width(Type type_id) - c_bool offset_bit_width(Type type_id) + int bit_width(Type type_id) + int offset_bit_width(Type type_id) From b1d9b10afb6e2f11e4d01c08372c2803f129783c Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Thu, 28 Mar 2024 17:02:45 +0900 Subject: [PATCH 13/28] update to support the logic usign the provided functions --- python/pyarrow/types.py | 71 +++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/python/pyarrow/types.py b/python/pyarrow/types.py index 735331f2fbe0..96ca27f4ad0b 100644 --- a/python/pyarrow/types.py +++ b/python/pyarrow/types.py @@ -55,14 +55,6 @@ from pyarrow.util import doc -_DATE_TYPES = {lib.Type_DATE32, lib.Type_DATE64} -_TIME_TYPES = {lib.Type_TIME32, lib.Type_TIME64} -_INTERVAL_TYPES = {lib.Type_INTERVAL_MONTH_DAY_NANO} -_TEMPORAL_TYPES = ({lib.Type_TIMESTAMP, - lib.Type_DURATION} | _TIME_TYPES | _DATE_TYPES | - _INTERVAL_TYPES) - - @doc(datatype="null") def is_null(t): """ @@ -77,7 +69,7 @@ def is_null(t): @doc(is_null, datatype="boolean") def is_boolean(t): - return t.id == lib.Type_BOOL + return is_primitive_type(t) and bit_width_type(t) == 1 @doc(is_null, datatype="any integer") @@ -97,42 +89,42 @@ def is_unsigned_integer(t): @doc(is_null, datatype="int8") def is_int8(t): - return t.id == lib.Type_INT8 + return is_integer_type(t) and bit_width_type(t) == 8 @doc(is_null, datatype="int16") def is_int16(t): - return t.id == lib.Type_INT16 + return is_integer_type(t) and bit_width_type(t) == 16 @doc(is_null, datatype="int32") def is_int32(t): - return t.id == lib.Type_INT32 + return is_integer_type(t) and bit_width_type(t) == 32 @doc(is_null, datatype="int64") def is_int64(t): - return t.id == lib.Type_INT64 + return is_integer_type(t) and bit_width_type(t) == 64 @doc(is_null, datatype="uint8") def is_uint8(t): - return t.id == lib.Type_UINT8 + return is_unsigned_integer_type(t) and bit_width_type(t) == 8 @doc(is_null, datatype="uint16") def is_uint16(t): - return t.id == lib.Type_UINT16 + return is_unsigned_integer_type(t) and bit_width_type(t) == 16 @doc(is_null, datatype="uint32") def is_uint32(t): - return t.id == lib.Type_UINT32 + return is_unsigned_integer_type(t) and bit_width_type(t) == 32 @doc(is_null, datatype="uint64") def is_uint64(t): - return t.id == lib.Type_UINT64 + return is_unsigned_integer_type(t) and bit_width_type(t) == 64 @doc(is_null, datatype="floating point numeric") @@ -142,27 +134,27 @@ def is_floating(t): @doc(is_null, datatype="float16 (half-precision)") def is_float16(t): - return t.id == lib.Type_HALF_FLOAT + return is_floating_type(t) and bit_width_type(t) == 16 @doc(is_null, datatype="float32 (single precision)") def is_float32(t): - return t.id == lib.Type_FLOAT + return is_floating_type(t) and bit_width_type(t) == 32 @doc(is_null, datatype="float64 (double precision)") def is_float64(t): - return t.id == lib.Type_DOUBLE + return is_floating_type(t) and bit_width_type(t) == 64 @doc(is_null, datatype="list") def is_list(t): - return t.id == lib.Type_LIST + return is_list_type(t) and offset_bit_width_type(t) == 32 @doc(is_null, datatype="large list") def is_large_list(t): - return t.id == lib.Type_LARGE_LIST + return is_list_type(t) and offset_bit_width_type(t) == 64 @doc(is_null, datatype="fixed size list") @@ -172,12 +164,12 @@ def is_fixed_size_list(t): @doc(is_null, datatype="list view") def is_list_view(t): - return t.id == lib.Type_LIST_VIEW + return is_list_view_type(t) and offset_bit_width_type(t) == 32 @doc(is_null, datatype="large list view") def is_large_list_view(t): - return t.id == lib.Type_LARGE_LIST_VIEW + return is_list_view_type(t) and offset_bit_width_type(t) == 64 @doc(is_null, datatype="struct") @@ -202,17 +194,20 @@ def is_run_end_encoded(t): @doc(is_null, datatype="date, time, timestamp or duration") def is_temporal(t): - return t.id in _TEMPORAL_TYPES + return is_primitive_type(t) and not is_integer_type(t) and \ + not is_floating_type(t) @doc(is_null, datatype="timestamp") def is_timestamp(t): - return t.id == lib.Type_TIMESTAMP + return is_temporal_type(t) and not is_time_type(t) and not is_date_type(t) @doc(is_null, datatype="duration") def is_duration(t): - return t.id == lib.Type_DURATION + return is_primitive_type(t) and not is_integer_type(t) and \ + not is_floating_type(t) and not is_temporal_type(t) and \ + not is_interval_type(t) @doc(is_null, datatype="time") @@ -222,22 +217,22 @@ def is_time(t): @doc(is_null, datatype="time32") def is_time32(t): - return t.id == lib.Type_TIME32 + return is_time_type(t) and bit_width_type(t) == 32 @doc(is_null, datatype="time64") def is_time64(t): - return t.id == lib.Type_TIME64 + return is_time_type(t) and bit_width_type(t) == 64 @doc(is_null, datatype="variable-length binary") def is_binary(t): - return t.id == lib.Type_BINARY + return is_binary_type(t) and offset_bit_width_type(t) == 32 @doc(is_null, datatype="large variable-length binary") def is_large_binary(t): - return t.id == lib.Type_LARGE_BINARY + return is_binary_type(t) and offset_bit_width_type(t) == 64 @doc(method="is_string") @@ -254,7 +249,7 @@ def is_unicode(t): @doc(is_null, datatype="string (utf8 unicode)") def is_string(t): - return t.id == lib.Type_STRING + return is_string_type(t) and offset_bit_width_type(t) == 32 @doc(is_unicode, method="is_large_string") @@ -264,7 +259,7 @@ def is_large_unicode(t): @doc(is_null, datatype="large string (utf8 unicode)") def is_large_string(t): - return t.id == lib.Type_LARGE_STRING + return is_string_type(t) and offset_bit_width_type(t) == 64 @doc(is_null, datatype="fixed size binary") @@ -289,17 +284,17 @@ def is_date(t): @doc(is_null, datatype="date32 (days)") def is_date32(t): - return t.id == lib.Type_DATE32 + return is_date_type(t) and bit_width_type(t) == 32 @doc(is_null, datatype="date64 (milliseconds)") def is_date64(t): - return t.id == lib.Type_DATE64 + return is_date_type(t) and bit_width_type(t) == 64 @doc(is_null, datatype="map") def is_map(t): - return t.id == lib.Type_MAP + return is_var_length_list_type(t) and not is_list_type(t) @doc(is_null, datatype="decimal") @@ -309,12 +304,12 @@ def is_decimal(t): @doc(is_null, datatype="decimal128") def is_decimal128(t): - return t.id == lib.Type_DECIMAL128 + return is_decimal_type(t) and bit_width_type(t) == 128 @doc(is_null, datatype="decimal256") def is_decimal256(t): - return t.id == lib.Type_DECIMAL256 + return is_decimal_type(t) and bit_width_type(t) == 256 @doc(is_null, datatype="dictionary-encoded") From 8810b4b8f2ead1617563ad9155d45884b5dc1097 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Wed, 3 Apr 2024 21:36:31 +0900 Subject: [PATCH 14/28] update docstring removing 'Check if~' and 'Useful for' --- python/pyarrow/type_traits.pxi | 135 --------------------------------- 1 file changed, 135 deletions(-) diff --git a/python/pyarrow/type_traits.pxi b/python/pyarrow/type_traits.pxi index 18d05b2fdfc7..274a4886ee65 100644 --- a/python/pyarrow/type_traits.pxi +++ b/python/pyarrow/type_traits.pxi @@ -48,14 +48,9 @@ from pyarrow.includes.libarrow cimport (is_integer, def is_integer_type(data_type): """ - Check if the data type is an integer type. - This function checks whether the `data_type` is an integer type, which includes signed and unsigned integers of various bit widths (8, 16, 32, 64 bits). - Useful for ensuring that a given data type is an integer type before performing - operations that are only valid for integers. - Parameters ---------- data_type : DataType @@ -71,14 +66,9 @@ def is_integer_type(data_type): def is_signed_integer_type(data_type): """ - Check if the data type is a signed integer type. - This function checks whether the `data_type` is a signed integer type, which includes signed integers of various bit widths (8, 16, 32, 64 bits). - Useful for ensuring that a given data type is a signed integer type before - performing operations that are only valid for signed integers. - Parameters ---------- data_type : DataType @@ -94,14 +84,9 @@ def is_signed_integer_type(data_type): def is_unsigned_integer_type(data_type): """ - Check if the data type is an unsigned integer type. - This function checks whether the `data_type` is an unsigned integer type, which includes unsigned integers of various bit widths (8, 16, 32, 64 bits). - Useful for ensuring that a given data type is an unsigned integer type before - performing operations that are only valid for unsigned integers. - Parameters ---------- data_type : DataType @@ -117,14 +102,9 @@ def is_unsigned_integer_type(data_type): def is_floating_type(data_type): """ - Check if the data type is a floating type. - This function checks whether the `data_type` is a floating type, which includes floating point numbers of various bit widths (16, 32, 64 bits). - Useful for ensuring that a given data type is a floating type before performing - operations that are only valid for floating point numbers. - Parameters ---------- data_type : DataType @@ -140,17 +120,12 @@ def is_floating_type(data_type): def is_numeric_type(data_type): """ - Check if the data type is a numeric type. - This function checks whether the `data_type` is a numeric type, which includes integers and floating point numbers with specific bit widths. Integer types include signed and unsigned integers of various bit widths (8, 16, 32, 64 bits), while floating point types include floating point numbers of various bit widths (16, 32, 64 bits). - Useful for ensuring that a given data type is a numeric type before performing - operations that are only valid for numeric types. - Parameters ---------- data_type : DataType @@ -166,14 +141,9 @@ def is_numeric_type(data_type): def is_decimal_type(data_type): """ - Check if the data type is a decimal type. - This function checks whether the `data_type` is a decimal type, which includes fixed-point decimal numbers with specific precision and scale. - Useful for ensuring that a given data type is a decimal type before performing - operations that are only valid for decimal numbers. - Parameters ---------- data_type : DataType @@ -189,14 +159,9 @@ def is_decimal_type(data_type): def is_run_end_type_py(data_type): """ - Check if the data type is a run end type. - This function checks whether the `data_type` is a run end type, which includes integers of various bit widths (16, 32, 64 bits). - Useful for ensuring that a given data type is a run end type before performing - operations that are only valid for run end types. - Parameters ---------- data_type : DataType @@ -212,8 +177,6 @@ def is_run_end_type_py(data_type): def is_primitive_type(data_type): """ - Check if the data type is a primitive type. - This function checks whether the `data_type` is a primitive type, which includes integers, floating point numbers, dates (days since the UNIX epoch and milliseconds since the UNIX epoch), times (seconds and milliseconds since midnight), @@ -221,9 +184,6 @@ def is_primitive_type(data_type): seconds, milliseconds, microseconds, and nanoseconds), and intervals (months, days plus nanoseconds, and day-time intervals) types. - Useful for ensuring that a given data type is a primitive type before performing - operations that are only valid for primitive types. - Parameters ---------- data_type : DataType @@ -239,14 +199,9 @@ def is_primitive_type(data_type): def is_base_binary_like_type(data_type): """ - Check if the data type is a base_binary_like type. - This function checks whether the `data_type` is a base binary-like type, which includes binary, string, large binary, and large string types. - Useful for ensuring that a given data type is a base binary-like type before - performing operations that are only valid for base binary-like types. - Parameters ---------- data_type : DataType @@ -262,14 +217,9 @@ def is_base_binary_like_type(data_type): def is_binary_like_type(data_type): """ - Check if the data type is a _binary_like type. - This function checks whether the `data_type` is a binary-like type, which includes binary and string types. - Useful for ensuring that a given data type is a binary-like type before performing - operations that are only valid for binary-like types. - Parameters ---------- data_type : DataType @@ -285,14 +235,9 @@ def is_binary_like_type(data_type): def is_large_binary_like_type(data_type): """ - Check if the data type is a large_binary_like type. - This function checks whether the `data_type` is a large binary-like type, which includes large binary and large string types. - Useful for ensuring that a given data type is a large binary-like type before - performing operations that are only valid for large binary-like types. - Parameters ---------- data_type : DataType @@ -308,14 +253,9 @@ def is_large_binary_like_type(data_type): def is_binary_type(data_type): """ - Check if the data type is a binary type. - This function checks whether the `data_type` is a binary type, which includes binary and large binary types. - Useful for ensuring that a given data type is a binary type before performing - operations that are only valid for binary types. - Parameters ---------- data_type : DataType @@ -331,14 +271,9 @@ def is_binary_type(data_type): def is_string_type(data_type): """ - Check if the data type is a string type. - This function checks whether the `data_type` is a string type, which includes string and large string types. - Useful for ensuring that a given data type is a string type before performing - operations that are only valid for string types. - Parameters ---------- data_type : DataType @@ -354,16 +289,11 @@ def is_string_type(data_type): def is_temporal_type(data_type): """ - Check if the data type is a temporal type. - This function checks whether the `data_type` is a temporal type, which includes dates (days since the UNIX epoch and milliseconds since the UNIX epoch), times (seconds and milliseconds since midnight), timestamp (milliseconds since the UNIX epoch) types. - Useful for ensuring that a given data type is a temporal type before performing - operations that are only valid for temporal types. - Parameters ---------- data_type : DataType @@ -379,14 +309,9 @@ def is_temporal_type(data_type): def is_time_type(data_type): """ - Check if the data type is a time type. - This function checks whether the `data_type` is a time type, which includes times (seconds and milliseconds since midnight) types. - Useful for ensuring that a given data type is a time type before performing - operations that are only valid for time types. - Parameters ---------- data_type : DataType @@ -402,14 +327,9 @@ def is_time_type(data_type): def is_date_type(data_type): """ - Check if the data type is a date type. - This function checks whether the `data_type` is a date type, which includes dates (days since the UNIX epoch and milliseconds since the UNIX epoch) types. - Useful for ensuring that a given data type is a date type before performing - operations that are only valid for date types. - Parameters ---------- data_type : DataType @@ -425,14 +345,9 @@ def is_date_type(data_type): def is_interval_type(data_type): """ - Check if the data type is a interval type. - This function checks whether the `data_type` is a interval type, which includes intervals (months, days plus nanoseconds, and day-time intervals) types. - Useful for ensuring that a given data type is a interval type before performing - operations that are only valid for interval types. - Parameters ---------- data_type : DataType @@ -448,13 +363,8 @@ def is_interval_type(data_type): def is_dictionary_type(data_type): """ - Check if the data type is a dictionary type. - This function checks whether the `data_type` is a dictionary type. - Useful for ensuring that a given data type is a dictionary type before performing - operations that are only valid for dictionary types. - Parameters ---------- data_type : DataType @@ -470,14 +380,9 @@ def is_dictionary_type(data_type): def is_fixed_size_binary_type(data_type): """ - Check if the data type is a fixed_size_binary type. - This function checks whether the `data_type` is a fixed size binary type which includes decimal and fixed size binary types. - Useful for ensuring that a given data type is a fixed size binary type before - performing operations that are only valid for fixed size binary types. - Parameters ---------- data_type : DataType @@ -493,14 +398,9 @@ def is_fixed_size_binary_type(data_type): def is_fixed_width_type(data_type): """ - Check if the data type is a fixed_width type. - This function checks whether the `data_type` is a fixed width type which includes is_primitive, is_dictionary, is_fixed_size_binary types. - Useful for ensuring that a given data type is a fixed width type before - performing operations that are only valid for fixed width types. - Parameters ---------- data_type : DataType @@ -516,14 +416,9 @@ def is_fixed_width_type(data_type): def is_var_length_list_type(data_type): """ - Check if the data type is a var_length_list type. - This function checks whether the `data_type` is a variable length list type which includes list, large list, and map types. - Useful for ensuring that a given data type is a variable length list type before - performing operations that are only valid for variable length list types. - Parameters ---------- data_type : DataType @@ -539,14 +434,9 @@ def is_var_length_list_type(data_type): def is_list_type(data_type): """ - Check if the data type is a list type. - This function checks whether the `data_type` is a list type which includes list, large list, fixed size list types. - Useful for ensuring that a given data type is a list type before performing - operations that are only valid for list types. - Parameters ---------- data_type : DataType @@ -562,14 +452,9 @@ def is_list_type(data_type): def is_list_like_type(data_type): """ - Check if the data type is a list_like type. - This function checks whether the `data_type` is a list_like type which includes list, large list, fixed size list, and map types. - Useful for ensuring that a given data type is a list_like type before performing - operations that are only valid for list_like types. - Parameters ---------- data_type : DataType @@ -585,14 +470,9 @@ def is_list_like_type(data_type): def is_var_length_list_like_type(data_type): """ - Check if the data type is a var_length_list_like type. - This function checks whether the `data_type` is a variable length list like type which includes list, large list, list view, large list view, and map types. - Useful for ensuring that a given data type is a variable length list like type - before performing operations that are only valid for variable length list like types. - Parameters ---------- data_type : DataType @@ -608,14 +488,9 @@ def is_var_length_list_like_type(data_type): def is_list_view_type(data_type): """ - Check if the data type is a list_view type. - This function checks whether the `data_type` is a list view type which includes list view and large list view types. - Useful for ensuring that a given data type is a list view type before performing - operations that are only valid for list view types. - Parameters ---------- data_type : DataType @@ -631,15 +506,10 @@ def is_list_view_type(data_type): def is_nested_type(data_type): """ - Check if the data type is a nested type. - This function checks whether the `data_type` is a nested type which includes list, large list, list view, large list view, fixed size list, map, struct, sparse union, dense union, and run end encoded types. - Useful for ensuring that a given data type is a nested type before performing - operations that are only valid for nested types. - Parameters ---------- data_type : DataType @@ -655,14 +525,9 @@ def is_nested_type(data_type): def is_union_type(data_type): """ - Check if the data type is a union type. - This function checks whether the `data_type` is a union type which includes sparse union and dense union types. - Useful for ensuring that a given data type is a union type before performing - operations that are only valid for union types. - Parameters ---------- data_type : DataType From cec4b1158f7003f34f5571420def6e36f01bf0fa Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Wed, 3 Apr 2024 22:10:33 +0900 Subject: [PATCH 15/28] update docstring removing (...) --- python/pyarrow/type_traits.pxi | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/python/pyarrow/type_traits.pxi b/python/pyarrow/type_traits.pxi index 274a4886ee65..a7ebdde852b1 100644 --- a/python/pyarrow/type_traits.pxi +++ b/python/pyarrow/type_traits.pxi @@ -178,11 +178,8 @@ def is_run_end_type_py(data_type): def is_primitive_type(data_type): """ This function checks whether the `data_type` is a primitive type, which includes - integers, floating point numbers, dates (days since the UNIX epoch and milliseconds - since the UNIX epoch), times (seconds and milliseconds since midnight), - timestamp (milliseconds since the UNIX epoch), and duration (elapsed time in - seconds, milliseconds, microseconds, and nanoseconds), and intervals (months, - days plus nanoseconds, and day-time intervals) types. + boolean, integers, floating point numbers, dates, times, timestamp, duration + and intervals types. Parameters ---------- From e9a49545b7c5532eee1dd3ff2b9a41c09eaaaca1 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Wed, 3 Apr 2024 22:31:44 +0900 Subject: [PATCH 16/28] update function to start with _ prefix --- python/pyarrow/type_traits.pxi | 58 ++++++------ python/pyarrow/types.pxi | 5 -- python/pyarrow/types.py | 156 ++++++++++++++++----------------- 3 files changed, 107 insertions(+), 112 deletions(-) diff --git a/python/pyarrow/type_traits.pxi b/python/pyarrow/type_traits.pxi index a7ebdde852b1..2952261194b0 100644 --- a/python/pyarrow/type_traits.pxi +++ b/python/pyarrow/type_traits.pxi @@ -46,7 +46,7 @@ from pyarrow.includes.libarrow cimport (is_integer, offset_bit_width) -def is_integer_type(data_type): +def _is_integer(data_type): """ This function checks whether the `data_type` is an integer type, which includes signed and unsigned integers of various bit widths (8, 16, 32, 64 bits). @@ -64,7 +64,7 @@ def is_integer_type(data_type): return is_integer(data_type.id) -def is_signed_integer_type(data_type): +def _is_signed_integer(data_type): """ This function checks whether the `data_type` is a signed integer type, which includes signed integers of various bit widths (8, 16, 32, 64 bits). @@ -82,7 +82,7 @@ def is_signed_integer_type(data_type): return is_signed_integer(data_type.id) -def is_unsigned_integer_type(data_type): +def _is_unsigned_integer(data_type): """ This function checks whether the `data_type` is an unsigned integer type, which includes unsigned integers of various bit widths (8, 16, 32, 64 bits). @@ -100,7 +100,7 @@ def is_unsigned_integer_type(data_type): return is_unsigned_integer(data_type.id) -def is_floating_type(data_type): +def _is_floating(data_type): """ This function checks whether the `data_type` is a floating type, which includes floating point numbers of various bit widths (16, 32, 64 bits). @@ -118,7 +118,7 @@ def is_floating_type(data_type): return is_floating(data_type.id) -def is_numeric_type(data_type): +def _is_numeric(data_type): """ This function checks whether the `data_type` is a numeric type, which includes integers and floating point numbers with specific bit widths. Integer types @@ -139,7 +139,7 @@ def is_numeric_type(data_type): return is_numeric(data_type.id) -def is_decimal_type(data_type): +def _is_decimal(data_type): """ This function checks whether the `data_type` is a decimal type, which includes fixed-point decimal numbers with specific precision and scale. @@ -157,7 +157,7 @@ def is_decimal_type(data_type): return is_decimal(data_type.id) -def is_run_end_type_py(data_type): +def _is_run_end_type(data_type): """ This function checks whether the `data_type` is a run end type, which includes integers of various bit widths (16, 32, 64 bits). @@ -175,7 +175,7 @@ def is_run_end_type_py(data_type): return is_run_end_type(data_type.id) -def is_primitive_type(data_type): +def _is_primitive(data_type): """ This function checks whether the `data_type` is a primitive type, which includes boolean, integers, floating point numbers, dates, times, timestamp, duration @@ -194,7 +194,7 @@ def is_primitive_type(data_type): return is_primitive(data_type.id) -def is_base_binary_like_type(data_type): +def _is_base_binary_like(data_type): """ This function checks whether the `data_type` is a base binary-like type, which includes binary, string, large binary, and large string types. @@ -212,7 +212,7 @@ def is_base_binary_like_type(data_type): return is_base_binary_like(data_type.id) -def is_binary_like_type(data_type): +def _is_binary_like(data_type): """ This function checks whether the `data_type` is a binary-like type, which includes binary and string types. @@ -230,7 +230,7 @@ def is_binary_like_type(data_type): return is_binary_like(data_type.id) -def is_large_binary_like_type(data_type): +def _is_large_binary_like(data_type): """ This function checks whether the `data_type` is a large binary-like type, which includes large binary and large string types. @@ -248,7 +248,7 @@ def is_large_binary_like_type(data_type): return is_large_binary_like(data_type.id) -def is_binary_type(data_type): +def _is_binary(data_type): """ This function checks whether the `data_type` is a binary type, which includes binary and large binary types. @@ -266,7 +266,7 @@ def is_binary_type(data_type): return is_binary(data_type.id) -def is_string_type(data_type): +def _is_string(data_type): """ This function checks whether the `data_type` is a string type, which includes string and large string types. @@ -284,7 +284,7 @@ def is_string_type(data_type): return is_string(data_type.id) -def is_temporal_type(data_type): +def _is_temporal(data_type): """ This function checks whether the `data_type` is a temporal type, which includes dates (days since the UNIX epoch and milliseconds since the UNIX epoch), times @@ -304,7 +304,7 @@ def is_temporal_type(data_type): return is_temporal(data_type.id) -def is_time_type(data_type): +def _is_time(data_type): """ This function checks whether the `data_type` is a time type, which includes times (seconds and milliseconds since midnight) types. @@ -322,7 +322,7 @@ def is_time_type(data_type): return is_time(data_type.id) -def is_date_type(data_type): +def _is_date(data_type): """ This function checks whether the `data_type` is a date type, which includes dates (days since the UNIX epoch and milliseconds since the UNIX epoch) types. @@ -340,7 +340,7 @@ def is_date_type(data_type): return is_date(data_type.id) -def is_interval_type(data_type): +def _is_interval(data_type): """ This function checks whether the `data_type` is a interval type, which includes intervals (months, days plus nanoseconds, and day-time intervals) types. @@ -358,7 +358,7 @@ def is_interval_type(data_type): return is_interval(data_type.id) -def is_dictionary_type(data_type): +def _is_dictionary(data_type): """ This function checks whether the `data_type` is a dictionary type. @@ -375,7 +375,7 @@ def is_dictionary_type(data_type): return is_dictionary(data_type.id) -def is_fixed_size_binary_type(data_type): +def _is_fixed_size_binary(data_type): """ This function checks whether the `data_type` is a fixed size binary type which includes decimal and fixed size binary types. @@ -393,7 +393,7 @@ def is_fixed_size_binary_type(data_type): return is_fixed_size_binary(data_type.id) -def is_fixed_width_type(data_type): +def _is_fixed_width(data_type): """ This function checks whether the `data_type` is a fixed width type which includes is_primitive, is_dictionary, is_fixed_size_binary types. @@ -411,7 +411,7 @@ def is_fixed_width_type(data_type): return is_fixed_width(data_type.id) -def is_var_length_list_type(data_type): +def _is_var_length_list(data_type): """ This function checks whether the `data_type` is a variable length list type which includes list, large list, and map types. @@ -429,7 +429,7 @@ def is_var_length_list_type(data_type): return is_var_length_list(data_type.id) -def is_list_type(data_type): +def _is_list(data_type): """ This function checks whether the `data_type` is a list type which includes list, large list, fixed size list types. @@ -447,7 +447,7 @@ def is_list_type(data_type): return is_list(data_type.id) -def is_list_like_type(data_type): +def _is_list_like(data_type): """ This function checks whether the `data_type` is a list_like type which includes list, large list, fixed size list, and map types. @@ -465,7 +465,7 @@ def is_list_like_type(data_type): return is_list_like(data_type.id) -def is_var_length_list_like_type(data_type): +def _is_var_length_list_like(data_type): """ This function checks whether the `data_type` is a variable length list like type which includes list, large list, list view, large list view, and map types. @@ -483,7 +483,7 @@ def is_var_length_list_like_type(data_type): return is_var_length_list_like(data_type.id) -def is_list_view_type(data_type): +def _is_list_view(data_type): """ This function checks whether the `data_type` is a list view type which includes list view and large list view types. @@ -501,7 +501,7 @@ def is_list_view_type(data_type): return is_list_view(data_type.id) -def is_nested_type(data_type): +def _is_nested(data_type): """ This function checks whether the `data_type` is a nested type which includes list, large list, list view, large list view, fixed size list, map, struct, @@ -520,7 +520,7 @@ def is_nested_type(data_type): return is_nested(data_type.id) -def is_union_type(data_type): +def _is_union(data_type): """ This function checks whether the `data_type` is a union type which includes sparse union and dense union types. @@ -538,7 +538,7 @@ def is_union_type(data_type): return is_union(data_type.id) -def bit_width_type(data_type): +def _bit_width(data_type): """ Determine the bit width of the data type. @@ -560,7 +560,7 @@ def bit_width_type(data_type): return bit_width(data_type.id) -def offset_bit_width_type(data_type): +def _offset_bit_width(data_type): """ Determine the offset bit width of the data type. diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index 018099ae7e65..c5bf4a8a4224 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -142,11 +142,6 @@ cdef void* _as_c_pointer(v, allow_null=False) except *: return c_ptr -def _is_primitive(Type type): - # This is simply a redirect, the official API is in pyarrow.types. - return is_primitive(type) - - def _get_pandas_type(arrow_type, coerce_to_ns=False): cdef Type type_id = arrow_type.id if type_id not in _pandas_type_map: diff --git a/python/pyarrow/types.py b/python/pyarrow/types.py index 96ca27f4ad0b..d30d63a1f6fb 100644 --- a/python/pyarrow/types.py +++ b/python/pyarrow/types.py @@ -21,35 +21,35 @@ from pyarrow.lib import (is_boolean_value, # noqa is_integer_value, is_float_value, - is_integer_type, - is_signed_integer_type, - is_unsigned_integer_type, - is_floating_type, - is_numeric_type, - is_decimal_type, - is_run_end_type_py, - is_primitive_type, - is_base_binary_like_type, - is_binary_like_type, - is_large_binary_like_type, - is_binary_type, - is_string_type, - is_temporal_type, - is_time_type, - is_date_type, - is_interval_type, - is_dictionary_type, - is_fixed_size_binary_type, - is_fixed_width_type, - is_var_length_list_type, - is_list_type, - is_list_like_type, - is_var_length_list_like_type, - is_list_view_type, - is_nested_type, - is_union_type, - bit_width_type, - offset_bit_width_type) + _is_integer, + _is_signed_integer, + _is_unsigned_integer, + _is_floating, + _is_numeric, + _is_decimal, + _is_run_end_type, + _is_primitive, + _is_base_binary_like, + _is_binary_like, + _is_large_binary_like, + _is_binary, + _is_string, + _is_temporal, + _is_time, + _is_date, + _is_interval, + _is_dictionary, + _is_fixed_size_binary, + _is_fixed_width, + _is_var_length_list, + _is_list, + _is_list_like, + _is_var_length_list_like, + _is_list_view, + _is_nested, + _is_union, + _bit_width, + _offset_bit_width) import pyarrow.lib as lib from pyarrow.util import doc @@ -69,92 +69,92 @@ def is_null(t): @doc(is_null, datatype="boolean") def is_boolean(t): - return is_primitive_type(t) and bit_width_type(t) == 1 + return _is_primitive(t) and _bit_width(t) == 1 @doc(is_null, datatype="any integer") def is_integer(t): - return is_integer_type(t) and bit_width_type(t) + return _is_integer(t) and _bit_width(t) @doc(is_null, datatype="signed integer") def is_signed_integer(t): - return is_signed_integer_type(t) + return _is_signed_integer(t) @doc(is_null, datatype="unsigned integer") def is_unsigned_integer(t): - return is_unsigned_integer_type(t) + return _is_unsigned_integer(t) @doc(is_null, datatype="int8") def is_int8(t): - return is_integer_type(t) and bit_width_type(t) == 8 + return _is_integer(t) and _bit_width(t) == 8 @doc(is_null, datatype="int16") def is_int16(t): - return is_integer_type(t) and bit_width_type(t) == 16 + return _is_integer(t) and _bit_width(t) == 16 @doc(is_null, datatype="int32") def is_int32(t): - return is_integer_type(t) and bit_width_type(t) == 32 + return _is_integer(t) and _bit_width(t) == 32 @doc(is_null, datatype="int64") def is_int64(t): - return is_integer_type(t) and bit_width_type(t) == 64 + return _is_integer(t) and _bit_width(t) == 64 @doc(is_null, datatype="uint8") def is_uint8(t): - return is_unsigned_integer_type(t) and bit_width_type(t) == 8 + return _is_unsigned_integer(t) and _bit_width(t) == 8 @doc(is_null, datatype="uint16") def is_uint16(t): - return is_unsigned_integer_type(t) and bit_width_type(t) == 16 + return _is_unsigned_integer(t) and _bit_width(t) == 16 @doc(is_null, datatype="uint32") def is_uint32(t): - return is_unsigned_integer_type(t) and bit_width_type(t) == 32 + return _is_unsigned_integer(t) and _bit_width(t) == 32 @doc(is_null, datatype="uint64") def is_uint64(t): - return is_unsigned_integer_type(t) and bit_width_type(t) == 64 + return _is_unsigned_integer(t) and _bit_width(t) == 64 @doc(is_null, datatype="floating point numeric") def is_floating(t): - return is_floating_type(t) + return _is_floating(t) @doc(is_null, datatype="float16 (half-precision)") def is_float16(t): - return is_floating_type(t) and bit_width_type(t) == 16 + return _is_floating(t) and _bit_width(t) == 16 @doc(is_null, datatype="float32 (single precision)") def is_float32(t): - return is_floating_type(t) and bit_width_type(t) == 32 + return _is_floating(t) and _bit_width(t) == 32 @doc(is_null, datatype="float64 (double precision)") def is_float64(t): - return is_floating_type(t) and bit_width_type(t) == 64 + return _is_floating(t) and _bit_width(t) == 64 @doc(is_null, datatype="list") def is_list(t): - return is_list_type(t) and offset_bit_width_type(t) == 32 + return _is_list(t) and _offset_bit_width(t) == 32 @doc(is_null, datatype="large list") def is_large_list(t): - return is_list_type(t) and offset_bit_width_type(t) == 64 + return _is_list(t) and _offset_bit_width(t) == 64 @doc(is_null, datatype="fixed size list") @@ -164,12 +164,12 @@ def is_fixed_size_list(t): @doc(is_null, datatype="list view") def is_list_view(t): - return is_list_view_type(t) and offset_bit_width_type(t) == 32 + return _is_list_view(t) and _offset_bit_width(t) == 32 @doc(is_null, datatype="large list view") def is_large_list_view(t): - return is_list_view_type(t) and offset_bit_width_type(t) == 64 + return _is_list_view(t) and _offset_bit_width(t) == 64 @doc(is_null, datatype="struct") @@ -179,12 +179,12 @@ def is_struct(t): @doc(is_null, datatype="union") def is_union(t): - return is_union_type(t) + return _is_union(t) @doc(is_null, datatype="nested type") def is_nested(t): - return is_nested_type(t) + return _is_nested(t) @doc(is_null, datatype="run-end encoded") @@ -194,48 +194,48 @@ def is_run_end_encoded(t): @doc(is_null, datatype="date, time, timestamp or duration") def is_temporal(t): - return is_primitive_type(t) and not is_integer_type(t) and \ - not is_floating_type(t) + return _is_primitive(t) and not _is_integer(t) and \ + not _is_floating(t) @doc(is_null, datatype="timestamp") def is_timestamp(t): - return is_temporal_type(t) and not is_time_type(t) and not is_date_type(t) + return _is_temporal(t) and not _is_time(t) and not _is_date(t) @doc(is_null, datatype="duration") def is_duration(t): - return is_primitive_type(t) and not is_integer_type(t) and \ - not is_floating_type(t) and not is_temporal_type(t) and \ - not is_interval_type(t) + return _is_primitive(t) and not _is_integer(t) and \ + not _is_floating(t) and not _is_temporal(t) and \ + not _is_interval(t) @doc(is_null, datatype="time") def is_time(t): - return is_time_type(t) + return _is_time(t) @doc(is_null, datatype="time32") def is_time32(t): - return is_time_type(t) and bit_width_type(t) == 32 + return _is_time(t) and _bit_width(t) == 32 @doc(is_null, datatype="time64") def is_time64(t): - return is_time_type(t) and bit_width_type(t) == 64 + return _is_time(t) and _bit_width(t) == 64 @doc(is_null, datatype="variable-length binary") def is_binary(t): - return is_binary_type(t) and offset_bit_width_type(t) == 32 + return _is_binary(t) and _offset_bit_width(t) == 32 @doc(is_null, datatype="large variable-length binary") def is_large_binary(t): - return is_binary_type(t) and offset_bit_width_type(t) == 64 + return _is_binary(t) and _offset_bit_width(t) == 64 -@doc(method="is_string") +@doc(method="_is_string") def is_unicode(t): """ Alias for {method}. @@ -244,27 +244,27 @@ def is_unicode(t): ---------- t : DataType """ - return is_string(t) + return _is_string(t) @doc(is_null, datatype="string (utf8 unicode)") def is_string(t): - return is_string_type(t) and offset_bit_width_type(t) == 32 + return _is_string(t) and _offset_bit_width(t) == 32 -@doc(is_unicode, method="is_large_string") +@doc(is_unicode, method="_is_large_string") def is_large_unicode(t): return is_large_string(t) @doc(is_null, datatype="large string (utf8 unicode)") def is_large_string(t): - return is_string_type(t) and offset_bit_width_type(t) == 64 + return _is_string(t) and _offset_bit_width(t) == 64 @doc(is_null, datatype="fixed size binary") def is_fixed_size_binary(t): - return is_fixed_size_binary_type(t) + return _is_fixed_size_binary(t) @doc(is_null, datatype="variable-length binary view") @@ -279,49 +279,49 @@ def is_string_view(t): @doc(is_null, datatype="date") def is_date(t): - return is_date_type(t) + return _is_date(t) @doc(is_null, datatype="date32 (days)") def is_date32(t): - return is_date_type(t) and bit_width_type(t) == 32 + return _is_date(t) and _bit_width(t) == 32 @doc(is_null, datatype="date64 (milliseconds)") def is_date64(t): - return is_date_type(t) and bit_width_type(t) == 64 + return _is_date(t) and _bit_width(t) == 64 @doc(is_null, datatype="map") def is_map(t): - return is_var_length_list_type(t) and not is_list_type(t) + return _is_var_length_list(t) and not _is_list(t) @doc(is_null, datatype="decimal") def is_decimal(t): - return is_decimal_type(t) + return _is_decimal(t) @doc(is_null, datatype="decimal128") def is_decimal128(t): - return is_decimal_type(t) and bit_width_type(t) == 128 + return _is_decimal(t) and _bit_width(t) == 128 @doc(is_null, datatype="decimal256") def is_decimal256(t): - return is_decimal_type(t) and bit_width_type(t) == 256 + return _is_decimal(t) and _bit_width(t) == 256 @doc(is_null, datatype="dictionary-encoded") def is_dictionary(t): - return is_dictionary_type(t) + return _is_dictionary(t) @doc(is_null, datatype="interval") def is_interval(t): - return is_interval_type(t) + return _is_interval(t) @doc(is_null, datatype="primitive type") def is_primitive(t): - return is_primitive_type(t) + return _is_primitive(t) From 974df0cc5ddbc9f1d25f3a48c2161bf8fa9e962c Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Fri, 5 Apr 2024 04:58:55 +0900 Subject: [PATCH 17/28] Update python/pyarrow/type_traits.pxi Co-authored-by: Alenka Frim --- python/pyarrow/type_traits.pxi | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pyarrow/type_traits.pxi b/python/pyarrow/type_traits.pxi index 2952261194b0..5e03492656e2 100644 --- a/python/pyarrow/type_traits.pxi +++ b/python/pyarrow/type_traits.pxi @@ -45,6 +45,7 @@ from pyarrow.includes.libarrow cimport (is_integer, bit_width, offset_bit_width) +# These functions are simply a redirect, the official APIs are in pyarrow.types. def _is_integer(data_type): """ From 30adfec2aa1dc700c2d3e9aba30dd4d7e3f7eace Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Fri, 5 Apr 2024 05:34:40 +0900 Subject: [PATCH 18/28] apply lint --- python/pyarrow/type_traits.pxi | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/type_traits.pxi b/python/pyarrow/type_traits.pxi index 5e03492656e2..3e54a62745d1 100644 --- a/python/pyarrow/type_traits.pxi +++ b/python/pyarrow/type_traits.pxi @@ -45,7 +45,8 @@ from pyarrow.includes.libarrow cimport (is_integer, bit_width, offset_bit_width) -# These functions are simply a redirect, the official APIs are in pyarrow.types. +# These functions are simply a redirect, the official APIs are in pyarrow.types. + def _is_integer(data_type): """ From 8f08f46cda19bfd516a147edb7694eddb0de9c60 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Fri, 5 Apr 2024 18:54:43 +0900 Subject: [PATCH 19/28] delete type_traits.pxi --- python/pyarrow/lib.pyx | 2 - python/pyarrow/type_traits.pxi | 585 --------------------------------- python/pyarrow/types.pxi | 537 ++++++++++++++++++++++++++++++ 3 files changed, 537 insertions(+), 587 deletions(-) delete mode 100644 python/pyarrow/type_traits.pxi diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx index 277d5df4cf62..1c84d0b38b5e 100644 --- a/python/pyarrow/lib.pyx +++ b/python/pyarrow/lib.pyx @@ -162,8 +162,6 @@ include "pandas-shim.pxi" # Memory pools and allocation include "memory.pxi" -# TypeTraits -include "type_traits.pxi" # DataType, Field, Schema include "types.pxi" diff --git a/python/pyarrow/type_traits.pxi b/python/pyarrow/type_traits.pxi deleted file mode 100644 index 3e54a62745d1..000000000000 --- a/python/pyarrow/type_traits.pxi +++ /dev/null @@ -1,585 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from pyarrow.includes.libarrow cimport (is_integer, - is_signed_integer, - is_unsigned_integer, - is_floating, - is_numeric, - is_decimal, - is_run_end_type, - is_primitive, - is_base_binary_like, - is_binary_like, - is_large_binary_like, - is_binary, - is_string, - is_temporal, - is_time, - is_date, - is_interval, - is_dictionary, - is_fixed_size_binary, - is_fixed_width, - is_var_length_list, - is_list, - is_list_like, - is_var_length_list_like, - is_list_view, - is_nested, - is_union, - bit_width, - offset_bit_width) - -# These functions are simply a redirect, the official APIs are in pyarrow.types. - - -def _is_integer(data_type): - """ - This function checks whether the `data_type` is an integer type, which - includes signed and unsigned integers of various bit widths (8, 16, 32, 64 bits). - - Parameters - ---------- - data_type : DataType - The data type to check against the set of supported integer types. - - Returns - ------- - bool - True if `data_type` is an integer type, False otherwise. - """ - return is_integer(data_type.id) - - -def _is_signed_integer(data_type): - """ - This function checks whether the `data_type` is a signed integer type, - which includes signed integers of various bit widths (8, 16, 32, 64 bits). - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a signed integer type, False otherwise. - """ - return is_signed_integer(data_type.id) - - -def _is_unsigned_integer(data_type): - """ - This function checks whether the `data_type` is an unsigned integer type, - which includes unsigned integers of various bit widths (8, 16, 32, 64 bits). - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is an unsigned integer type, False otherwise. - """ - return is_unsigned_integer(data_type.id) - - -def _is_floating(data_type): - """ - This function checks whether the `data_type` is a floating type, which includes - floating point numbers of various bit widths (16, 32, 64 bits). - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a floating type, False otherwise. - """ - return is_floating(data_type.id) - - -def _is_numeric(data_type): - """ - This function checks whether the `data_type` is a numeric type, which includes - integers and floating point numbers with specific bit widths. Integer types - include signed and unsigned integers of various bit widths (8, 16, 32, 64 bits), - while floating point types include floating point numbers of various bit widths - (16, 32, 64 bits). - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a numeric type, False otherwise. - """ - return is_numeric(data_type.id) - - -def _is_decimal(data_type): - """ - This function checks whether the `data_type` is a decimal type, which includes - fixed-point decimal numbers with specific precision and scale. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a decimal type, False otherwise. - """ - return is_decimal(data_type.id) - - -def _is_run_end_type(data_type): - """ - This function checks whether the `data_type` is a run end type, which includes - integers of various bit widths (16, 32, 64 bits). - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a run end type, False otherwise. - """ - return is_run_end_type(data_type.id) - - -def _is_primitive(data_type): - """ - This function checks whether the `data_type` is a primitive type, which includes - boolean, integers, floating point numbers, dates, times, timestamp, duration - and intervals types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a primitive type, False otherwise. - """ - return is_primitive(data_type.id) - - -def _is_base_binary_like(data_type): - """ - This function checks whether the `data_type` is a base binary-like type, which - includes binary, string, large binary, and large string types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a base binary-like type, False otherwise. - """ - return is_base_binary_like(data_type.id) - - -def _is_binary_like(data_type): - """ - This function checks whether the `data_type` is a binary-like type, which includes - binary and string types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a binary-like type, False otherwise. - """ - return is_binary_like(data_type.id) - - -def _is_large_binary_like(data_type): - """ - This function checks whether the `data_type` is a large binary-like type, which - includes large binary and large string types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a large binary-like type, False otherwise. - """ - return is_large_binary_like(data_type.id) - - -def _is_binary(data_type): - """ - This function checks whether the `data_type` is a binary type, which includes - binary and large binary types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a binary type, False otherwise. - """ - return is_binary(data_type.id) - - -def _is_string(data_type): - """ - This function checks whether the `data_type` is a string type, which includes - string and large string types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a string type, False otherwise. - """ - return is_string(data_type.id) - - -def _is_temporal(data_type): - """ - This function checks whether the `data_type` is a temporal type, which includes - dates (days since the UNIX epoch and milliseconds since the UNIX epoch), times - (seconds and milliseconds since midnight), timestamp (milliseconds since the UNIX - epoch) types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a temporal type, False otherwise. - """ - return is_temporal(data_type.id) - - -def _is_time(data_type): - """ - This function checks whether the `data_type` is a time type, which includes - times (seconds and milliseconds since midnight) types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a time type, False otherwise. - """ - return is_time(data_type.id) - - -def _is_date(data_type): - """ - This function checks whether the `data_type` is a date type, which includes - dates (days since the UNIX epoch and milliseconds since the UNIX epoch) types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a date type, False otherwise. - """ - return is_date(data_type.id) - - -def _is_interval(data_type): - """ - This function checks whether the `data_type` is a interval type, which includes - intervals (months, days plus nanoseconds, and day-time intervals) types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a interval type, False otherwise. - """ - return is_interval(data_type.id) - - -def _is_dictionary(data_type): - """ - This function checks whether the `data_type` is a dictionary type. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a dictionary type, False otherwise. - """ - return is_dictionary(data_type.id) - - -def _is_fixed_size_binary(data_type): - """ - This function checks whether the `data_type` is a fixed size binary type - which includes decimal and fixed size binary types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a fixed size binary type, False otherwise. - """ - return is_fixed_size_binary(data_type.id) - - -def _is_fixed_width(data_type): - """ - This function checks whether the `data_type` is a fixed width type which includes - is_primitive, is_dictionary, is_fixed_size_binary types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a fixed width type, False otherwise. - """ - return is_fixed_width(data_type.id) - - -def _is_var_length_list(data_type): - """ - This function checks whether the `data_type` is a variable length list type - which includes list, large list, and map types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a variable length list type, False otherwise. - """ - return is_var_length_list(data_type.id) - - -def _is_list(data_type): - """ - This function checks whether the `data_type` is a list type which includes - list, large list, fixed size list types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a list type, False otherwise. - """ - return is_list(data_type.id) - - -def _is_list_like(data_type): - """ - This function checks whether the `data_type` is a list_like type which includes - list, large list, fixed size list, and map types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a list_like type, False otherwise. - """ - return is_list_like(data_type.id) - - -def _is_var_length_list_like(data_type): - """ - This function checks whether the `data_type` is a variable length list like - type which includes list, large list, list view, large list view, and map types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a variable length list like type, False otherwise. - """ - return is_var_length_list_like(data_type.id) - - -def _is_list_view(data_type): - """ - This function checks whether the `data_type` is a list view type which includes - list view and large list view types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a list view type, False otherwise. - """ - return is_list_view(data_type.id) - - -def _is_nested(data_type): - """ - This function checks whether the `data_type` is a nested type which includes - list, large list, list view, large list view, fixed size list, map, struct, - sparse union, dense union, and run end encoded types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a nested type, False otherwise. - """ - return is_nested(data_type.id) - - -def _is_union(data_type): - """ - This function checks whether the `data_type` is a union type which includes - sparse union and dense union types. - - Parameters - ---------- - data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a union type, False otherwise. - """ - return is_union(data_type.id) - - -def _bit_width(data_type): - """ - Determine the bit width of the data type. - - This function evaluates the `data_type` and returns its bit width, which is - essential for understanding the storage size and precision of data types. - It supports a wide range of data types, including boolean, integers, floating - point numbers, date, time, timestamp, duration, intervals, and decimal types. - - Parameters - ---------- - data_type : DataType - The data type for which to determine the bit width. - - Returns - ------- - int - The bit width of the `data_type`, or 0 if the bit width is not applicable. - """ - return bit_width(data_type.id) - - -def _offset_bit_width(data_type): - """ - Determine the offset bit width of the data type. - - This function evaluates the `data_type` and returns its offset bit width, which is - essential for understanding the storage size and precision of data types. - It supports a wide range of data types, including string, binary, list, list - view, map, dense union, large string, large binary, large list, and large list view - types. - - Parameters - ---------- - data_type : DataType - The data type for which to determine the offset bit width. - - Returns - ------- - int - The offset bit width of the `data_type`, or 0 if the offset bit width is not applicable. - """ - return offset_bit_width(data_type.id) diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index c5bf4a8a4224..0471b46421b6 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -5412,6 +5412,543 @@ def is_float_value(object obj): return IsPyFloat(obj) +def _is_integer(data_type): + """ + This function checks whether the `data_type` is an integer type, which + includes signed and unsigned integers of various bit widths (8, 16, 32, 64 bits). + + Parameters + ---------- + data_type : DataType + The data type to check against the set of supported integer types. + + Returns + ------- + bool + True if `data_type` is an integer type, False otherwise. + """ + return is_integer(data_type.id) + + +def _is_signed_integer(data_type): + """ + This function checks whether the `data_type` is a signed integer type, + which includes signed integers of various bit widths (8, 16, 32, 64 bits). + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a signed integer type, False otherwise. + """ + return is_signed_integer(data_type.id) + + +def _is_unsigned_integer(data_type): + """ + This function checks whether the `data_type` is an unsigned integer type, + which includes unsigned integers of various bit widths (8, 16, 32, 64 bits). + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is an unsigned integer type, False otherwise. + """ + return is_unsigned_integer(data_type.id) + + +def _is_floating(data_type): + """ + This function checks whether the `data_type` is a floating type, which includes + floating point numbers of various bit widths (16, 32, 64 bits). + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a floating type, False otherwise. + """ + return is_floating(data_type.id) + + +def _is_numeric(data_type): + """ + This function checks whether the `data_type` is a numeric type, which includes + integers and floating point numbers with specific bit widths. Integer types + include signed and unsigned integers of various bit widths (8, 16, 32, 64 bits), + while floating point types include floating point numbers of various bit widths + (16, 32, 64 bits). + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a numeric type, False otherwise. + """ + return is_numeric(data_type.id) + + +def _is_decimal(data_type): + """ + This function checks whether the `data_type` is a decimal type, which includes + fixed-point decimal numbers with specific precision and scale. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a decimal type, False otherwise. + """ + return is_decimal(data_type.id) + + +def _is_run_end_type(data_type): + """ + This function checks whether the `data_type` is a run end type, which includes + integers of various bit widths (16, 32, 64 bits). + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a run end type, False otherwise. + """ + return is_run_end_type(data_type.id) + + +def _is_primitive(data_type): + """ + This function checks whether the `data_type` is a primitive type, which includes + boolean, integers, floating point numbers, dates, times, timestamp, duration + and intervals types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a primitive type, False otherwise. + """ + return is_primitive(data_type.id) + + +def _is_base_binary_like(data_type): + """ + This function checks whether the `data_type` is a base binary-like type, which + includes binary, string, large binary, and large string types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a base binary-like type, False otherwise. + """ + return is_base_binary_like(data_type.id) + + +def _is_binary_like(data_type): + """ + This function checks whether the `data_type` is a binary-like type, which includes + binary and string types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a binary-like type, False otherwise. + """ + return is_binary_like(data_type.id) + + +def _is_large_binary_like(data_type): + """ + This function checks whether the `data_type` is a large binary-like type, which + includes large binary and large string types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a large binary-like type, False otherwise. + """ + return is_large_binary_like(data_type.id) + + +def _is_binary(data_type): + """ + This function checks whether the `data_type` is a binary type, which includes + binary and large binary types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a binary type, False otherwise. + """ + return is_binary(data_type.id) + + +def _is_string(data_type): + """ + This function checks whether the `data_type` is a string type, which includes + string and large string types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a string type, False otherwise. + """ + return is_string(data_type.id) + + +def _is_temporal(data_type): + """ + This function checks whether the `data_type` is a temporal type, which includes + dates (days since the UNIX epoch and milliseconds since the UNIX epoch), times + (seconds and milliseconds since midnight), timestamp (milliseconds since the UNIX + epoch) types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a temporal type, False otherwise. + """ + return is_temporal(data_type.id) + + +def _is_time(data_type): + """ + This function checks whether the `data_type` is a time type, which includes + times (seconds and milliseconds since midnight) types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a time type, False otherwise. + """ + return is_time(data_type.id) + + +def _is_date(data_type): + """ + This function checks whether the `data_type` is a date type, which includes + dates (days since the UNIX epoch and milliseconds since the UNIX epoch) types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a date type, False otherwise. + """ + return is_date(data_type.id) + + +def _is_interval(data_type): + """ + This function checks whether the `data_type` is a interval type, which includes + intervals (months, days plus nanoseconds, and day-time intervals) types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a interval type, False otherwise. + """ + return is_interval(data_type.id) + + +def _is_dictionary(data_type): + """ + This function checks whether the `data_type` is a dictionary type. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a dictionary type, False otherwise. + """ + return is_dictionary(data_type.id) + + +def _is_fixed_size_binary(data_type): + """ + This function checks whether the `data_type` is a fixed size binary type + which includes decimal and fixed size binary types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a fixed size binary type, False otherwise. + """ + return is_fixed_size_binary(data_type.id) + + +def _is_fixed_width(data_type): + """ + This function checks whether the `data_type` is a fixed width type which includes + is_primitive, is_dictionary, is_fixed_size_binary types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a fixed width type, False otherwise. + """ + return is_fixed_width(data_type.id) + + +def _is_var_length_list(data_type): + """ + This function checks whether the `data_type` is a variable length list type + which includes list, large list, and map types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a variable length list type, False otherwise. + """ + return is_var_length_list(data_type.id) + + +def _is_list(data_type): + """ + This function checks whether the `data_type` is a list type which includes + list, large list, fixed size list types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a list type, False otherwise. + """ + return is_list(data_type.id) + + +def _is_list_like(data_type): + """ + This function checks whether the `data_type` is a list_like type which includes + list, large list, fixed size list, and map types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a list_like type, False otherwise. + """ + return is_list_like(data_type.id) + + +def _is_var_length_list_like(data_type): + """ + This function checks whether the `data_type` is a variable length list like + type which includes list, large list, list view, large list view, and map types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a variable length list like type, False otherwise. + """ + return is_var_length_list_like(data_type.id) + + +def _is_list_view(data_type): + """ + This function checks whether the `data_type` is a list view type which includes + list view and large list view types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a list view type, False otherwise. + """ + return is_list_view(data_type.id) + + +def _is_nested(data_type): + """ + This function checks whether the `data_type` is a nested type which includes + list, large list, list view, large list view, fixed size list, map, struct, + sparse union, dense union, and run end encoded types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a nested type, False otherwise. + """ + return is_nested(data_type.id) + + +def _is_union(data_type): + """ + This function checks whether the `data_type` is a union type which includes + sparse union and dense union types. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a union type, False otherwise. + """ + return is_union(data_type.id) + + +def _bit_width(data_type): + """ + Determine the bit width of the data type. + + This function evaluates the `data_type` and returns its bit width, which is + essential for understanding the storage size and precision of data types. + It supports a wide range of data types, including boolean, integers, floating + point numbers, date, time, timestamp, duration, intervals, and decimal types. + + Parameters + ---------- + data_type : DataType + The data type for which to determine the bit width. + + Returns + ------- + int + The bit width of the `data_type`, or 0 if the bit width is not applicable. + """ + return bit_width(data_type.id) + + +def _offset_bit_width(data_type): + """ + Determine the offset bit width of the data type. + + This function evaluates the `data_type` and returns its offset bit width, which is + essential for understanding the storage size and precision of data types. + It supports a wide range of data types, including string, binary, list, list + view, map, dense union, large string, large binary, large list, and large list view + types. + + Parameters + ---------- + data_type : DataType + The data type for which to determine the offset bit width. + + Returns + ------- + int + The offset bit width of the `data_type`, or 0 if the offset bit width is not applicable. + """ + return offset_bit_width(data_type.id) + + cdef class _ExtensionRegistryNanny(_Weakrefable): # Keep the registry alive until we have unregistered PyExtensionType cdef: From 153de4bebfac87ff613ecdfbbedccc7852a11503 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Sat, 6 Apr 2024 12:43:35 +0900 Subject: [PATCH 20/28] migrate functions to types.pxi and expose through types.py --- python/pyarrow/__init__.py | 49 +++ python/pyarrow/compute.py | 2 +- python/pyarrow/tests/test_types.py | 249 ++++++------ python/pyarrow/types.pxi | 602 +++++++++++++++++++++++++++-- python/pyarrow/types.py | 354 +++-------------- 5 files changed, 788 insertions(+), 468 deletions(-) diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py index 936f4736977c..4fef2ba4dbc4 100644 --- a/python/pyarrow/__init__.py +++ b/python/pyarrow/__init__.py @@ -184,6 +184,55 @@ def print_entry(label, value): BaseExtensionType, ExtensionType, RunEndEncodedType, FixedShapeTensorType, PyExtensionType, UnknownExtensionType, + # _is_integer as is_integer, + # _is_signed_integer as is_signed_integer, + # _is_unsigned_integer as is_unsigned_integer, + # _is_floating as is_floating, + # _is_numeric as is_numeric, + # _is_decimal as is_decimal, + # _is_run_end_type as is_run_end_type, + # _is_primitive as is_primitive, + # _is_base_binary_like as is_base_binary_like, + # _is_binary_like as is_binary_like, + # _is_large_binary_like as is_large_binary_like, + # _is_binary as is_binary, + # _is_string as is_string, _is_string as is_unicode, + # _is_temporal as is_temporal, + # _is_time as is_time, + # _is_date as is_date, + # _is_interval as is_interval, + # _is_dictionary as is_dictionary, + # _is_fixed_size_binary as is_fixed_size_binary, + # _is_fixed_width as is_fixed_width, + # _is_var_length_list as is_var_length_list, + # _is_list as is_list, + # _is_list_like as is_list_like, + # _is_var_length_list_like as is_var_length_list_like, + # _is_list_view as is_list_view, + # _is_large_list_view as is_large_list_view, + # _is_nested as is_nested, + # _is_union as is_union, + # _bit_width as bit_width, + # _offset_bit_width as offset_bit_width, + # is_run_end_encoded, + # is_null, + # is_boolean, + # is_int8, is_int16, is_int32, is_int64, + # is_uint8, is_uint16, is_uint32, is_uint64, + # is_float16, is_float32, is_float64, + # is_large_list, + # is_fixed_size_list, + # is_struct, + # is_timestamp, + # is_duration, + # is_time32, is_time64, + # is_large_binary, + # is_large_string, is_large_string as is_large_unicode, + # is_binary_view, + # is_string_view, + # is_date32, is_date64, + # is_map, + # is_decimal128, is_decimal256, register_extension_type, unregister_extension_type, DictionaryMemo, KeyValueMetadata, diff --git a/python/pyarrow/compute.py b/python/pyarrow/compute.py index 83612f66d21e..b8cf5dbfdef6 100644 --- a/python/pyarrow/compute.py +++ b/python/pyarrow/compute.py @@ -397,7 +397,7 @@ def cast(arr, target_type=None, safe=None, options=None, memory_pool=None): " or pass a value for 'options'") if options is None: - target_type = pa.types.lib.ensure_type(target_type) + target_type = pa.types.ensure_type(target_type) if safe is False: options = CastOptions.unsafe(target_type) else: diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py index 75cf54be5b1c..fa66e4a28596 100644 --- a/python/pyarrow/tests/test_types.py +++ b/python/pyarrow/tests/test_types.py @@ -33,7 +33,6 @@ import numpy as np import pyarrow as pa -import pyarrow.types as types import pyarrow.tests.strategies as past @@ -95,8 +94,8 @@ def get_many_types(): def test_is_boolean(): - assert types.is_boolean(pa.bool_()) - assert not types.is_boolean(pa.int8()) + assert pa.types.is_boolean(pa.bool_()) + assert not pa.types.is_boolean(pa.int8()) def test_is_integer(): @@ -104,30 +103,30 @@ def test_is_integer(): unsigned_ints = [pa.uint8(), pa.uint16(), pa.uint32(), pa.uint64()] for t in signed_ints + unsigned_ints: - assert types.is_integer(t) + assert pa.types.is_integer(t) for t in signed_ints: - assert types.is_signed_integer(t) - assert not types.is_unsigned_integer(t) + assert pa.types.is_signed_integer(t) + assert not pa.types.is_unsigned_integer(t) for t in unsigned_ints: - assert types.is_unsigned_integer(t) - assert not types.is_signed_integer(t) + assert pa.types.is_unsigned_integer(t) + assert not pa.types.is_signed_integer(t) - assert not types.is_integer(pa.float32()) - assert not types.is_signed_integer(pa.float32()) + assert not pa.types.is_integer(pa.float32()) + assert not pa.types.is_signed_integer(pa.float32()) def test_is_floating(): for t in [pa.float16(), pa.float32(), pa.float64()]: - assert types.is_floating(t) + assert pa.types.is_floating(t) - assert not types.is_floating(pa.int32()) + assert not pa.types.is_floating(pa.int32()) def test_is_null(): - assert types.is_null(pa.null()) - assert not types.is_null(pa.list_(pa.int32())) + assert pa.types.is_null(pa.null()) + assert not pa.types.is_null(pa.list_(pa.int32())) def test_null_field_may_not_be_non_nullable(): @@ -141,17 +140,17 @@ def test_is_decimal(): decimal256 = pa.decimal256(76, 38) int32 = pa.int32() - assert types.is_decimal(decimal128) - assert types.is_decimal(decimal256) - assert not types.is_decimal(int32) + assert pa.types.is_decimal(decimal128) + assert pa.types.is_decimal(decimal256) + assert not pa.types.is_decimal(int32) - assert types.is_decimal128(decimal128) - assert not types.is_decimal128(decimal256) - assert not types.is_decimal128(int32) + assert pa.types.is_decimal128(decimal128) + assert not pa.types.is_decimal128(decimal256) + assert not pa.types.is_decimal128(int32) - assert not types.is_decimal256(decimal128) - assert types.is_decimal256(decimal256) - assert not types.is_decimal256(int32) + assert not pa.types.is_decimal256(decimal128) + assert pa.types.is_decimal256(decimal256) + assert not pa.types.is_decimal256(int32) def test_is_list(): @@ -159,50 +158,50 @@ def test_is_list(): b = pa.large_list(pa.int32()) c = pa.list_(pa.int32(), 3) - assert types.is_list(a) - assert not types.is_large_list(a) - assert not types.is_fixed_size_list(a) - assert types.is_large_list(b) - assert not types.is_list(b) - assert not types.is_fixed_size_list(b) - assert types.is_fixed_size_list(c) - assert not types.is_list(c) - assert not types.is_large_list(c) + assert pa.types.is_list(a) + assert not pa.types.is_large_list(a) + assert not pa.types.is_fixed_size_list(a) + assert pa.types.is_large_list(b) + assert not pa.types.is_list(b) + assert not pa.types.is_fixed_size_list(b) + assert pa.types.is_fixed_size_list(c) + assert not pa.types.is_list(c) + assert not pa.types.is_large_list(c) - assert not types.is_list(pa.int32()) + assert not pa.types.is_list(pa.int32()) def test_is_list_view(): a = pa.list_view(pa.int32()) b = pa.large_list_view(pa.int32()) - assert types.is_list_view(a) - assert not types.is_large_list_view(a) - assert not types.is_list(a) - assert types.is_large_list_view(b) - assert not types.is_list_view(b) - assert not types.is_large_list(b) + assert pa.types.is_list_view(a) + assert not pa.types.is_large_list_view(a) + assert not pa.types.is_list(a) + assert pa.types.is_large_list_view(b) + assert not pa.types.is_list_view(b) + assert not pa.types.is_large_list(b) def test_is_map(): m = pa.map_(pa.utf8(), pa.int32()) - assert types.is_map(m) - assert not types.is_map(pa.int32()) + assert pa.types.is_map(m) + assert not pa.types.is_map(pa.int32()) fields = pa.map_(pa.field('key_name', pa.utf8(), nullable=False), pa.field('value_name', pa.int32())) - assert types.is_map(fields) + assert pa.types.is_map(fields) entries_type = pa.struct([pa.field('key', pa.int8()), pa.field('value', pa.int8())]) list_type = pa.list_(entries_type) - assert not types.is_map(list_type) + assert not pa.types.is_map(list_type) def test_is_dictionary(): - assert types.is_dictionary(pa.dictionary(pa.int32(), pa.string())) - assert not types.is_dictionary(pa.int32()) + assert pa.types.is_dictionary(pa.dictionary(pa.int32(), pa.string())) + assert not pa.types.is_dictionary(pa.int32()) def test_is_nested_or_struct(): @@ -210,65 +209,65 @@ def test_is_nested_or_struct(): pa.field('b', pa.int8()), pa.field('c', pa.string())]) - assert types.is_struct(struct_ex) - assert not types.is_struct(pa.list_(pa.int32())) + assert pa.types.is_struct(struct_ex) + assert not pa.types.is_struct(pa.list_(pa.int32())) - assert types.is_nested(struct_ex) - assert types.is_nested(pa.list_(pa.int32())) - assert types.is_nested(pa.list_(pa.int32(), 3)) - assert types.is_nested(pa.large_list(pa.int32())) - assert types.is_nested(pa.list_view(pa.int32())) - assert types.is_nested(pa.large_list_view(pa.int32())) - assert not types.is_nested(pa.int32()) + assert pa.types.is_nested(struct_ex) + assert pa.types.is_nested(pa.list_(pa.int32())) + assert pa.types.is_nested(pa.list_(pa.int32(), 3)) + assert pa.types.is_nested(pa.large_list(pa.int32())) + assert pa.types.is_nested(pa.list_view(pa.int32())) + assert pa.types.is_nested(pa.large_list_view(pa.int32())) + assert not pa.types.is_nested(pa.int32()) def test_is_union(): for mode in [pa.lib.UnionMode_SPARSE, pa.lib.UnionMode_DENSE]: - assert types.is_union(pa.union([pa.field('a', pa.int32()), + assert pa.types.is_union(pa.union([pa.field('a', pa.int32()), pa.field('b', pa.int8()), pa.field('c', pa.string())], mode=mode)) - assert not types.is_union(pa.list_(pa.int32())) + assert not pa.types.is_union(pa.list_(pa.int32())) def test_is_run_end_encoded(): - assert types.is_run_end_encoded(pa.run_end_encoded(pa.int32(), pa.int64())) - assert not types.is_run_end_encoded(pa.utf8()) + assert pa.types.is_run_end_encoded(pa.run_end_encoded(pa.int32(), pa.int64())) + assert not pa.types.is_run_end_encoded(pa.utf8()) # TODO(wesm): is_map, once implemented def test_is_binary_string(): - assert types.is_binary(pa.binary()) - assert not types.is_binary(pa.string()) - assert not types.is_binary(pa.large_binary()) - assert not types.is_binary(pa.large_string()) + assert pa.types.is_binary(pa.binary()) + assert not pa.types.is_binary(pa.string()) + assert not pa.types.is_binary(pa.large_binary()) + assert not pa.types.is_binary(pa.large_string()) - assert types.is_string(pa.string()) - assert types.is_unicode(pa.string()) - assert not types.is_string(pa.binary()) - assert not types.is_string(pa.large_string()) - assert not types.is_string(pa.large_binary()) + assert pa.types.is_string(pa.string()) + assert pa.types.is_unicode(pa.string()) + assert not pa.types.is_string(pa.binary()) + assert not pa.types.is_string(pa.large_string()) + assert not pa.types.is_string(pa.large_binary()) - assert types.is_large_binary(pa.large_binary()) - assert not types.is_large_binary(pa.large_string()) - assert not types.is_large_binary(pa.binary()) - assert not types.is_large_binary(pa.string()) + assert pa.types.is_large_binary(pa.large_binary()) + assert not pa.types.is_large_binary(pa.large_string()) + assert not pa.types.is_large_binary(pa.binary()) + assert not pa.types.is_large_binary(pa.string()) - assert types.is_large_string(pa.large_string()) - assert not types.is_large_string(pa.large_binary()) - assert not types.is_large_string(pa.string()) - assert not types.is_large_string(pa.binary()) + assert pa.types.is_large_string(pa.large_string()) + assert not pa.types.is_large_string(pa.large_binary()) + assert not pa.types.is_large_string(pa.string()) + assert not pa.types.is_large_string(pa.binary()) - assert types.is_fixed_size_binary(pa.binary(5)) - assert not types.is_fixed_size_binary(pa.binary()) + assert pa.types.is_fixed_size_binary(pa.binary(5)) + assert not pa.types.is_fixed_size_binary(pa.binary()) - assert types.is_string_view(pa.string_view()) - assert not types.is_string_view(pa.string()) - assert types.is_binary_view(pa.binary_view()) - assert not types.is_binary_view(pa.binary()) - assert not types.is_binary_view(pa.string_view()) + assert pa.types.is_string_view(pa.string_view()) + assert not pa.types.is_string_view(pa.string()) + assert pa.types.is_binary_view(pa.binary_view()) + assert not pa.types.is_binary_view(pa.binary()) + assert not pa.types.is_binary_view(pa.string_view()) def test_is_temporal_date_time_timestamp(): @@ -280,48 +279,48 @@ def test_is_temporal_date_time_timestamp(): for case in (date_types + time_types + timestamp_types + duration_types + interval_types): - assert types.is_temporal(case) + assert pa.types.is_temporal(case) for case in date_types: - assert types.is_date(case) - assert not types.is_time(case) - assert not types.is_timestamp(case) - assert not types.is_duration(case) - assert not types.is_interval(case) + assert pa.types.is_date(case) + assert not pa.types.is_time(case) + assert not pa.types.is_timestamp(case) + assert not pa.types.is_duration(case) + assert not pa.types.is_interval(case) for case in time_types: - assert types.is_time(case) - assert not types.is_date(case) - assert not types.is_timestamp(case) - assert not types.is_duration(case) - assert not types.is_interval(case) + assert pa.types.is_time(case) + assert not pa.types.is_date(case) + assert not pa.types.is_timestamp(case) + assert not pa.types.is_duration(case) + assert not pa.types.is_interval(case) for case in timestamp_types: - assert types.is_timestamp(case) - assert not types.is_date(case) - assert not types.is_time(case) - assert not types.is_duration(case) - assert not types.is_interval(case) + assert pa.types.is_timestamp(case) + assert not pa.types.is_date(case) + assert not pa.types.is_time(case) + assert not pa.types.is_duration(case) + assert not pa.types.is_interval(case) for case in duration_types: - assert types.is_duration(case) - assert not types.is_date(case) - assert not types.is_time(case) - assert not types.is_timestamp(case) - assert not types.is_interval(case) + assert pa.types.is_duration(case) + assert not pa.types.is_date(case) + assert not pa.types.is_time(case) + assert not pa.types.is_timestamp(case) + assert not pa.types.is_interval(case) for case in interval_types: - assert types.is_interval(case) - assert not types.is_date(case) - assert not types.is_time(case) - assert not types.is_timestamp(case) + assert pa.types.is_interval(case) + assert not pa.types.is_date(case) + assert not pa.types.is_time(case) + assert not pa.types.is_timestamp(case) - assert not types.is_temporal(pa.int32()) + assert not pa.types.is_temporal(pa.int32()) def test_is_primitive(): - assert types.is_primitive(pa.int32()) - assert not types.is_primitive(pa.list_(pa.int32())) + assert pa.types.is_primitive(pa.int32()) + assert not pa.types.is_primitive(pa.list_(pa.int32())) @pytest.mark.parametrize(('tz', 'expected'), [ @@ -925,21 +924,21 @@ def test_run_end_encoded_type(): @pytest.mark.parametrize('t,check_func', [ - (pa.date32(), types.is_date32), - (pa.date64(), types.is_date64), - (pa.time32('s'), types.is_time32), - (pa.time64('ns'), types.is_time64), - (pa.int8(), types.is_int8), - (pa.int16(), types.is_int16), - (pa.int32(), types.is_int32), - (pa.int64(), types.is_int64), - (pa.uint8(), types.is_uint8), - (pa.uint16(), types.is_uint16), - (pa.uint32(), types.is_uint32), - (pa.uint64(), types.is_uint64), - (pa.float16(), types.is_float16), - (pa.float32(), types.is_float32), - (pa.float64(), types.is_float64) + (pa.date32(), pa.types.is_date32), + (pa.date64(), pa.types.is_date64), + (pa.time32('s'), pa.types.is_time32), + (pa.time64('ns'), pa.types.is_time64), + (pa.int8(), pa.types.is_int8), + (pa.int16(), pa.types.is_int16), + (pa.int32(), pa.types.is_int32), + (pa.int64(), pa.types.is_int64), + (pa.uint8(), pa.types.is_uint8), + (pa.uint16(), pa.types.is_uint16), + (pa.uint32(), pa.types.is_uint32), + (pa.uint64(), pa.types.is_uint64), + (pa.float16(), pa.types.is_float16), + (pa.float32(), pa.types.is_float32), + (pa.float64(), pa.types.is_float64) ]) def test_exact_primitive_types(t, check_func): assert check_func(t) diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index 0471b46421b6..4699caf51a97 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -5629,13 +5629,12 @@ def _is_binary(data_type): bool True if `data_type` is a binary type, False otherwise. """ - return is_binary(data_type.id) + return is_binary(data_type.id) and offset_bit_width(data_type.id) == 32 def _is_string(data_type): """ - This function checks whether the `data_type` is a string type, which includes - string and large string types. + This function checks whether the `data_type` is a string type. Parameters ---------- @@ -5647,7 +5646,7 @@ def _is_string(data_type): bool True if `data_type` is a string type, False otherwise. """ - return is_string(data_type.id) + return is_string(data_type.id) and offset_bit_width(data_type.id) == 32 def _is_temporal(data_type): @@ -5667,7 +5666,8 @@ def _is_temporal(data_type): bool True if `data_type` is a temporal type, False otherwise. """ - return is_temporal(data_type.id) + return is_primitive(data_type.id) and not is_integer(data_type.id) and \ + not is_floating(data_type.id) def _is_time(data_type): @@ -5797,8 +5797,7 @@ def _is_var_length_list(data_type): def _is_list(data_type): """ - This function checks whether the `data_type` is a list type which includes - list, large list, fixed size list types. + This function checks whether the `data_type` is a list type. Parameters ---------- @@ -5810,7 +5809,7 @@ def _is_list(data_type): bool True if `data_type` is a list type, False otherwise. """ - return is_list(data_type.id) + return is_list(data_type.id) and offset_bit_width(data_type.id) == 32 def _is_list_like(data_type): @@ -5849,10 +5848,360 @@ def _is_var_length_list_like(data_type): return is_var_length_list_like(data_type.id) +def _bit_width(data_type): + """ + Determine the bit width of the data type. + + This function evaluates the `data_type` and returns its bit width, which is + essential for understanding the storage size and precision of data types. + It supports a wide range of data types, including boolean, integers, floating + point numbers, date, time, timestamp, duration, intervals, and decimal types. + + Parameters + ---------- + data_type : DataType + The data type for which to determine the bit width. + + Returns + ------- + int + The bit width of the `data_type`, or 0 if the bit width is not applicable. + """ + return bit_width(data_type.id) + + +def _offset_bit_width(data_type): + """ + Determine the offset bit width of the data type. + + This function evaluates the `data_type` and returns its offset bit width, which is + essential for understanding the storage size and precision of data types. + It supports a wide range of data types, including string, binary, list, list + view, map, dense union, large string, large binary, large list, and large list view + types. + + Parameters + ---------- + data_type : DataType + The data type for which to determine the offset bit width. + + Returns + ------- + int + The offset bit width of the `data_type`, or 0 if the offset bit width is not applicable. + """ + return offset_bit_width(data_type.id) + + +def is_null(data_type): + """ + This function checks whether the `data_type` is a null type. + + Parameters + ---------- + data_type : DataType + The data type to check against the null type. + + Returns + ------- + bool + True if `data_type` is a null type, False otherwise. + """ + return data_type.id == Type_NA + + +def is_boolean(data_type): + """ + This function checks whether the `data_type` is a boolean type. + + Parameters + ---------- + data_type : DataType + The data type to check against the boolean type. + + Returns + ------- + bool + True if `data_type` is a boolean type, False otherwise. + """ + return data_type.id == Type_BOOL + + +def is_int8(data_type): + """ + This function checks whether the `data_type` is an 8-bit integer. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is an 8-bit integer, False otherwise. + """ + return is_integer(data_type.id) and bit_width(data_type.id) == 8 + + +def is_int16(data_type): + """ + This function checks whether the `data_type` is an 16-bit integer. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is an 16-bit integer, False otherwise. + """ + return is_integer(data_type.id) and bit_width(data_type.id) == 16 + + +def is_int32(data_type): + """ + This function checks whether the `data_type` is an 32-bit integer. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is an 32-bit integer, False otherwise. + """ + return is_integer(data_type.id) and bit_width(data_type.id) == 32 + + +def is_int64(data_type): + """ + This function checks whether the `data_type` is an 64-bit integer. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is an 64-bit integer, False otherwise. + """ + return is_integer(data_type.id) and bit_width(data_type.id) == 64 + + +def is_uint8(data_type): + """ + This function checks whether the `data_type` is an 8-bit unsigned integer. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is an 8-bit unsigned integer, False otherwise. + """ + return is_unsigned_integer(data_type.id) and bit_width(data_type.id) == 8 + + +def is_uint16(data_type): + """ + This function checks whether the `data_type` is an 16-bit unsigned integer. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is an 16-bit unsigned integer, False otherwise. + """ + return is_unsigned_integer(data_type.id) and bit_width(data_type.id) == 16 + + +def is_uint32(data_type): + """ + This function checks whether the `data_type` is an 32-bit unsigned integer. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is an 32-bit unsigned integer, False otherwise. + """ + return is_unsigned_integer(data_type.id) and bit_width(data_type.id) == 32 + + +def is_uint64(data_type): + """ + This function checks whether the `data_type` is an 64-bit unsigned integer. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is an 64-bit unsigned integer, False otherwise. + """ + return is_unsigned_integer(data_type.id) and bit_width(data_type.id) == 64 + + +def is_float16(data_type): + """ + This function checks whether the `data_type` is a 16-bit floating point number. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a 16-bit floating point number, False otherwise. + """ + return is_floating(data_type.id) and bit_width(data_type.id) == 16 + + +def is_float32(data_type): + """ + This function checks whether the `data_type` is a 32-bit floating point number. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a 32-bit floating point number, False otherwise. + """ + return is_floating(data_type.id) and bit_width(data_type.id) == 32 + + +def is_float64(data_type): + """ + This function checks whether the `data_type` is a 64-bit floating point number. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a 64-bit floating point number, False otherwise. + """ + return is_floating(data_type.id) and bit_width(data_type.id) == 64 + + +def is_large_list(data_type): + """ + This function checks whether the `data_type` is a large list type. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a large list type, False otherwise. + """ + return is_list(data_type.id) and offset_bit_width(data_type.id) == 64 + + +def is_fixed_size_list(data_type): + """ + This function checks whether the `data_type` is a fixed size list type. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a fixed size list type, False otherwise. + """ + return data_type.id == Type_FIXED_SIZE_LIST + + def _is_list_view(data_type): """ - This function checks whether the `data_type` is a list view type which includes - list view and large list view types. + This function checks whether the `data_type` is a list view. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a list view, False otherwise. + """ + return is_list_view(data_type.id) and offset_bit_width(data_type.id) == 32 + + +def _is_large_list_view(data_type): + """ + This function checks whether the `data_type` is a large list view. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a large list view, False otherwise. + """ + return is_list_view(data_type.id) and offset_bit_width(data_type.id) == 64 + + +def is_struct(data_type): + """ + This function checks whether the `data_type` is a struct. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a struct, False otherwise. + """ + return data_type.id == Type_STRUCT + + +def _is_union(data_type): + """ + This function checks whether the `data_type` is a union. Parameters ---------- @@ -5862,9 +6211,9 @@ def _is_list_view(data_type): Returns ------- bool - True if `data_type` is a list view type, False otherwise. + True if `data_type` is a union, False otherwise. """ - return is_list_view(data_type.id) + return is_union(data_type.id) def _is_nested(data_type): @@ -5886,10 +6235,9 @@ def _is_nested(data_type): return is_nested(data_type.id) -def _is_union(data_type): +def is_timestamp(data_type): """ - This function checks whether the `data_type` is a union type which includes - sparse union and dense union types. + This function checks whether the `data_type` is a timestamp type. Parameters ---------- @@ -5899,54 +6247,232 @@ def _is_union(data_type): Returns ------- bool - True if `data_type` is a union type, False otherwise. + True if `data_type` is a timestamp type, False otherwise. """ - return is_union(data_type.id) + return is_temporal(data_type.id) and not is_time(data_type.id) and not is_date(data_type.id) -def _bit_width(data_type): +def is_duration(data_type): """ - Determine the bit width of the data type. + This function checks whether the `data_type` is a duration type. - This function evaluates the `data_type` and returns its bit width, which is - essential for understanding the storage size and precision of data types. - It supports a wide range of data types, including boolean, integers, floating - point numbers, date, time, timestamp, duration, intervals, and decimal types. + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a duration type, False otherwise. + """ + return is_primitive(data_type.id) and not is_integer(data_type.id) and \ + not is_floating(data_type.id) and not is_temporal(data_type.id) and \ + not is_interval(data_type.id) + + +def is_time32(data_type): + """ + This function checks whether the `data_type` is a time32 type. Parameters ---------- data_type : DataType - The data type for which to determine the bit width. + The data type to check Returns ------- - int - The bit width of the `data_type`, or 0 if the bit width is not applicable. + bool + True if `data_type` is a time32 type, False otherwise. """ - return bit_width(data_type.id) + return is_time(data_type.id) and bit_width(data_type.id) == 32 -def _offset_bit_width(data_type): +def is_time64(data_type): """ - Determine the offset bit width of the data type. + This function checks whether the `data_type` is a time64 type. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a time64 type, False otherwise. + """ + return is_time(data_type.id) and bit_width(data_type.id) == 64 - This function evaluates the `data_type` and returns its offset bit width, which is - essential for understanding the storage size and precision of data types. - It supports a wide range of data types, including string, binary, list, list - view, map, dense union, large string, large binary, large list, and large list view - types. + +def is_large_binary(data_type): + """ + This function checks whether the `data_type` is a large binary type. Parameters ---------- data_type : DataType - The data type for which to determine the offset bit width. + The data type to check Returns ------- - int - The offset bit width of the `data_type`, or 0 if the offset bit width is not applicable. + bool + True if `data_type` is a large binary type, False otherwise. """ - return offset_bit_width(data_type.id) + return is_binary(data_type.id) and offset_bit_width(data_type.id) == 64 + + +def is_large_string(data_type): + """ + This function checks whether the `data_type` is a large string type. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a large string type, False otherwise. + """ + return is_string(data_type.id) and offset_bit_width(data_type.id) == 64 + + +def is_binary_view(data_type): + """ + This function checks whether the `data_type` is a binary view. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a binary view, False otherwise. + """ + return data_type.id == Type_BINARY_VIEW + + +def is_string_view(data_type): + """ + This function checks whether the `data_type` is a string view. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a string view, False otherwise. + """ + return data_type.id == Type_STRING_VIEW + + +def is_date32(data_type): + """ + This function checks whether the `data_type` is a date32 type. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a date32 type, False otherwise. + """ + return is_date(data_type.id) and bit_width(data_type.id) == 32 + + +def is_date64(data_type): + """ + This function checks whether the `data_type` is a date64 type. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a date64 type, False otherwise. + """ + return is_date(data_type.id) and bit_width(data_type.id) == 64 + + +def is_run_end_encoded(data_type): + """ + This function checks whether the `data_type` is a run end encoded type. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a run end encoded type, False otherwise. + """ + return data_type.id == Type_RUN_END_ENCODED + + +def is_map(data_type): + """ + This function checks whether the `data_type` is a map type. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a map type, False otherwise. + """ + return is_var_length_list(data_type.id) and not is_list(data_type.id) + + +def is_decimal128(data_type): + """ + This function checks whether the `data_type` is a decimal128 type. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a decimal128 type, False otherwise. + """ + return is_decimal(data_type.id) and bit_width(data_type.id) == 128 + + +def is_decimal256(data_type): + """ + This function checks whether the `data_type` is a decimal256 type. + + Parameters + ---------- + data_type : DataType + The data type to check + + Returns + ------- + bool + True if `data_type` is a decimal256 type, False otherwise. + """ + return is_decimal(data_type.id) and bit_width(data_type.id) == 256 cdef class _ExtensionRegistryNanny(_Weakrefable): diff --git a/python/pyarrow/types.py b/python/pyarrow/types.py index d30d63a1f6fb..85a550ea8dda 100644 --- a/python/pyarrow/types.py +++ b/python/pyarrow/types.py @@ -21,307 +21,53 @@ from pyarrow.lib import (is_boolean_value, # noqa is_integer_value, is_float_value, - _is_integer, - _is_signed_integer, - _is_unsigned_integer, - _is_floating, - _is_numeric, - _is_decimal, - _is_run_end_type, - _is_primitive, - _is_base_binary_like, - _is_binary_like, - _is_large_binary_like, - _is_binary, - _is_string, - _is_temporal, - _is_time, - _is_date, - _is_interval, - _is_dictionary, - _is_fixed_size_binary, - _is_fixed_width, - _is_var_length_list, - _is_list, - _is_list_like, - _is_var_length_list_like, - _is_list_view, - _is_nested, - _is_union, - _bit_width, - _offset_bit_width) - -import pyarrow.lib as lib -from pyarrow.util import doc - - -@doc(datatype="null") -def is_null(t): - """ - Return True if value is an instance of type: {datatype}. - - Parameters - ---------- - t : DataType - """ - return t.id == lib.Type_NA - - -@doc(is_null, datatype="boolean") -def is_boolean(t): - return _is_primitive(t) and _bit_width(t) == 1 - - -@doc(is_null, datatype="any integer") -def is_integer(t): - return _is_integer(t) and _bit_width(t) - - -@doc(is_null, datatype="signed integer") -def is_signed_integer(t): - return _is_signed_integer(t) - - -@doc(is_null, datatype="unsigned integer") -def is_unsigned_integer(t): - return _is_unsigned_integer(t) - - -@doc(is_null, datatype="int8") -def is_int8(t): - return _is_integer(t) and _bit_width(t) == 8 - - -@doc(is_null, datatype="int16") -def is_int16(t): - return _is_integer(t) and _bit_width(t) == 16 - - -@doc(is_null, datatype="int32") -def is_int32(t): - return _is_integer(t) and _bit_width(t) == 32 - - -@doc(is_null, datatype="int64") -def is_int64(t): - return _is_integer(t) and _bit_width(t) == 64 - - -@doc(is_null, datatype="uint8") -def is_uint8(t): - return _is_unsigned_integer(t) and _bit_width(t) == 8 - - -@doc(is_null, datatype="uint16") -def is_uint16(t): - return _is_unsigned_integer(t) and _bit_width(t) == 16 - - -@doc(is_null, datatype="uint32") -def is_uint32(t): - return _is_unsigned_integer(t) and _bit_width(t) == 32 - - -@doc(is_null, datatype="uint64") -def is_uint64(t): - return _is_unsigned_integer(t) and _bit_width(t) == 64 - - -@doc(is_null, datatype="floating point numeric") -def is_floating(t): - return _is_floating(t) - - -@doc(is_null, datatype="float16 (half-precision)") -def is_float16(t): - return _is_floating(t) and _bit_width(t) == 16 - - -@doc(is_null, datatype="float32 (single precision)") -def is_float32(t): - return _is_floating(t) and _bit_width(t) == 32 - - -@doc(is_null, datatype="float64 (double precision)") -def is_float64(t): - return _is_floating(t) and _bit_width(t) == 64 - - -@doc(is_null, datatype="list") -def is_list(t): - return _is_list(t) and _offset_bit_width(t) == 32 - - -@doc(is_null, datatype="large list") -def is_large_list(t): - return _is_list(t) and _offset_bit_width(t) == 64 - - -@doc(is_null, datatype="fixed size list") -def is_fixed_size_list(t): - return t.id == lib.Type_FIXED_SIZE_LIST - - -@doc(is_null, datatype="list view") -def is_list_view(t): - return _is_list_view(t) and _offset_bit_width(t) == 32 - - -@doc(is_null, datatype="large list view") -def is_large_list_view(t): - return _is_list_view(t) and _offset_bit_width(t) == 64 - - -@doc(is_null, datatype="struct") -def is_struct(t): - return t.id == lib.Type_STRUCT - - -@doc(is_null, datatype="union") -def is_union(t): - return _is_union(t) - - -@doc(is_null, datatype="nested type") -def is_nested(t): - return _is_nested(t) - - -@doc(is_null, datatype="run-end encoded") -def is_run_end_encoded(t): - return t.id == lib.Type_RUN_END_ENCODED - - -@doc(is_null, datatype="date, time, timestamp or duration") -def is_temporal(t): - return _is_primitive(t) and not _is_integer(t) and \ - not _is_floating(t) - - -@doc(is_null, datatype="timestamp") -def is_timestamp(t): - return _is_temporal(t) and not _is_time(t) and not _is_date(t) - - -@doc(is_null, datatype="duration") -def is_duration(t): - return _is_primitive(t) and not _is_integer(t) and \ - not _is_floating(t) and not _is_temporal(t) and \ - not _is_interval(t) - - -@doc(is_null, datatype="time") -def is_time(t): - return _is_time(t) - - -@doc(is_null, datatype="time32") -def is_time32(t): - return _is_time(t) and _bit_width(t) == 32 - - -@doc(is_null, datatype="time64") -def is_time64(t): - return _is_time(t) and _bit_width(t) == 64 - - -@doc(is_null, datatype="variable-length binary") -def is_binary(t): - return _is_binary(t) and _offset_bit_width(t) == 32 - - -@doc(is_null, datatype="large variable-length binary") -def is_large_binary(t): - return _is_binary(t) and _offset_bit_width(t) == 64 - - -@doc(method="_is_string") -def is_unicode(t): - """ - Alias for {method}. - - Parameters - ---------- - t : DataType - """ - return _is_string(t) - - -@doc(is_null, datatype="string (utf8 unicode)") -def is_string(t): - return _is_string(t) and _offset_bit_width(t) == 32 - - -@doc(is_unicode, method="_is_large_string") -def is_large_unicode(t): - return is_large_string(t) - - -@doc(is_null, datatype="large string (utf8 unicode)") -def is_large_string(t): - return _is_string(t) and _offset_bit_width(t) == 64 - - -@doc(is_null, datatype="fixed size binary") -def is_fixed_size_binary(t): - return _is_fixed_size_binary(t) - - -@doc(is_null, datatype="variable-length binary view") -def is_binary_view(t): - return t.id == lib.Type_BINARY_VIEW - - -@doc(is_null, datatype="variable-length string (utf-8) view") -def is_string_view(t): - return t.id == lib.Type_STRING_VIEW - - -@doc(is_null, datatype="date") -def is_date(t): - return _is_date(t) - - -@doc(is_null, datatype="date32 (days)") -def is_date32(t): - return _is_date(t) and _bit_width(t) == 32 - - -@doc(is_null, datatype="date64 (milliseconds)") -def is_date64(t): - return _is_date(t) and _bit_width(t) == 64 - - -@doc(is_null, datatype="map") -def is_map(t): - return _is_var_length_list(t) and not _is_list(t) - - -@doc(is_null, datatype="decimal") -def is_decimal(t): - return _is_decimal(t) - - -@doc(is_null, datatype="decimal128") -def is_decimal128(t): - return _is_decimal(t) and _bit_width(t) == 128 - - -@doc(is_null, datatype="decimal256") -def is_decimal256(t): - return _is_decimal(t) and _bit_width(t) == 256 - - -@doc(is_null, datatype="dictionary-encoded") -def is_dictionary(t): - return _is_dictionary(t) - - -@doc(is_null, datatype="interval") -def is_interval(t): - return _is_interval(t) - - -@doc(is_null, datatype="primitive type") -def is_primitive(t): - return _is_primitive(t) + ensure_type, + _is_integer as is_integer, + _is_signed_integer as is_signed_integer, + _is_unsigned_integer as is_unsigned_integer, + _is_floating as is_floating, + _is_numeric as is_numeric, + _is_decimal as is_decimal, + _is_run_end_type as is_run_end_type, + _is_primitive as is_primitive, + _is_base_binary_like as is_base_binary_like, + _is_binary_like as is_binary_like, + _is_large_binary_like as is_large_binary_like, + _is_binary as is_binary, + _is_string as is_string, _is_string as is_unicode, + _is_temporal as is_temporal, + _is_time as is_time, + _is_date as is_date, + _is_interval as is_interval, + _is_dictionary as is_dictionary, + _is_fixed_size_binary as is_fixed_size_binary, + _is_fixed_width as is_fixed_width, + _is_var_length_list as is_var_length_list, + _is_list as is_list, + _is_list_like as is_list_like, + _is_var_length_list_like as is_var_length_list_like, + _is_list_view as is_list_view, + _is_large_list_view as is_large_list_view, + _is_nested as is_nested, + _is_union as is_union, + _bit_width as bit_width, + _offset_bit_width as offset_bit_width, + is_run_end_encoded, + is_null, + is_boolean, + is_int8, is_int16, is_int32, is_int64, + is_uint8, is_uint16, is_uint32, is_uint64, + is_float16, is_float32, is_float64, + is_large_list, + is_fixed_size_list, + is_struct, + is_timestamp, + is_duration, + is_time32, is_time64, + is_large_binary, + is_large_string, is_large_string as is_large_unicode, + is_binary_view, + is_string_view, + is_date32, is_date64, + is_map, + is_decimal128, is_decimal256) From cbd5c2e0eda8f4b47ce396d7ed48bde0e01bfe84 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Sat, 6 Apr 2024 13:05:03 +0900 Subject: [PATCH 21/28] apply lint --- python/pyarrow/tests/test_types.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py index fa66e4a28596..f5bc770690be 100644 --- a/python/pyarrow/tests/test_types.py +++ b/python/pyarrow/tests/test_types.py @@ -224,9 +224,9 @@ def test_is_nested_or_struct(): def test_is_union(): for mode in [pa.lib.UnionMode_SPARSE, pa.lib.UnionMode_DENSE]: assert pa.types.is_union(pa.union([pa.field('a', pa.int32()), - pa.field('b', pa.int8()), - pa.field('c', pa.string())], - mode=mode)) + pa.field('b', pa.int8()), + pa.field('c', pa.string())], + mode=mode)) assert not pa.types.is_union(pa.list_(pa.int32())) From 06f1f1ebeade2fdadebe97a21d168c835a048428 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Sat, 6 Apr 2024 13:15:22 +0900 Subject: [PATCH 22/28] update by self-review --- python/pyarrow/__init__.py | 49 ------------------------------ python/pyarrow/lib.pyx | 1 - python/pyarrow/tests/test_types.py | 1 - 3 files changed, 51 deletions(-) diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py index 4fef2ba4dbc4..936f4736977c 100644 --- a/python/pyarrow/__init__.py +++ b/python/pyarrow/__init__.py @@ -184,55 +184,6 @@ def print_entry(label, value): BaseExtensionType, ExtensionType, RunEndEncodedType, FixedShapeTensorType, PyExtensionType, UnknownExtensionType, - # _is_integer as is_integer, - # _is_signed_integer as is_signed_integer, - # _is_unsigned_integer as is_unsigned_integer, - # _is_floating as is_floating, - # _is_numeric as is_numeric, - # _is_decimal as is_decimal, - # _is_run_end_type as is_run_end_type, - # _is_primitive as is_primitive, - # _is_base_binary_like as is_base_binary_like, - # _is_binary_like as is_binary_like, - # _is_large_binary_like as is_large_binary_like, - # _is_binary as is_binary, - # _is_string as is_string, _is_string as is_unicode, - # _is_temporal as is_temporal, - # _is_time as is_time, - # _is_date as is_date, - # _is_interval as is_interval, - # _is_dictionary as is_dictionary, - # _is_fixed_size_binary as is_fixed_size_binary, - # _is_fixed_width as is_fixed_width, - # _is_var_length_list as is_var_length_list, - # _is_list as is_list, - # _is_list_like as is_list_like, - # _is_var_length_list_like as is_var_length_list_like, - # _is_list_view as is_list_view, - # _is_large_list_view as is_large_list_view, - # _is_nested as is_nested, - # _is_union as is_union, - # _bit_width as bit_width, - # _offset_bit_width as offset_bit_width, - # is_run_end_encoded, - # is_null, - # is_boolean, - # is_int8, is_int16, is_int32, is_int64, - # is_uint8, is_uint16, is_uint32, is_uint64, - # is_float16, is_float32, is_float64, - # is_large_list, - # is_fixed_size_list, - # is_struct, - # is_timestamp, - # is_duration, - # is_time32, is_time64, - # is_large_binary, - # is_large_string, is_large_string as is_large_unicode, - # is_binary_view, - # is_string_view, - # is_date32, is_date64, - # is_map, - # is_decimal128, is_decimal256, register_extension_type, unregister_extension_type, DictionaryMemo, KeyValueMetadata, diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx index 1c84d0b38b5e..3245e50f0fe6 100644 --- a/python/pyarrow/lib.pyx +++ b/python/pyarrow/lib.pyx @@ -162,7 +162,6 @@ include "pandas-shim.pxi" # Memory pools and allocation include "memory.pxi" - # DataType, Field, Schema include "types.pxi" diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py index f5bc770690be..b87896dac362 100644 --- a/python/pyarrow/tests/test_types.py +++ b/python/pyarrow/tests/test_types.py @@ -24,7 +24,6 @@ import pytest import hypothesis as h import hypothesis.strategies as st - try: import hypothesis.extra.pytz as tzst except ImportError: From ed78341720690bae80f7cb8a151529ee78dba45a Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Sat, 6 Apr 2024 14:27:57 +0900 Subject: [PATCH 23/28] add docstring to avoid below error Parameters {'ty', 'allow_none'} not documented --- python/pyarrow/types.pxi | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index 4699caf51a97..1cc4cbf6094b 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -5268,6 +5268,38 @@ def type_for_alias(name): cpdef DataType ensure_type(object ty, bint allow_none=False): + """ + Ensures that the input is converted to a valid DataType object. + + Parameters + ---------- + ty : object + The input to validate and convert. Can be a DataType object, a string + alias for a DataType, or None if 'allow_none' is True. + allow_none : bool, default False + Allows 'ty' to be None. If True and 'ty' is None, returns None. + + Returns + ------- + DataType or None + A DataType object corresponding to 'ty', or None if 'ty' is None and + 'allow_none' is True. + + Raises + ------ + TypeError + If 'ty' is not a valid input for conversion. + + Examples + -------- + >>> import pyarrow as pa + >>> pa.types.ensure_type('int32') + DataType(int32) + >>> pa.types.ensure_type(pa.int32()) + DataType(int32) + >>> pa.types.ensure_type(None, allow_none=True) + + """ if allow_none and ty is None: return None elif isinstance(ty, DataType): From c5833519655ebb04ccdc693f8feca5e3841c4eb8 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Thu, 11 Apr 2024 18:38:57 +0900 Subject: [PATCH 24/28] update test_types to avoid pa.types. --- python/pyarrow/tests/test_types.py | 279 +++++++++++++++-------------- 1 file changed, 140 insertions(+), 139 deletions(-) diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py index b87896dac362..9ad78e819374 100644 --- a/python/pyarrow/tests/test_types.py +++ b/python/pyarrow/tests/test_types.py @@ -33,6 +33,7 @@ import numpy as np import pyarrow as pa import pyarrow.tests.strategies as past +from pyarrow import types def get_many_types(): @@ -93,8 +94,8 @@ def get_many_types(): def test_is_boolean(): - assert pa.types.is_boolean(pa.bool_()) - assert not pa.types.is_boolean(pa.int8()) + assert types.is_boolean(pa.bool_()) + assert not types.is_boolean(pa.int8()) def test_is_integer(): @@ -102,30 +103,30 @@ def test_is_integer(): unsigned_ints = [pa.uint8(), pa.uint16(), pa.uint32(), pa.uint64()] for t in signed_ints + unsigned_ints: - assert pa.types.is_integer(t) + assert types.is_integer(t) for t in signed_ints: - assert pa.types.is_signed_integer(t) - assert not pa.types.is_unsigned_integer(t) + assert types.is_signed_integer(t) + assert not types.is_unsigned_integer(t) for t in unsigned_ints: - assert pa.types.is_unsigned_integer(t) - assert not pa.types.is_signed_integer(t) + assert types.is_unsigned_integer(t) + assert not types.is_signed_integer(t) - assert not pa.types.is_integer(pa.float32()) - assert not pa.types.is_signed_integer(pa.float32()) + assert not types.is_integer(pa.float32()) + assert not types.is_signed_integer(pa.float32()) def test_is_floating(): for t in [pa.float16(), pa.float32(), pa.float64()]: - assert pa.types.is_floating(t) + assert types.is_floating(t) - assert not pa.types.is_floating(pa.int32()) + assert not types.is_floating(pa.int32()) def test_is_null(): - assert pa.types.is_null(pa.null()) - assert not pa.types.is_null(pa.list_(pa.int32())) + assert types.is_null(pa.null()) + assert not types.is_null(pa.list_(pa.int32())) def test_null_field_may_not_be_non_nullable(): @@ -139,17 +140,17 @@ def test_is_decimal(): decimal256 = pa.decimal256(76, 38) int32 = pa.int32() - assert pa.types.is_decimal(decimal128) - assert pa.types.is_decimal(decimal256) - assert not pa.types.is_decimal(int32) + assert types.is_decimal(decimal128) + assert types.is_decimal(decimal256) + assert not types.is_decimal(int32) - assert pa.types.is_decimal128(decimal128) - assert not pa.types.is_decimal128(decimal256) - assert not pa.types.is_decimal128(int32) + assert types.is_decimal128(decimal128) + assert not types.is_decimal128(decimal256) + assert not types.is_decimal128(int32) - assert not pa.types.is_decimal256(decimal128) - assert pa.types.is_decimal256(decimal256) - assert not pa.types.is_decimal256(int32) + assert not types.is_decimal256(decimal128) + assert types.is_decimal256(decimal256) + assert not types.is_decimal256(int32) def test_is_list(): @@ -157,50 +158,50 @@ def test_is_list(): b = pa.large_list(pa.int32()) c = pa.list_(pa.int32(), 3) - assert pa.types.is_list(a) - assert not pa.types.is_large_list(a) - assert not pa.types.is_fixed_size_list(a) - assert pa.types.is_large_list(b) - assert not pa.types.is_list(b) - assert not pa.types.is_fixed_size_list(b) - assert pa.types.is_fixed_size_list(c) - assert not pa.types.is_list(c) - assert not pa.types.is_large_list(c) + assert types.is_list(a) + assert not types.is_large_list(a) + assert not types.is_fixed_size_list(a) + assert types.is_large_list(b) + assert not types.is_list(b) + assert not types.is_fixed_size_list(b) + assert types.is_fixed_size_list(c) + assert not types.is_list(c) + assert not types.is_large_list(c) - assert not pa.types.is_list(pa.int32()) + assert not types.is_list(pa.int32()) def test_is_list_view(): a = pa.list_view(pa.int32()) b = pa.large_list_view(pa.int32()) - assert pa.types.is_list_view(a) - assert not pa.types.is_large_list_view(a) - assert not pa.types.is_list(a) - assert pa.types.is_large_list_view(b) - assert not pa.types.is_list_view(b) - assert not pa.types.is_large_list(b) + assert types.is_list_view(a) + assert not types.is_large_list_view(a) + assert not types.is_list(a) + assert types.is_large_list_view(b) + assert not types.is_list_view(b) + assert not types.is_large_list(b) def test_is_map(): m = pa.map_(pa.utf8(), pa.int32()) - assert pa.types.is_map(m) - assert not pa.types.is_map(pa.int32()) + assert types.is_map(m) + assert not types.is_map(pa.int32()) fields = pa.map_(pa.field('key_name', pa.utf8(), nullable=False), pa.field('value_name', pa.int32())) - assert pa.types.is_map(fields) + assert types.is_map(fields) entries_type = pa.struct([pa.field('key', pa.int8()), pa.field('value', pa.int8())]) list_type = pa.list_(entries_type) - assert not pa.types.is_map(list_type) + assert not types.is_map(list_type) def test_is_dictionary(): - assert pa.types.is_dictionary(pa.dictionary(pa.int32(), pa.string())) - assert not pa.types.is_dictionary(pa.int32()) + assert types.is_dictionary(pa.dictionary(pa.int32(), pa.string())) + assert not types.is_dictionary(pa.int32()) def test_is_nested_or_struct(): @@ -208,65 +209,65 @@ def test_is_nested_or_struct(): pa.field('b', pa.int8()), pa.field('c', pa.string())]) - assert pa.types.is_struct(struct_ex) - assert not pa.types.is_struct(pa.list_(pa.int32())) + assert types.is_struct(struct_ex) + assert not types.is_struct(pa.list_(pa.int32())) - assert pa.types.is_nested(struct_ex) - assert pa.types.is_nested(pa.list_(pa.int32())) - assert pa.types.is_nested(pa.list_(pa.int32(), 3)) - assert pa.types.is_nested(pa.large_list(pa.int32())) - assert pa.types.is_nested(pa.list_view(pa.int32())) - assert pa.types.is_nested(pa.large_list_view(pa.int32())) - assert not pa.types.is_nested(pa.int32()) + assert types.is_nested(struct_ex) + assert types.is_nested(pa.list_(pa.int32())) + assert types.is_nested(pa.list_(pa.int32(), 3)) + assert types.is_nested(pa.large_list(pa.int32())) + assert types.is_nested(pa.list_view(pa.int32())) + assert types.is_nested(pa.large_list_view(pa.int32())) + assert not types.is_nested(pa.int32()) def test_is_union(): for mode in [pa.lib.UnionMode_SPARSE, pa.lib.UnionMode_DENSE]: - assert pa.types.is_union(pa.union([pa.field('a', pa.int32()), - pa.field('b', pa.int8()), - pa.field('c', pa.string())], - mode=mode)) - assert not pa.types.is_union(pa.list_(pa.int32())) + assert types.is_union(pa.union([pa.field('a', pa.int32()), + pa.field('b', pa.int8()), + pa.field('c', pa.string())], + mode=mode)) + assert not types.is_union(pa.list_(pa.int32())) def test_is_run_end_encoded(): - assert pa.types.is_run_end_encoded(pa.run_end_encoded(pa.int32(), pa.int64())) - assert not pa.types.is_run_end_encoded(pa.utf8()) + assert types.is_run_end_encoded(pa.run_end_encoded(pa.int32(), pa.int64())) + assert not types.is_run_end_encoded(pa.utf8()) # TODO(wesm): is_map, once implemented def test_is_binary_string(): - assert pa.types.is_binary(pa.binary()) - assert not pa.types.is_binary(pa.string()) - assert not pa.types.is_binary(pa.large_binary()) - assert not pa.types.is_binary(pa.large_string()) + assert types.is_binary(pa.binary()) + assert not types.is_binary(pa.string()) + assert not types.is_binary(pa.large_binary()) + assert not types.is_binary(pa.large_string()) - assert pa.types.is_string(pa.string()) - assert pa.types.is_unicode(pa.string()) - assert not pa.types.is_string(pa.binary()) - assert not pa.types.is_string(pa.large_string()) - assert not pa.types.is_string(pa.large_binary()) + assert types.is_string(pa.string()) + assert types.is_unicode(pa.string()) + assert not types.is_string(pa.binary()) + assert not types.is_string(pa.large_string()) + assert not types.is_string(pa.large_binary()) - assert pa.types.is_large_binary(pa.large_binary()) - assert not pa.types.is_large_binary(pa.large_string()) - assert not pa.types.is_large_binary(pa.binary()) - assert not pa.types.is_large_binary(pa.string()) + assert types.is_large_binary(pa.large_binary()) + assert not types.is_large_binary(pa.large_string()) + assert not types.is_large_binary(pa.binary()) + assert not types.is_large_binary(pa.string()) - assert pa.types.is_large_string(pa.large_string()) - assert not pa.types.is_large_string(pa.large_binary()) - assert not pa.types.is_large_string(pa.string()) - assert not pa.types.is_large_string(pa.binary()) + assert types.is_large_string(pa.large_string()) + assert not types.is_large_string(pa.large_binary()) + assert not types.is_large_string(pa.string()) + assert not types.is_large_string(pa.binary()) - assert pa.types.is_fixed_size_binary(pa.binary(5)) - assert not pa.types.is_fixed_size_binary(pa.binary()) + assert types.is_fixed_size_binary(pa.binary(5)) + assert not types.is_fixed_size_binary(pa.binary()) - assert pa.types.is_string_view(pa.string_view()) - assert not pa.types.is_string_view(pa.string()) - assert pa.types.is_binary_view(pa.binary_view()) - assert not pa.types.is_binary_view(pa.binary()) - assert not pa.types.is_binary_view(pa.string_view()) + assert types.is_string_view(pa.string_view()) + assert not types.is_string_view(pa.string()) + assert types.is_binary_view(pa.binary_view()) + assert not types.is_binary_view(pa.binary()) + assert not types.is_binary_view(pa.string_view()) def test_is_temporal_date_time_timestamp(): @@ -278,48 +279,48 @@ def test_is_temporal_date_time_timestamp(): for case in (date_types + time_types + timestamp_types + duration_types + interval_types): - assert pa.types.is_temporal(case) + assert types.is_temporal(case) for case in date_types: - assert pa.types.is_date(case) - assert not pa.types.is_time(case) - assert not pa.types.is_timestamp(case) - assert not pa.types.is_duration(case) - assert not pa.types.is_interval(case) + assert types.is_date(case) + assert not types.is_time(case) + assert not types.is_timestamp(case) + assert not types.is_duration(case) + assert not types.is_interval(case) for case in time_types: - assert pa.types.is_time(case) - assert not pa.types.is_date(case) - assert not pa.types.is_timestamp(case) - assert not pa.types.is_duration(case) - assert not pa.types.is_interval(case) + assert types.is_time(case) + assert not types.is_date(case) + assert not types.is_timestamp(case) + assert not types.is_duration(case) + assert not types.is_interval(case) for case in timestamp_types: - assert pa.types.is_timestamp(case) - assert not pa.types.is_date(case) - assert not pa.types.is_time(case) - assert not pa.types.is_duration(case) - assert not pa.types.is_interval(case) + assert types.is_timestamp(case) + assert not types.is_date(case) + assert not types.is_time(case) + assert not types.is_duration(case) + assert not types.is_interval(case) for case in duration_types: - assert pa.types.is_duration(case) - assert not pa.types.is_date(case) - assert not pa.types.is_time(case) - assert not pa.types.is_timestamp(case) - assert not pa.types.is_interval(case) + assert types.is_duration(case) + assert not types.is_date(case) + assert not types.is_time(case) + assert not types.is_timestamp(case) + assert not types.is_interval(case) for case in interval_types: - assert pa.types.is_interval(case) - assert not pa.types.is_date(case) - assert not pa.types.is_time(case) - assert not pa.types.is_timestamp(case) + assert types.is_interval(case) + assert not types.is_date(case) + assert not types.is_time(case) + assert not types.is_timestamp(case) - assert not pa.types.is_temporal(pa.int32()) + assert not types.is_temporal(pa.int32()) def test_is_primitive(): - assert pa.types.is_primitive(pa.int32()) - assert not pa.types.is_primitive(pa.list_(pa.int32())) + assert types.is_primitive(pa.int32()) + assert not types.is_primitive(pa.list_(pa.int32())) @pytest.mark.parametrize(('tz', 'expected'), [ @@ -923,21 +924,21 @@ def test_run_end_encoded_type(): @pytest.mark.parametrize('t,check_func', [ - (pa.date32(), pa.types.is_date32), - (pa.date64(), pa.types.is_date64), - (pa.time32('s'), pa.types.is_time32), - (pa.time64('ns'), pa.types.is_time64), - (pa.int8(), pa.types.is_int8), - (pa.int16(), pa.types.is_int16), - (pa.int32(), pa.types.is_int32), - (pa.int64(), pa.types.is_int64), - (pa.uint8(), pa.types.is_uint8), - (pa.uint16(), pa.types.is_uint16), - (pa.uint32(), pa.types.is_uint32), - (pa.uint64(), pa.types.is_uint64), - (pa.float16(), pa.types.is_float16), - (pa.float32(), pa.types.is_float32), - (pa.float64(), pa.types.is_float64) + (pa.date32(), types.is_date32), + (pa.date64(), types.is_date64), + (pa.time32('s'), types.is_time32), + (pa.time64('ns'), types.is_time64), + (pa.int8(), types.is_int8), + (pa.int16(), types.is_int16), + (pa.int32(), types.is_int32), + (pa.int64(), types.is_int64), + (pa.uint8(), types.is_uint8), + (pa.uint16(), types.is_uint16), + (pa.uint32(), types.is_uint32), + (pa.uint64(), types.is_uint64), + (pa.float16(), types.is_float16), + (pa.float32(), types.is_float32), + (pa.float64(), types.is_float64) ]) def test_exact_primitive_types(t, check_func): assert check_func(t) @@ -1259,24 +1260,24 @@ def test_field_modified_copies(): def test_is_integer_value(): - assert pa.types.is_integer_value(1) - assert pa.types.is_integer_value(np.int64(1)) - assert not pa.types.is_integer_value('1') + assert types.is_integer_value(1) + assert types.is_integer_value(np.int64(1)) + assert not types.is_integer_value('1') def test_is_float_value(): - assert not pa.types.is_float_value(1) - assert pa.types.is_float_value(1.) - assert pa.types.is_float_value(np.float64(1)) - assert not pa.types.is_float_value('1.0') + assert not types.is_float_value(1) + assert types.is_float_value(1.) + assert types.is_float_value(np.float64(1)) + assert not types.is_float_value('1.0') def test_is_boolean_value(): - assert not pa.types.is_boolean_value(1) - assert pa.types.is_boolean_value(True) - assert pa.types.is_boolean_value(False) - assert pa.types.is_boolean_value(np.bool_(True)) - assert pa.types.is_boolean_value(np.bool_(False)) + assert not types.is_boolean_value(1) + assert types.is_boolean_value(True) + assert types.is_boolean_value(False) + assert types.is_boolean_value(np.bool_(True)) + assert types.is_boolean_value(np.bool_(False)) @h.settings(suppress_health_check=(h.HealthCheck.too_slow,)) From ac1d5f9cd472f112c64eee236193adcd01494884 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Thu, 11 Apr 2024 22:10:07 +0900 Subject: [PATCH 25/28] unexport ensure_type function --- python/pyarrow/compute.py | 2 +- python/pyarrow/types.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/python/pyarrow/compute.py b/python/pyarrow/compute.py index b8cf5dbfdef6..af35718e1a3c 100644 --- a/python/pyarrow/compute.py +++ b/python/pyarrow/compute.py @@ -397,7 +397,7 @@ def cast(arr, target_type=None, safe=None, options=None, memory_pool=None): " or pass a value for 'options'") if options is None: - target_type = pa.types.ensure_type(target_type) + target_type = pa.lib.ensure_type(target_type) if safe is False: options = CastOptions.unsafe(target_type) else: diff --git a/python/pyarrow/types.py b/python/pyarrow/types.py index 85a550ea8dda..505768b01415 100644 --- a/python/pyarrow/types.py +++ b/python/pyarrow/types.py @@ -21,7 +21,6 @@ from pyarrow.lib import (is_boolean_value, # noqa is_integer_value, is_float_value, - ensure_type, _is_integer as is_integer, _is_signed_integer as is_signed_integer, _is_unsigned_integer as is_unsigned_integer, From 4ba8af0fd99abb886b8a1a005d1b13457d4bbec2 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Thu, 11 Apr 2024 23:35:40 +0900 Subject: [PATCH 26/28] update docstring --- python/pyarrow/types.pxi | 608 ++++++++------------------------------- 1 file changed, 118 insertions(+), 490 deletions(-) diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index 1cc4cbf6094b..8a7d752dd756 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -5268,38 +5268,6 @@ def type_for_alias(name): cpdef DataType ensure_type(object ty, bint allow_none=False): - """ - Ensures that the input is converted to a valid DataType object. - - Parameters - ---------- - ty : object - The input to validate and convert. Can be a DataType object, a string - alias for a DataType, or None if 'allow_none' is True. - allow_none : bool, default False - Allows 'ty' to be None. If True and 'ty' is None, returns None. - - Returns - ------- - DataType or None - A DataType object corresponding to 'ty', or None if 'ty' is None and - 'allow_none' is True. - - Raises - ------ - TypeError - If 'ty' is not a valid input for conversion. - - Examples - -------- - >>> import pyarrow as pa - >>> pa.types.ensure_type('int32') - DataType(int32) - >>> pa.types.ensure_type(pa.int32()) - DataType(int32) - >>> pa.types.ensure_type(None, allow_none=True) - - """ if allow_none and ty is None: return None elif isinstance(ty, DataType): @@ -5446,257 +5414,168 @@ def is_float_value(object obj): def _is_integer(data_type): """ - This function checks whether the `data_type` is an integer type, which - includes signed and unsigned integers of various bit widths (8, 16, 32, 64 bits). + Returns whether the `data_type` is an integer type. Parameters ---------- data_type : DataType - The data type to check against the set of supported integer types. - - Returns - ------- - bool - True if `data_type` is an integer type, False otherwise. + The data type to check. """ return is_integer(data_type.id) def _is_signed_integer(data_type): """ - This function checks whether the `data_type` is a signed integer type, - which includes signed integers of various bit widths (8, 16, 32, 64 bits). + Returns whether the `data_type` is a signed integer type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a signed integer type, False otherwise. + The data type to check. """ return is_signed_integer(data_type.id) def _is_unsigned_integer(data_type): """ - This function checks whether the `data_type` is an unsigned integer type, - which includes unsigned integers of various bit widths (8, 16, 32, 64 bits). + Returns whether the `data_type` is an unsigned integer type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is an unsigned integer type, False otherwise. + The data type to check. """ return is_unsigned_integer(data_type.id) def _is_floating(data_type): """ - This function checks whether the `data_type` is a floating type, which includes - floating point numbers of various bit widths (16, 32, 64 bits). + Returns whether the `data_type` is a floating type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a floating type, False otherwise. + The data type to check. """ return is_floating(data_type.id) def _is_numeric(data_type): """ - This function checks whether the `data_type` is a numeric type, which includes - integers and floating point numbers with specific bit widths. Integer types - include signed and unsigned integers of various bit widths (8, 16, 32, 64 bits), - while floating point types include floating point numbers of various bit widths - (16, 32, 64 bits). + Returns whether the `data_type` is a numeric type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a numeric type, False otherwise. + The data type to check. """ return is_numeric(data_type.id) def _is_decimal(data_type): """ - This function checks whether the `data_type` is a decimal type, which includes - fixed-point decimal numbers with specific precision and scale. + Returns whether the `data_type` is a decimal type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a decimal type, False otherwise. + The data type to check. """ return is_decimal(data_type.id) def _is_run_end_type(data_type): """ - This function checks whether the `data_type` is a run end type, which includes - integers of various bit widths (16, 32, 64 bits). + Returns whether the `data_type` is a run end type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a run end type, False otherwise. + The data type to check. """ return is_run_end_type(data_type.id) def _is_primitive(data_type): """ - This function checks whether the `data_type` is a primitive type, which includes - boolean, integers, floating point numbers, dates, times, timestamp, duration - and intervals types. + Returns whether the `data_type` is a primitive type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a primitive type, False otherwise. + The data type to check. """ return is_primitive(data_type.id) def _is_base_binary_like(data_type): """ - This function checks whether the `data_type` is a base binary-like type, which - includes binary, string, large binary, and large string types. + Returns whether the `data_type` is a base binary-like type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a base binary-like type, False otherwise. + The data type to check. """ return is_base_binary_like(data_type.id) def _is_binary_like(data_type): """ - This function checks whether the `data_type` is a binary-like type, which includes - binary and string types. + Returns whether the `data_type` is a binary-like type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a binary-like type, False otherwise. + The data type to check. """ return is_binary_like(data_type.id) def _is_large_binary_like(data_type): """ - This function checks whether the `data_type` is a large binary-like type, which - includes large binary and large string types. + Returns whether the `data_type` is a large binary-like type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a large binary-like type, False otherwise. + The data type to check. """ return is_large_binary_like(data_type.id) def _is_binary(data_type): """ - This function checks whether the `data_type` is a binary type, which includes - binary and large binary types. + Returns whether the `data_type` is a binary type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a binary type, False otherwise. + The data type to check. """ return is_binary(data_type.id) and offset_bit_width(data_type.id) == 32 def _is_string(data_type): """ - This function checks whether the `data_type` is a string type. + Returns whether the `data_type` is a string type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a string type, False otherwise. + The data type to check. """ return is_string(data_type.id) and offset_bit_width(data_type.id) == 32 def _is_temporal(data_type): """ - This function checks whether the `data_type` is a temporal type, which includes - dates (days since the UNIX epoch and milliseconds since the UNIX epoch), times - (seconds and milliseconds since midnight), timestamp (milliseconds since the UNIX - epoch) types. + Returns whether the `data_type` is a temporal type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a temporal type, False otherwise. + The data type to check. """ return is_primitive(data_type.id) and not is_integer(data_type.id) and \ not is_floating(data_type.id) @@ -5704,178 +5583,120 @@ def _is_temporal(data_type): def _is_time(data_type): """ - This function checks whether the `data_type` is a time type, which includes - times (seconds and milliseconds since midnight) types. + Returns whether the `data_type` is a time type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a time type, False otherwise. + The data type to check. """ return is_time(data_type.id) def _is_date(data_type): """ - This function checks whether the `data_type` is a date type, which includes - dates (days since the UNIX epoch and milliseconds since the UNIX epoch) types. + Returns whether the `data_type` is a date type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a date type, False otherwise. + The data type to check. """ return is_date(data_type.id) def _is_interval(data_type): """ - This function checks whether the `data_type` is a interval type, which includes - intervals (months, days plus nanoseconds, and day-time intervals) types. + Returns whether the `data_type` is an interval type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a interval type, False otherwise. + The data type to check. """ return is_interval(data_type.id) def _is_dictionary(data_type): """ - This function checks whether the `data_type` is a dictionary type. + Returns whether the `data_type` is a dictionary type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a dictionary type, False otherwise. + The data type to check. """ return is_dictionary(data_type.id) def _is_fixed_size_binary(data_type): """ - This function checks whether the `data_type` is a fixed size binary type - which includes decimal and fixed size binary types. + Returns whether the `data_type` is a fixed size binary type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a fixed size binary type, False otherwise. + The data type to check. """ return is_fixed_size_binary(data_type.id) def _is_fixed_width(data_type): """ - This function checks whether the `data_type` is a fixed width type which includes - is_primitive, is_dictionary, is_fixed_size_binary types. + Returns whether the `data_type` is a fixed width type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a fixed width type, False otherwise. + The data type to check. """ return is_fixed_width(data_type.id) def _is_var_length_list(data_type): """ - This function checks whether the `data_type` is a variable length list type - which includes list, large list, and map types. + Returns whether the `data_type` is a variable length list type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a variable length list type, False otherwise. + The data type to check. """ return is_var_length_list(data_type.id) def _is_list(data_type): """ - This function checks whether the `data_type` is a list type. + Returns whether the `data_type` is a list type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a list type, False otherwise. + The data type to check. """ return is_list(data_type.id) and offset_bit_width(data_type.id) == 32 def _is_list_like(data_type): """ - This function checks whether the `data_type` is a list_like type which includes - list, large list, fixed size list, and map types. + Returns whether the `data_type` is a list like type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a list_like type, False otherwise. + The data type to check. """ return is_list_like(data_type.id) def _is_var_length_list_like(data_type): """ - This function checks whether the `data_type` is a variable length list like - type which includes list, large list, list view, large list view, and map types. + Returns whether the `data_type` is a variable length list like type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a variable length list like type, False otherwise. + The data type to check. """ return is_var_length_list_like(data_type.id) @@ -5884,20 +5705,10 @@ def _bit_width(data_type): """ Determine the bit width of the data type. - This function evaluates the `data_type` and returns its bit width, which is - essential for understanding the storage size and precision of data types. - It supports a wide range of data types, including boolean, integers, floating - point numbers, date, time, timestamp, duration, intervals, and decimal types. - Parameters ---------- data_type : DataType - The data type for which to determine the bit width. - - Returns - ------- - int - The bit width of the `data_type`, or 0 if the bit width is not applicable. + The data type to check. """ return bit_width(data_type.id) @@ -5906,397 +5717,274 @@ def _offset_bit_width(data_type): """ Determine the offset bit width of the data type. - This function evaluates the `data_type` and returns its offset bit width, which is - essential for understanding the storage size and precision of data types. - It supports a wide range of data types, including string, binary, list, list - view, map, dense union, large string, large binary, large list, and large list view - types. - Parameters ---------- data_type : DataType - The data type for which to determine the offset bit width. - - Returns - ------- - int - The offset bit width of the `data_type`, or 0 if the offset bit width is not applicable. + The data type to check. """ return offset_bit_width(data_type.id) def is_null(data_type): """ - This function checks whether the `data_type` is a null type. + Returns whether the `data_type` is a null type. Parameters ---------- data_type : DataType - The data type to check against the null type. - - Returns - ------- - bool - True if `data_type` is a null type, False otherwise. + The data type to check. """ return data_type.id == Type_NA def is_boolean(data_type): """ - This function checks whether the `data_type` is a boolean type. + Returns whether the `data_type` is a boolean type. Parameters ---------- data_type : DataType - The data type to check against the boolean type. - - Returns - ------- - bool - True if `data_type` is a boolean type, False otherwise. + The data type to check. """ return data_type.id == Type_BOOL def is_int8(data_type): """ - This function checks whether the `data_type` is an 8-bit integer. + Returns whether the `data_type` is an 8-bit integer. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is an 8-bit integer, False otherwise. + The data type to check. """ return is_integer(data_type.id) and bit_width(data_type.id) == 8 def is_int16(data_type): """ - This function checks whether the `data_type` is an 16-bit integer. + Returns whether the `data_type` is an 16-bit integer. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is an 16-bit integer, False otherwise. + The data type to check. """ return is_integer(data_type.id) and bit_width(data_type.id) == 16 def is_int32(data_type): """ - This function checks whether the `data_type` is an 32-bit integer. + Returns whether the `data_type` is an 32-bit integer. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is an 32-bit integer, False otherwise. + The data type to check. """ return is_integer(data_type.id) and bit_width(data_type.id) == 32 def is_int64(data_type): """ - This function checks whether the `data_type` is an 64-bit integer. + Returns whether the `data_type` is an 64-bit integer. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is an 64-bit integer, False otherwise. + The data type to check. """ return is_integer(data_type.id) and bit_width(data_type.id) == 64 def is_uint8(data_type): """ - This function checks whether the `data_type` is an 8-bit unsigned integer. + Returns whether the `data_type` is an 8-bit unsigned integer. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is an 8-bit unsigned integer, False otherwise. + The data type to check. """ return is_unsigned_integer(data_type.id) and bit_width(data_type.id) == 8 def is_uint16(data_type): """ - This function checks whether the `data_type` is an 16-bit unsigned integer. + Returns whether the `data_type` is an 16-bit unsigned integer. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is an 16-bit unsigned integer, False otherwise. + The data type to check. """ return is_unsigned_integer(data_type.id) and bit_width(data_type.id) == 16 def is_uint32(data_type): """ - This function checks whether the `data_type` is an 32-bit unsigned integer. + Returns whether the `data_type` is an 32-bit unsigned integer. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is an 32-bit unsigned integer, False otherwise. + The data type to check. """ return is_unsigned_integer(data_type.id) and bit_width(data_type.id) == 32 def is_uint64(data_type): """ - This function checks whether the `data_type` is an 64-bit unsigned integer. + Returns whether the `data_type` is an 64-bit unsigned integer. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is an 64-bit unsigned integer, False otherwise. + The data type to check. """ return is_unsigned_integer(data_type.id) and bit_width(data_type.id) == 64 def is_float16(data_type): """ - This function checks whether the `data_type` is a 16-bit floating point number. + Returns whether the `data_type` is a 16-bit floating point number. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a 16-bit floating point number, False otherwise. + The data type to check. """ return is_floating(data_type.id) and bit_width(data_type.id) == 16 def is_float32(data_type): """ - This function checks whether the `data_type` is a 32-bit floating point number. + Returns whether the `data_type` is a 32-bit floating point number. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a 32-bit floating point number, False otherwise. + The data type to check. """ return is_floating(data_type.id) and bit_width(data_type.id) == 32 def is_float64(data_type): """ - This function checks whether the `data_type` is a 64-bit floating point number. + Returns whether the `data_type` is a 64-bit floating point number. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a 64-bit floating point number, False otherwise. + The data type to check. """ return is_floating(data_type.id) and bit_width(data_type.id) == 64 def is_large_list(data_type): """ - This function checks whether the `data_type` is a large list type. + Returns whether the `data_type` is a large list type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a large list type, False otherwise. + The data type to check. """ return is_list(data_type.id) and offset_bit_width(data_type.id) == 64 def is_fixed_size_list(data_type): """ - This function checks whether the `data_type` is a fixed size list type. + Returns whether the `data_type` is a fixed size list type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a fixed size list type, False otherwise. + The data type to check. """ return data_type.id == Type_FIXED_SIZE_LIST def _is_list_view(data_type): """ - This function checks whether the `data_type` is a list view. + Returns whether the `data_type` is a list view type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a list view, False otherwise. + The data type to check. """ return is_list_view(data_type.id) and offset_bit_width(data_type.id) == 32 def _is_large_list_view(data_type): """ - This function checks whether the `data_type` is a large list view. + Returns whether the `data_type` is a large list view type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a large list view, False otherwise. + The data type to check. """ return is_list_view(data_type.id) and offset_bit_width(data_type.id) == 64 def is_struct(data_type): """ - This function checks whether the `data_type` is a struct. + Returns whether the `data_type` is a struct type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a struct, False otherwise. + The data type to check. """ return data_type.id == Type_STRUCT def _is_union(data_type): """ - This function checks whether the `data_type` is a union. + Returns whether the `data_type` is a union type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a union, False otherwise. + The data type to check. """ return is_union(data_type.id) def _is_nested(data_type): """ - This function checks whether the `data_type` is a nested type which includes - list, large list, list view, large list view, fixed size list, map, struct, - sparse union, dense union, and run end encoded types. + Returns whether the `data_type` is a nested type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a nested type, False otherwise. + The data type to check. """ return is_nested(data_type.id) def is_timestamp(data_type): """ - This function checks whether the `data_type` is a timestamp type. + Returns whether the `data_type` is a timestamp type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a timestamp type, False otherwise. + The data type to check. """ return is_temporal(data_type.id) and not is_time(data_type.id) and not is_date(data_type.id) def is_duration(data_type): """ - This function checks whether the `data_type` is a duration type. + Returns whether the `data_type` is a duration type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a duration type, False otherwise. + The data type to check. """ return is_primitive(data_type.id) and not is_integer(data_type.id) and \ not is_floating(data_type.id) and not is_temporal(data_type.id) and \ @@ -6305,204 +5993,144 @@ def is_duration(data_type): def is_time32(data_type): """ - This function checks whether the `data_type` is a time32 type. + Returns whether the `data_type` is a time32 type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a time32 type, False otherwise. + The data type to check. """ return is_time(data_type.id) and bit_width(data_type.id) == 32 def is_time64(data_type): """ - This function checks whether the `data_type` is a time64 type. + Returns whether the `data_type` is a time64 type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a time64 type, False otherwise. + The data type to check. """ return is_time(data_type.id) and bit_width(data_type.id) == 64 def is_large_binary(data_type): """ - This function checks whether the `data_type` is a large binary type. + Returns whether the `data_type` is a large binary type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a large binary type, False otherwise. + The data type to check. """ return is_binary(data_type.id) and offset_bit_width(data_type.id) == 64 def is_large_string(data_type): """ - This function checks whether the `data_type` is a large string type. + Returns whether the `data_type` is a large string type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a large string type, False otherwise. + The data type to check. """ return is_string(data_type.id) and offset_bit_width(data_type.id) == 64 def is_binary_view(data_type): """ - This function checks whether the `data_type` is a binary view. + Returns whether the `data_type` is a binary view type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a binary view, False otherwise. + The data type to check. """ return data_type.id == Type_BINARY_VIEW def is_string_view(data_type): """ - This function checks whether the `data_type` is a string view. + Returns whether the `data_type` is a string view type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a string view, False otherwise. + The data type to check. """ return data_type.id == Type_STRING_VIEW def is_date32(data_type): """ - This function checks whether the `data_type` is a date32 type. + Returns whether the `data_type` is a date32 type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a date32 type, False otherwise. + The data type to check. """ return is_date(data_type.id) and bit_width(data_type.id) == 32 def is_date64(data_type): """ - This function checks whether the `data_type` is a date64 type. + Returns whether the `data_type` is a date64 type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a date64 type, False otherwise. + The data type to check. """ return is_date(data_type.id) and bit_width(data_type.id) == 64 def is_run_end_encoded(data_type): """ - This function checks whether the `data_type` is a run end encoded type. + Returns whether the `data_type` is a run end encoded type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a run end encoded type, False otherwise. + The data type to check. """ return data_type.id == Type_RUN_END_ENCODED def is_map(data_type): """ - This function checks whether the `data_type` is a map type. + Returns whether the `data_type` is a map type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a map type, False otherwise. + The data type to check. """ return is_var_length_list(data_type.id) and not is_list(data_type.id) def is_decimal128(data_type): """ - This function checks whether the `data_type` is a decimal128 type. + Returns whether the `data_type` is a decimal128 type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a decimal128 type, False otherwise. + The data type to check. """ return is_decimal(data_type.id) and bit_width(data_type.id) == 128 def is_decimal256(data_type): """ - This function checks whether the `data_type` is a decimal256 type. + Returns whether the `data_type` is a decimal256 type. Parameters ---------- data_type : DataType - The data type to check - - Returns - ------- - bool - True if `data_type` is a decimal256 type, False otherwise. + The data type to check. """ return is_decimal(data_type.id) and bit_width(data_type.id) == 256 From 9b2e795fae057a7ef7700e89995f5491cb11c154 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Mon, 6 May 2024 22:19:14 +0800 Subject: [PATCH 27/28] update extern statements --- python/pyarrow/includes/libarrow.pxd | 58 ++++----- python/pyarrow/types.pxi | 172 +++++++++++++-------------- python/pyarrow/types.py | 60 +++++----- 3 files changed, 145 insertions(+), 145 deletions(-) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index a77a530de5b8..4cf0b5735050 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -3025,32 +3025,32 @@ cdef extern from "arrow/compute/cast.h" namespace "arrow::compute": CResult[CDatum] Cast(const CDatum& value, const CCastOptions& options) cdef extern from "arrow/type_traits.h" namespace "arrow": - c_bool is_integer(Type type_id) - c_bool is_signed_integer(Type type_id) - c_bool is_unsigned_integer(Type type_id) - c_bool is_floating(Type type_id) - c_bool is_numeric(Type type_id) - c_bool is_decimal(Type type_id) - c_bool is_run_end_type(Type type_id) - c_bool is_primitive(Type type_id) - c_bool is_base_binary_like(Type type_id) - c_bool is_binary_like(Type type_id) - c_bool is_large_binary_like(Type type_id) - c_bool is_binary(Type type_id) - c_bool is_string(Type type_id) - c_bool is_temporal(Type type_id) - c_bool is_time(Type type_id) - c_bool is_date(Type type_id) - c_bool is_interval(Type type_id) - c_bool is_dictionary(Type type_id) - c_bool is_fixed_size_binary(Type type_id) - c_bool is_fixed_width(Type type_id) - c_bool is_var_length_list(Type type_id) - c_bool is_list(Type type_id) - c_bool is_list_like(Type type_id) - c_bool is_var_length_list_like(Type type_id) - c_bool is_list_view(Type type_id) - c_bool is_nested(Type type_id) - c_bool is_union(Type type_id) - int bit_width(Type type_id) - int offset_bit_width(Type type_id) + c_bool c_is_integer "is_integer"(Type type_id) + c_bool c_is_signed_integer "is_signed_integer"(Type type_id) + c_bool c_is_unsigned_integer "is_unsigned_integer"(Type type_id) + c_bool c_is_floating "is_floating"(Type type_id) + c_bool c_is_numeric "is_numeric"(Type type_id) + c_bool c_is_decimal "is_decimal"(Type type_id) + c_bool c_is_run_end "is_run_end_type"(Type type_id) + c_bool c_is_primitive "is_primitive"(Type type_id) + c_bool c_is_base_binary_like "is_base_binary_like"(Type type_id) + c_bool c_is_binary_like "is_binary_like"(Type type_id) + c_bool c_is_large_binary_like "is_large_binary_like"(Type type_id) + c_bool c_is_binary "is_binary"(Type type_id) + c_bool c_is_string "is_string"(Type type_id) + c_bool c_is_temporal "is_temporal"(Type type_id) + c_bool c_is_time "is_time"(Type type_id) + c_bool c_is_date "is_date"(Type type_id) + c_bool c_is_interval "is_interval"(Type type_id) + c_bool c_is_dictionary "is_dictionary"(Type type_id) + c_bool c_is_fixed_size_binary "is_fixed_size_binary"(Type type_id) + c_bool c_is_fixed_width "is_fixed_width"(Type type_id) + c_bool c_is_var_length_list "is_var_length_list"(Type type_id) + c_bool c_is_list "is_list"(Type type_id) + c_bool c_is_list_like "is_list_like"(Type type_id) + c_bool c_is_var_length_list_like "is_var_length_list_like"(Type type_id) + c_bool c_is_list_view "is_list_view"(Type type_id) + c_bool c_is_nested "is_nested"(Type type_id) + c_bool c_is_union "is_union"(Type type_id) + int c_bit_width "bit_width"(Type type_id) + int c_offset_bit_width "offset_bit_width"(Type type_id) diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index 8a7d752dd756..1aa3759b168c 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -5412,7 +5412,7 @@ def is_float_value(object obj): return IsPyFloat(obj) -def _is_integer(data_type): +def is_integer(data_type): """ Returns whether the `data_type` is an integer type. @@ -5421,10 +5421,10 @@ def _is_integer(data_type): data_type : DataType The data type to check. """ - return is_integer(data_type.id) + return c_is_integer(data_type.id) -def _is_signed_integer(data_type): +def is_signed_integer(data_type): """ Returns whether the `data_type` is a signed integer type. @@ -5433,10 +5433,10 @@ def _is_signed_integer(data_type): data_type : DataType The data type to check. """ - return is_signed_integer(data_type.id) + return c_is_signed_integer(data_type.id) -def _is_unsigned_integer(data_type): +def is_unsigned_integer(data_type): """ Returns whether the `data_type` is an unsigned integer type. @@ -5445,10 +5445,10 @@ def _is_unsigned_integer(data_type): data_type : DataType The data type to check. """ - return is_unsigned_integer(data_type.id) + return c_is_unsigned_integer(data_type.id) -def _is_floating(data_type): +def is_floating(data_type): """ Returns whether the `data_type` is a floating type. @@ -5457,10 +5457,10 @@ def _is_floating(data_type): data_type : DataType The data type to check. """ - return is_floating(data_type.id) + return c_is_floating(data_type.id) -def _is_numeric(data_type): +def is_numeric(data_type): """ Returns whether the `data_type` is a numeric type. @@ -5469,10 +5469,10 @@ def _is_numeric(data_type): data_type : DataType The data type to check. """ - return is_numeric(data_type.id) + return c_is_numeric(data_type.id) -def _is_decimal(data_type): +def is_decimal(data_type): """ Returns whether the `data_type` is a decimal type. @@ -5481,10 +5481,10 @@ def _is_decimal(data_type): data_type : DataType The data type to check. """ - return is_decimal(data_type.id) + return c_is_decimal(data_type.id) -def _is_run_end_type(data_type): +def is_run_end_encoded(data_type): """ Returns whether the `data_type` is a run end type. @@ -5493,10 +5493,10 @@ def _is_run_end_type(data_type): data_type : DataType The data type to check. """ - return is_run_end_type(data_type.id) + return c_is_run_end(data_type.id) -def _is_primitive(data_type): +def is_primitive(data_type): """ Returns whether the `data_type` is a primitive type. @@ -5505,10 +5505,10 @@ def _is_primitive(data_type): data_type : DataType The data type to check. """ - return is_primitive(data_type.id) + return c_is_primitive(data_type.id) -def _is_base_binary_like(data_type): +def is_base_binary_like(data_type): """ Returns whether the `data_type` is a base binary-like type. @@ -5517,10 +5517,10 @@ def _is_base_binary_like(data_type): data_type : DataType The data type to check. """ - return is_base_binary_like(data_type.id) + return c_is_base_binary_like(data_type.id) -def _is_binary_like(data_type): +def is_binary_like(data_type): """ Returns whether the `data_type` is a binary-like type. @@ -5529,10 +5529,10 @@ def _is_binary_like(data_type): data_type : DataType The data type to check. """ - return is_binary_like(data_type.id) + return c_is_binary_like(data_type.id) -def _is_large_binary_like(data_type): +def is_large_binary_like(data_type): """ Returns whether the `data_type` is a large binary-like type. @@ -5541,10 +5541,10 @@ def _is_large_binary_like(data_type): data_type : DataType The data type to check. """ - return is_large_binary_like(data_type.id) + return c_is_large_binary_like(data_type.id) -def _is_binary(data_type): +def is_binary(data_type): """ Returns whether the `data_type` is a binary type. @@ -5553,10 +5553,10 @@ def _is_binary(data_type): data_type : DataType The data type to check. """ - return is_binary(data_type.id) and offset_bit_width(data_type.id) == 32 + return c_is_binary(data_type.id) and c_offset_bit_width(data_type.id) == 32 -def _is_string(data_type): +def is_string(data_type): """ Returns whether the `data_type` is a string type. @@ -5565,10 +5565,10 @@ def _is_string(data_type): data_type : DataType The data type to check. """ - return is_string(data_type.id) and offset_bit_width(data_type.id) == 32 + return c_is_string(data_type.id) and c_offset_bit_width(data_type.id) == 32 -def _is_temporal(data_type): +def is_temporal(data_type): """ Returns whether the `data_type` is a temporal type. @@ -5577,11 +5577,11 @@ def _is_temporal(data_type): data_type : DataType The data type to check. """ - return is_primitive(data_type.id) and not is_integer(data_type.id) and \ - not is_floating(data_type.id) + return c_is_primitive(data_type.id) and not c_is_integer(data_type.id) and \ + not c_is_floating(data_type.id) -def _is_time(data_type): +def is_time(data_type): """ Returns whether the `data_type` is a time type. @@ -5590,10 +5590,10 @@ def _is_time(data_type): data_type : DataType The data type to check. """ - return is_time(data_type.id) + return c_is_time(data_type.id) -def _is_date(data_type): +def is_date(data_type): """ Returns whether the `data_type` is a date type. @@ -5602,10 +5602,10 @@ def _is_date(data_type): data_type : DataType The data type to check. """ - return is_date(data_type.id) + return c_is_date(data_type.id) -def _is_interval(data_type): +def is_interval(data_type): """ Returns whether the `data_type` is an interval type. @@ -5614,10 +5614,10 @@ def _is_interval(data_type): data_type : DataType The data type to check. """ - return is_interval(data_type.id) + return c_is_interval(data_type.id) -def _is_dictionary(data_type): +def is_dictionary(data_type): """ Returns whether the `data_type` is a dictionary type. @@ -5626,10 +5626,10 @@ def _is_dictionary(data_type): data_type : DataType The data type to check. """ - return is_dictionary(data_type.id) + return c_is_dictionary(data_type.id) -def _is_fixed_size_binary(data_type): +def is_fixed_size_binary(data_type): """ Returns whether the `data_type` is a fixed size binary type. @@ -5638,10 +5638,10 @@ def _is_fixed_size_binary(data_type): data_type : DataType The data type to check. """ - return is_fixed_size_binary(data_type.id) + return c_is_fixed_size_binary(data_type.id) -def _is_fixed_width(data_type): +def is_fixed_width(data_type): """ Returns whether the `data_type` is a fixed width type. @@ -5650,10 +5650,10 @@ def _is_fixed_width(data_type): data_type : DataType The data type to check. """ - return is_fixed_width(data_type.id) + return c_is_fixed_width(data_type.id) -def _is_var_length_list(data_type): +def is_var_length_list(data_type): """ Returns whether the `data_type` is a variable length list type. @@ -5662,10 +5662,10 @@ def _is_var_length_list(data_type): data_type : DataType The data type to check. """ - return is_var_length_list(data_type.id) + return c_is_var_length_list(data_type.id) -def _is_list(data_type): +def is_list(data_type): """ Returns whether the `data_type` is a list type. @@ -5674,10 +5674,10 @@ def _is_list(data_type): data_type : DataType The data type to check. """ - return is_list(data_type.id) and offset_bit_width(data_type.id) == 32 + return c_is_list(data_type.id) and c_offset_bit_width(data_type.id) == 32 -def _is_list_like(data_type): +def is_list_like(data_type): """ Returns whether the `data_type` is a list like type. @@ -5686,10 +5686,10 @@ def _is_list_like(data_type): data_type : DataType The data type to check. """ - return is_list_like(data_type.id) + return c_is_list_like(data_type.id) -def _is_var_length_list_like(data_type): +def is_var_length_list_like(data_type): """ Returns whether the `data_type` is a variable length list like type. @@ -5698,10 +5698,10 @@ def _is_var_length_list_like(data_type): data_type : DataType The data type to check. """ - return is_var_length_list_like(data_type.id) + return c_is_var_length_list_like(data_type.id) -def _bit_width(data_type): +def bit_width_length(data_type): """ Determine the bit width of the data type. @@ -5710,10 +5710,10 @@ def _bit_width(data_type): data_type : DataType The data type to check. """ - return bit_width(data_type.id) + return c_bit_width(data_type.id) -def _offset_bit_width(data_type): +def offset_bit_width_length(data_type): """ Determine the offset bit width of the data type. @@ -5722,7 +5722,7 @@ def _offset_bit_width(data_type): data_type : DataType The data type to check. """ - return offset_bit_width(data_type.id) + return c_offset_bit_width(data_type.id) def is_null(data_type): @@ -5758,7 +5758,7 @@ def is_int8(data_type): data_type : DataType The data type to check. """ - return is_integer(data_type.id) and bit_width(data_type.id) == 8 + return c_is_integer(data_type.id) and c_bit_width(data_type.id) == 8 def is_int16(data_type): @@ -5770,7 +5770,7 @@ def is_int16(data_type): data_type : DataType The data type to check. """ - return is_integer(data_type.id) and bit_width(data_type.id) == 16 + return c_is_integer(data_type.id) and c_bit_width(data_type.id) == 16 def is_int32(data_type): @@ -5782,7 +5782,7 @@ def is_int32(data_type): data_type : DataType The data type to check. """ - return is_integer(data_type.id) and bit_width(data_type.id) == 32 + return c_is_integer(data_type.id) and c_bit_width(data_type.id) == 32 def is_int64(data_type): @@ -5794,7 +5794,7 @@ def is_int64(data_type): data_type : DataType The data type to check. """ - return is_integer(data_type.id) and bit_width(data_type.id) == 64 + return c_is_integer(data_type.id) and c_bit_width(data_type.id) == 64 def is_uint8(data_type): @@ -5806,7 +5806,7 @@ def is_uint8(data_type): data_type : DataType The data type to check. """ - return is_unsigned_integer(data_type.id) and bit_width(data_type.id) == 8 + return c_is_unsigned_integer(data_type.id) and c_bit_width(data_type.id) == 8 def is_uint16(data_type): @@ -5818,7 +5818,7 @@ def is_uint16(data_type): data_type : DataType The data type to check. """ - return is_unsigned_integer(data_type.id) and bit_width(data_type.id) == 16 + return c_is_unsigned_integer(data_type.id) and c_bit_width(data_type.id) == 16 def is_uint32(data_type): @@ -5830,7 +5830,7 @@ def is_uint32(data_type): data_type : DataType The data type to check. """ - return is_unsigned_integer(data_type.id) and bit_width(data_type.id) == 32 + return c_is_unsigned_integer(data_type.id) and c_bit_width(data_type.id) == 32 def is_uint64(data_type): @@ -5842,7 +5842,7 @@ def is_uint64(data_type): data_type : DataType The data type to check. """ - return is_unsigned_integer(data_type.id) and bit_width(data_type.id) == 64 + return c_is_unsigned_integer(data_type.id) and c_bit_width(data_type.id) == 64 def is_float16(data_type): @@ -5854,7 +5854,7 @@ def is_float16(data_type): data_type : DataType The data type to check. """ - return is_floating(data_type.id) and bit_width(data_type.id) == 16 + return c_is_floating(data_type.id) and c_bit_width(data_type.id) == 16 def is_float32(data_type): @@ -5866,7 +5866,7 @@ def is_float32(data_type): data_type : DataType The data type to check. """ - return is_floating(data_type.id) and bit_width(data_type.id) == 32 + return c_is_floating(data_type.id) and c_bit_width(data_type.id) == 32 def is_float64(data_type): @@ -5878,7 +5878,7 @@ def is_float64(data_type): data_type : DataType The data type to check. """ - return is_floating(data_type.id) and bit_width(data_type.id) == 64 + return c_is_floating(data_type.id) and c_bit_width(data_type.id) == 64 def is_large_list(data_type): @@ -5890,7 +5890,7 @@ def is_large_list(data_type): data_type : DataType The data type to check. """ - return is_list(data_type.id) and offset_bit_width(data_type.id) == 64 + return c_is_list(data_type.id) and c_offset_bit_width(data_type.id) == 64 def is_fixed_size_list(data_type): @@ -5905,7 +5905,7 @@ def is_fixed_size_list(data_type): return data_type.id == Type_FIXED_SIZE_LIST -def _is_list_view(data_type): +def is_list_view(data_type): """ Returns whether the `data_type` is a list view type. @@ -5914,10 +5914,10 @@ def _is_list_view(data_type): data_type : DataType The data type to check. """ - return is_list_view(data_type.id) and offset_bit_width(data_type.id) == 32 + return c_is_list_view(data_type.id) and c_offset_bit_width(data_type.id) == 32 -def _is_large_list_view(data_type): +def is_large_list_view(data_type): """ Returns whether the `data_type` is a large list view type. @@ -5926,7 +5926,7 @@ def _is_large_list_view(data_type): data_type : DataType The data type to check. """ - return is_list_view(data_type.id) and offset_bit_width(data_type.id) == 64 + return c_is_list_view(data_type.id) and c_offset_bit_width(data_type.id) == 64 def is_struct(data_type): @@ -5941,7 +5941,7 @@ def is_struct(data_type): return data_type.id == Type_STRUCT -def _is_union(data_type): +def is_union(data_type): """ Returns whether the `data_type` is a union type. @@ -5950,10 +5950,10 @@ def _is_union(data_type): data_type : DataType The data type to check. """ - return is_union(data_type.id) + return c_is_union(data_type.id) -def _is_nested(data_type): +def is_nested(data_type): """ Returns whether the `data_type` is a nested type. @@ -5962,7 +5962,7 @@ def _is_nested(data_type): data_type : DataType The data type to check. """ - return is_nested(data_type.id) + return c_is_nested(data_type.id) def is_timestamp(data_type): @@ -5974,7 +5974,7 @@ def is_timestamp(data_type): data_type : DataType The data type to check. """ - return is_temporal(data_type.id) and not is_time(data_type.id) and not is_date(data_type.id) + return c_is_temporal(data_type.id) and not c_is_time(data_type.id) and not c_is_date(data_type.id) def is_duration(data_type): @@ -5986,9 +5986,9 @@ def is_duration(data_type): data_type : DataType The data type to check. """ - return is_primitive(data_type.id) and not is_integer(data_type.id) and \ - not is_floating(data_type.id) and not is_temporal(data_type.id) and \ - not is_interval(data_type.id) + return c_is_primitive(data_type.id) and not c_is_integer(data_type.id) and \ + not c_is_floating(data_type.id) and not c_is_temporal(data_type.id) and \ + not c_is_interval(data_type.id) def is_time32(data_type): @@ -6000,7 +6000,7 @@ def is_time32(data_type): data_type : DataType The data type to check. """ - return is_time(data_type.id) and bit_width(data_type.id) == 32 + return c_is_time(data_type.id) and c_bit_width(data_type.id) == 32 def is_time64(data_type): @@ -6012,7 +6012,7 @@ def is_time64(data_type): data_type : DataType The data type to check. """ - return is_time(data_type.id) and bit_width(data_type.id) == 64 + return c_is_time(data_type.id) and c_bit_width(data_type.id) == 64 def is_large_binary(data_type): @@ -6024,7 +6024,7 @@ def is_large_binary(data_type): data_type : DataType The data type to check. """ - return is_binary(data_type.id) and offset_bit_width(data_type.id) == 64 + return c_is_binary(data_type.id) and c_offset_bit_width(data_type.id) == 64 def is_large_string(data_type): @@ -6036,7 +6036,7 @@ def is_large_string(data_type): data_type : DataType The data type to check. """ - return is_string(data_type.id) and offset_bit_width(data_type.id) == 64 + return c_is_string(data_type.id) and c_offset_bit_width(data_type.id) == 64 def is_binary_view(data_type): @@ -6072,7 +6072,7 @@ def is_date32(data_type): data_type : DataType The data type to check. """ - return is_date(data_type.id) and bit_width(data_type.id) == 32 + return c_is_date(data_type.id) and c_bit_width(data_type.id) == 32 def is_date64(data_type): @@ -6084,7 +6084,7 @@ def is_date64(data_type): data_type : DataType The data type to check. """ - return is_date(data_type.id) and bit_width(data_type.id) == 64 + return c_is_date(data_type.id) and c_bit_width(data_type.id) == 64 def is_run_end_encoded(data_type): @@ -6108,7 +6108,7 @@ def is_map(data_type): data_type : DataType The data type to check. """ - return is_var_length_list(data_type.id) and not is_list(data_type.id) + return c_is_var_length_list(data_type.id) and not c_is_list(data_type.id) def is_decimal128(data_type): @@ -6120,7 +6120,7 @@ def is_decimal128(data_type): data_type : DataType The data type to check. """ - return is_decimal(data_type.id) and bit_width(data_type.id) == 128 + return c_is_decimal(data_type.id) and c_bit_width(data_type.id) == 128 def is_decimal256(data_type): @@ -6132,7 +6132,7 @@ def is_decimal256(data_type): data_type : DataType The data type to check. """ - return is_decimal(data_type.id) and bit_width(data_type.id) == 256 + return c_is_decimal(data_type.id) and c_bit_width(data_type.id) == 256 cdef class _ExtensionRegistryNanny(_Weakrefable): diff --git a/python/pyarrow/types.py b/python/pyarrow/types.py index 505768b01415..759e88ac0fac 100644 --- a/python/pyarrow/types.py +++ b/python/pyarrow/types.py @@ -21,36 +21,36 @@ from pyarrow.lib import (is_boolean_value, # noqa is_integer_value, is_float_value, - _is_integer as is_integer, - _is_signed_integer as is_signed_integer, - _is_unsigned_integer as is_unsigned_integer, - _is_floating as is_floating, - _is_numeric as is_numeric, - _is_decimal as is_decimal, - _is_run_end_type as is_run_end_type, - _is_primitive as is_primitive, - _is_base_binary_like as is_base_binary_like, - _is_binary_like as is_binary_like, - _is_large_binary_like as is_large_binary_like, - _is_binary as is_binary, - _is_string as is_string, _is_string as is_unicode, - _is_temporal as is_temporal, - _is_time as is_time, - _is_date as is_date, - _is_interval as is_interval, - _is_dictionary as is_dictionary, - _is_fixed_size_binary as is_fixed_size_binary, - _is_fixed_width as is_fixed_width, - _is_var_length_list as is_var_length_list, - _is_list as is_list, - _is_list_like as is_list_like, - _is_var_length_list_like as is_var_length_list_like, - _is_list_view as is_list_view, - _is_large_list_view as is_large_list_view, - _is_nested as is_nested, - _is_union as is_union, - _bit_width as bit_width, - _offset_bit_width as offset_bit_width, + is_integer, + is_signed_integer, + is_unsigned_integer, + is_floating, + is_numeric, + is_decimal, + is_run_end_encoded, + is_primitive, + is_base_binary_like, + is_binary_like, + is_large_binary_like, + is_binary, + is_string, is_string as is_unicode, + is_temporal, + is_time, + is_date, + is_interval, + is_dictionary, + is_fixed_size_binary, + is_fixed_width, + is_var_length_list, + is_list, + is_list_like, + is_var_length_list_like, + is_list_view, + is_large_list_view, + is_nested, + is_union, + bit_width_length, + offset_bit_width_length, is_run_end_encoded, is_null, is_boolean, From 3bbc1e29661af0801c1c3c7e143a08a741d789b3 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Sat, 11 May 2024 00:22:59 +0900 Subject: [PATCH 28/28] fix code for test_numpy_array_protocol --- python/pyarrow/array.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 946c82b25824..9bfe9750b061 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -1531,7 +1531,7 @@ cdef class Array(_PandasConvertible): return np.array(values, dtype=dtype, copy=False) values = self.to_numpy(zero_copy_only=False) - if copy is True and is_numeric(self.type.id) and self.null_count == 0: + if copy is True and is_numeric(self.type) and self.null_count == 0: # to_numpy did not yet make a copy (is_numeric = integer/floats, no decimal) return np.array(values, dtype=dtype, copy=True)