From 800fb5aebb9f5f66047a772968c00584bb06dee5 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 24 Sep 2026 14:54:42 +0200 Subject: [PATCH 1/2] GH-51302: [Python] Avoid deprecated .values call in pandas->pyarrow conversion --- python/pyarrow/array.pxi | 10 +++--- python/pyarrow/pandas-shim.pxi | 3 ++ python/pyarrow/pandas_compat.py | 2 +- python/pyarrow/tests/test_pandas.py | 55 +++++++++++++++++++++++------ python/pyarrow/tests/test_schema.py | 9 +++++ 5 files changed, 62 insertions(+), 17 deletions(-) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 3060c533255b..da1fb6e11ec8 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -5393,16 +5393,16 @@ cdef object get_array_class_from_type( return _array_classes[data_type.id()] -cdef object get_values(object obj, bint* is_series): +cdef object get_values(object obj, bint* is_pandas_object): if pandas_api.is_series(obj) or pandas_api.is_index(obj): result = pandas_api.get_values(obj) - is_series[0] = True + is_pandas_object[0] = True elif isinstance(obj, np.ndarray): result = obj - is_series[0] = False + is_pandas_object[0] = False else: - result = pandas_api.series(obj, copy=False).values - is_series[0] = False + result = pandas_api.get_values(pandas_api.series(obj, copy=False)) + is_pandas_object[0] = False return result diff --git a/python/pyarrow/pandas-shim.pxi b/python/pyarrow/pandas-shim.pxi index 603b2b09d40b..bd29eb032221 100644 --- a/python/pyarrow/pandas-shim.pxi +++ b/python/pyarrow/pandas-shim.pxi @@ -239,6 +239,9 @@ cdef class _PandasAPIShim(object): if isinstance(obj.dtype, (self.pd.api.types.IntervalDtype, self.pd.api.types.PeriodDtype)): return obj.array + elif isinstance(obj.dtype, self.pd.api.types.DatetimeTZDtype): + # get the underlying to numpy datetime64 UTC values + return obj.to_numpy("datetime64[{}]".format(obj.dtype.unit)) return obj.values cdef _PandasAPIShim pandas_api = _PandasAPIShim() diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index 36d336d51351..95455bb16510 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -578,7 +578,7 @@ def dataframe_to_types(df, preserve_index, columns=None): types = [] # If pandas knows type, skip conversion for c in columns_to_convert: - values = c.values + values = _pandas_api.get_values(c) if _pandas_api.is_categorical(values): type_ = pa.array(c, from_pandas=True).type elif _pandas_api.is_extension_array_dtype(values): diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index dd20a0aa9777..a36fffedab88 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -3334,21 +3334,29 @@ def _fully_loaded_dataframe_example(): c1 = pd.date_range('2000-01-01', periods=10) data = { - 0: c1, - 1: c1.tz_localize('utc'), - 2: c1.tz_localize('US/Eastern'), - 3: c1[::2].tz_localize('utc').repeat(2).astype('category'), - 4: ['foo', 'bar'] * 5, - 5: pd.Series(['foo', 'bar'] * 5).astype('category').values, - 6: [True, False] * 5, - 7: np.random.randn(10), - 8: np.random.randint(0, 100, size=10), - 9: pd.period_range('2013', periods=10, freq='M'), - 10: pd.interval_range(start=1, freq=1, periods=10), + "col0": c1, + "col1": c1.tz_localize('utc'), + "col2": c1.tz_localize('US/Eastern'), + "col3": c1[::2].tz_localize('utc').repeat(2).astype('category'), + "col4": ['foo', 'bar'] * 5, + "col5": pd.Series(['foo', 'bar'] * 5).astype('category').values, + "col6": [True, False] * 5, + "col7": np.random.randn(10), + "col8": np.random.randint(0, 100, size=10), + "col9": pd.period_range('2013', periods=10, freq='M'), + "col10": pd.interval_range(start=1, freq=1, periods=10), } return pd.DataFrame(data, index=index) +def test_roundtrip_fully_loaded_dataframe_example(): + df = _fully_loaded_dataframe_example() + expected = df.copy() + expected["col3"] = df["col3"].cat.rename_categories( + df["col3"].cat.categories.tz_convert(None)) + _check_pandas_roundtrip(df, preserve_index=None, expected=expected) + + @pytest.mark.parametrize('columns', ([b'foo'], ['foo'])) def test_roundtrip_with_bytes_unicode(columns): df = pd.DataFrame(columns=columns) @@ -5327,3 +5335,28 @@ def test_json_unserializable_pd_df_attrs(): pd_metadata = json.loads(df_table.schema.metadata[b"pandas"]) assert not pd_metadata["attributes"] + + +def test_pandas_array_likes_with_extension_arrays(): + # https://github.com/apache/arrow/issues/51302 + dtidx = pd.date_range("2025-01-01", periods=10) + + arrays = [ + (dtidx.array.tz_localize("UTC"), None), + (dtidx.array.tz_localize("Europe/Paris"), None), + (dtidx.array.tz_localize("Europe/Paris").as_unit("s"), None), + (pd.period_range('2013', periods=10, freq='M'), None), + (pd.interval_range(start=1, freq=1, periods=10), None), + (pd.array([1, 2, 3], dtype="Int32"), np.array([1, 2, 3], dtype="int32")), + (pd.array([1.1, 2.2, None], dtype="Float64"), + np.array([1.1, 2.2, np.nan], dtype="float64")), + (pd.array([True, False, True], dtype="boolean"), + np.array([True, False, True], dtype="bool")), + (pd.array(["a", "b", None], dtype="string"), + np.array(["a", "b", None], dtype="object")), + ] + + for arr in arrays: + for box in [pd.array, pd.Index, pd.Series]: + _check_array_roundtrip(box(arr[0]), expected=pd.Series( + arr[1]) if arr[1] is not None else None) diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py index 27b6419f9add..93519fc01730 100644 --- a/python/pyarrow/tests/test_schema.py +++ b/python/pyarrow/tests/test_schema.py @@ -27,6 +27,7 @@ import pyarrow as pa import pyarrow.tests.util as test_util +from pyarrow.tests.test_pandas import _fully_loaded_dataframe_example def test_schema_constructor_errors(): @@ -719,6 +720,14 @@ def test_schema_from_pandas(): assert schema == expected +@pytest.mark.pandas +def test_schema_from_pandas_all_types(): + df = _fully_loaded_dataframe_example() + schema = pa.Schema.from_pandas(df) + expected = pa.Table.from_pandas(df).schema + assert schema == expected + + def test_schema_sizeof(): schema = pa.schema([ pa.field('foo', pa.int32()), From fe46da223e2bc0e89078acded8e1097443c878e8 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 24 Sep 2026 15:38:12 +0200 Subject: [PATCH 2/2] avoid remaining .values in multithreaded conversion --- python/pyarrow/pandas_compat.py | 13 ++++++++----- python/pyarrow/tests/test_pandas.py | 2 ++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index 95455bb16510..495df5645f3f 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -646,10 +646,13 @@ def convert_column(col, field): f"had {result.null_count} null values") return result - def _can_definitely_zero_copy(arr): - return (isinstance(arr, np.ndarray) and - arr.flags.contiguous and - issubclass(arr.dtype.type, np.integer)) + def _can_definitely_zero_copy(ser): + if isinstance(ser.dtype, np.dtype): + arr = ser.values + return (isinstance(arr, np.ndarray) and + arr.flags.contiguous and + issubclass(arr.dtype.type, np.integer)) + return False if nthreads == 1: arrays = [convert_column(c, f) @@ -658,7 +661,7 @@ def _can_definitely_zero_copy(arr): arrays = [] with futures.ThreadPoolExecutor(nthreads) as executor: for c, f in zip(columns_to_convert, convert_fields): - if _can_definitely_zero_copy(c.values): + if _can_definitely_zero_copy(c): arrays.append(convert_column(c, f)) else: arrays.append(executor.submit(convert_column, c, f)) diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index a36fffedab88..6fc9925eb012 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -3355,6 +3355,8 @@ def test_roundtrip_fully_loaded_dataframe_example(): expected["col3"] = df["col3"].cat.rename_categories( df["col3"].cat.categories.tz_convert(None)) _check_pandas_roundtrip(df, preserve_index=None, expected=expected) + _check_pandas_roundtrip(df, preserve_index=None, + expected=expected, use_threads=True) @pytest.mark.parametrize('columns', ([b'foo'], ['foo']))