GH-51302: [Python] Avoid deprecated .values call in pandas->pyarrow conversion

python/pyarrow/array.pxi

@@ -5393,16 +5393,16 @@ cdef object get_array_class_from_type(
         return _array_classes[data_type.id()]
 
 
-cdef object get_values(object obj, bint* is_series):
+cdef object get_values(object obj, bint* is_pandas_object):
     if pandas_api.is_series(obj) or pandas_api.is_index(obj):
         result = pandas_api.get_values(obj)
-        is_series[0] = True
+        is_pandas_object[0] = True
     elif isinstance(obj, np.ndarray):
         result = obj
-        is_series[0] = False
+        is_pandas_object[0] = False
     else:
-        result = pandas_api.series(obj, copy=False).values
-        is_series[0] = False
+        result = pandas_api.get_values(pandas_api.series(obj, copy=False))
+        is_pandas_object[0] = False
 
     return result
 

python/pyarrow/pandas-shim.pxi

@@ -239,6 +239,9 @@ cdef class _PandasAPIShim(object):
         if isinstance(obj.dtype, (self.pd.api.types.IntervalDtype,
                                   self.pd.api.types.PeriodDtype)):
             return obj.array
+        elif isinstance(obj.dtype, self.pd.api.types.DatetimeTZDtype):
+            # get the underlying to numpy datetime64 UTC values
+            return obj.to_numpy("datetime64[{}]".format(obj.dtype.unit))
         return obj.values
 
 cdef _PandasAPIShim pandas_api = _PandasAPIShim()

python/pyarrow/pandas_compat.py

@@ -578,7 +578,7 @@ def dataframe_to_types(df, preserve_index, columns=None):
     types = []
     # If pandas knows type, skip conversion
     for c in columns_to_convert:
-        values = c.values
+        values = _pandas_api.get_values(c)
         if _pandas_api.is_categorical(values):
             type_ = pa.array(c, from_pandas=True).type
         elif _pandas_api.is_extension_array_dtype(values):
@@ -646,10 +646,13 @@ def convert_column(col, field):
                              f"had {result.null_count} null values")
         return result
 
-    def _can_definitely_zero_copy(arr):
-        return (isinstance(arr, np.ndarray) and
-                arr.flags.contiguous and
-                issubclass(arr.dtype.type, np.integer))
+    def _can_definitely_zero_copy(ser):
+        if isinstance(ser.dtype, np.dtype):
+            arr = ser.values
+            return (isinstance(arr, np.ndarray) and
+                    arr.flags.contiguous and
+                    issubclass(arr.dtype.type, np.integer))
+        return False
 
     if nthreads == 1:
         arrays = [convert_column(c, f)
@@ -658,7 +661,7 @@ def _can_definitely_zero_copy(arr):
         arrays = []
         with futures.ThreadPoolExecutor(nthreads) as executor:
             for c, f in zip(columns_to_convert, convert_fields):
-                if _can_definitely_zero_copy(c.values):
+                if _can_definitely_zero_copy(c):
                     arrays.append(convert_column(c, f))
                 else:
                     arrays.append(executor.submit(convert_column, c, f))

python/pyarrow/tests/test_pandas.py

@@ -3334,21 +3334,31 @@ def _fully_loaded_dataframe_example():
 
     c1 = pd.date_range('2000-01-01', periods=10)
     data = {
-        0: c1,
-        1: c1.tz_localize('utc'),
-        2: c1.tz_localize('US/Eastern'),
-        3: c1[::2].tz_localize('utc').repeat(2).astype('category'),
-        4: ['foo', 'bar'] * 5,
-        5: pd.Series(['foo', 'bar'] * 5).astype('category').values,
-        6: [True, False] * 5,
-        7: np.random.randn(10),
-        8: np.random.randint(0, 100, size=10),
-        9: pd.period_range('2013', periods=10, freq='M'),
-        10: pd.interval_range(start=1, freq=1, periods=10),
+        "col0": c1,
+        "col1": c1.tz_localize('utc'),
+        "col2": c1.tz_localize('US/Eastern'),
+        "col3": c1[::2].tz_localize('utc').repeat(2).astype('category'),
+        "col4": ['foo', 'bar'] * 5,
+        "col5": pd.Series(['foo', 'bar'] * 5).astype('category').values,
+        "col6": [True, False] * 5,
+        "col7": np.random.randn(10),
+        "col8": np.random.randint(0, 100, size=10),
+        "col9": pd.period_range('2013', periods=10, freq='M'),
+        "col10": pd.interval_range(start=1, freq=1, periods=10),
     }
     return pd.DataFrame(data, index=index)
 
 
+def test_roundtrip_fully_loaded_dataframe_example():
+    df = _fully_loaded_dataframe_example()
+    expected = df.copy()
+    expected["col3"] = df["col3"].cat.rename_categories(
+        df["col3"].cat.categories.tz_convert(None))
+    _check_pandas_roundtrip(df, preserve_index=None, expected=expected)
+    _check_pandas_roundtrip(df, preserve_index=None,
+                            expected=expected, use_threads=True)
+
+
 @pytest.mark.parametrize('columns', ([b'foo'], ['foo']))
 def test_roundtrip_with_bytes_unicode(columns):
     df = pd.DataFrame(columns=columns)
@@ -5327,3 +5337,28 @@ def test_json_unserializable_pd_df_attrs():
     pd_metadata = json.loads(df_table.schema.metadata[b"pandas"])
 
     assert not pd_metadata["attributes"]
+
+
+def test_pandas_array_likes_with_extension_arrays():
+    # https://github.com/apache/arrow/issues/51302
+    dtidx = pd.date_range("2025-01-01", periods=10)
+
+    arrays = [
+        (dtidx.array.tz_localize("UTC"), None),
+        (dtidx.array.tz_localize("Europe/Paris"), None),
+        (dtidx.array.tz_localize("Europe/Paris").as_unit("s"), None),
+        (pd.period_range('2013', periods=10, freq='M'), None),
+        (pd.interval_range(start=1, freq=1, periods=10), None),
+        (pd.array([1, 2, 3], dtype="Int32"), np.array([1, 2, 3], dtype="int32")),
+        (pd.array([1.1, 2.2, None], dtype="Float64"),
+         np.array([1.1, 2.2, np.nan], dtype="float64")),
+        (pd.array([True, False, True], dtype="boolean"),
+         np.array([True, False, True], dtype="bool")),
+        (pd.array(["a", "b", None], dtype="string"),
+         np.array(["a", "b", None], dtype="object")),
+    ]
+
+    for arr in arrays:
+        for box in [pd.array, pd.Index, pd.Series]:
+            _check_array_roundtrip(box(arr[0]), expected=pd.Series(
+                arr[1]) if arr[1] is not None else None)

python/pyarrow/tests/test_schema.py

@@ -27,6 +27,7 @@
 import pyarrow as pa
 
 import pyarrow.tests.util as test_util
+from pyarrow.tests.test_pandas import _fully_loaded_dataframe_example
 
 
 def test_schema_constructor_errors():
@@ -719,6 +720,14 @@ def test_schema_from_pandas():
         assert schema == expected
 
 
+@pytest.mark.pandas
+def test_schema_from_pandas_all_types():
+    df = _fully_loaded_dataframe_example()
+    schema = pa.Schema.from_pandas(df)
+    expected = pa.Table.from_pandas(df).schema
+    assert schema == expected
+
+
 def test_schema_sizeof():
     schema = pa.schema([
         pa.field('foo', pa.int32()),