GH-40749: [Python][Packaging] Publish separate wheel debug symbols

.github/workflows/python_wheel_debug_symbols.yml

@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Python Wheel Debug Symbols
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - '.github/workflows/python_wheel_debug_symbols.yml'
+      - 'ci/scripts/python_wheel_linux_debug_symbols_test.sh'
+      - 'ci/scripts/python_wheel_xlinux_build.sh'
+  pull_request:
+    paths:
+      - '.github/workflows/python_wheel_debug_symbols.yml'
+      - 'ci/scripts/python_wheel_linux_debug_symbols_test.sh'
+      - 'ci/scripts/python_wheel_xlinux_build.sh'
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  verify:
+    name: ${{ matrix.kind }} backtraces
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - kind: manylinux
+            build-service: python-wheel-manylinux-2-28
+            test-image: python:3.11-bookworm
+            install-gdb: apt-get update && apt-get install -y gdb
+          - kind: musllinux
+            build-service: python-wheel-musllinux-1-2
+            test-image: python:3.11-alpine3.23
+            install-gdb: apk add --no-cache bash gdb
+    env:
+      ARCHERY_DEBUG: 1
+      PYTHON: '3.11'
+      PYTHON_ABI_TAG: cp311
+      ULIMIT_CORE: 0
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+          submodules: recursive
+      - name: Free up disk space
+        run: ci/scripts/util_free_space.sh
+      - uses: actions/setup-python@v7
+        with:
+          python-version: '3.12'
+      - name: Install Archery
+        run: python -m pip install -e 'dev/archery[docker]'
+      - name: Build this revision's wheel and debug-symbol bundle
+        # Reuse the public wheel build image, but run the checked-out build script
+        # against this revision's sources. No Crossbow submission or upload.
+        run: |
+          archery docker run --no-build \
+            -e CMAKE_BUILD_PARALLEL_LEVEL=2 \
+            ${{ matrix.build-service }}
+      - name: Verify stripped and symbolized backtraces
+        run: |
+          mkdir -p verification
+          docker run --rm \
+            --cap-add=SYS_PTRACE \
+            --security-opt seccomp=unconfined \
+            --volume "${GITHUB_WORKSPACE}:/arrow" \
+            ${{ matrix.test-image }} \
+            sh -ec '${{ matrix.install-gdb }} &&
+              python -m pip install numpy &&
+              /arrow/ci/scripts/python_wheel_linux_debug_symbols_test.sh \
+                /arrow/python/repaired_wheels/*.whl \
+                /arrow/python/debug_symbols/*.tar.gz' \
+            2>&1 | tee verification/backtraces.txt
+      - name: Preserve wheels, symbols, and verification logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: debug-symbol-verification-${{ matrix.kind }}
+          path: |
+            python/repaired_wheels/*.whl
+            python/debug_symbols/*.tar.gz
+            verification/*.txt

ci/scripts/python_wheel_linux_debug_symbols_test.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+set -o pipefail
+
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <pyarrow-wheel> <debug-symbols.tar.gz>" >&2
+  exit 1
+fi
+
+wheel=$(realpath "$1")
+debug_symbols=$(realpath "$2")
+
+for command in gdb grep tar; do
+  if ! command -v "${command}" > /dev/null; then
+    echo "Required command not found: ${command}" >&2
+    exit 1
+  fi
+done
+
+work_dir=$(mktemp -d)
+trap 'rm -rf "${work_dir}"' EXIT
+site_packages="${work_dir}/site-packages"
+mkdir -p "${site_packages}"
+
+python -m pip install --no-deps --target "${site_packages}" "${wheel}"
+python -c "import numpy"
+
+# GH-40749: preserve non-public frames, as in the GH-38770 backtrace.
+# That bug is fixed, so abort in a scalar UDF to exercise the compute executor
+# reliably on current wheels instead of relying on the old reproducer crashing.
+ulimit -c 0
+cat > "${work_dir}/backtrace_reproducer.py" <<'PYTHON'
+import os
+
+import pyarrow as pa
+import pyarrow.compute as pc
+
+
+def crash(ctx, value):
+    os.abort()
+
+
+pc.register_scalar_function(
+    crash,
+    "arrow_backtrace_crash",
+    {
+        "summary": "Crash for backtrace validation",
+        "description": "Crash for backtrace validation",
+    },
+    {"value": pa.int64()},
+    pa.int64(),
+)
+pc.call_function("arrow_backtrace_crash", [pa.array([1])])
+PYTHON
+
+run_gdb() {
+  local output=$1
+  # Only use the supplied bundle, not system debug files or debuginfod.
+  if ! DEBUGINFOD_URLS= \
+    PYTHONPATH="${site_packages}${PYTHONPATH:+:${PYTHONPATH}}" \
+    gdb --nx --batch --quiet \
+      -ex "set debug-file-directory ${work_dir}/no-system-debug" \
+      -ex run \
+      -ex "thread apply all backtrace" \
+      --args python "${work_dir}/backtrace_reproducer.py" \
+      > "${output}" 2>&1; then
+    cat "${output}" >&2
+    return 1
+  fi
+  cat "${output}"
+  if ! grep -F "received signal SIGABRT" "${output}"; then
+    echo "The reproducer did not reach the expected abort" >&2
+    return 1
+  fi
+}
+
+# Verify that the main wheel is stripped of non-public function symbols.
+echo "=== Backtrace without the debug-symbol bundle ==="
+run_gdb "${work_dir}/stripped-backtrace.txt"
+if grep -F "ScalarExecutor::Execute" "${work_dir}/stripped-backtrace.txt"; then
+  echo "The wheel still contains non-public function symbols" >&2
+  exit 1
+fi
+
+# GDB searches an adjacent .debug directory for GNU debuglink targets.
+tar -xzf "${debug_symbols}" -C "${site_packages}"
+echo "=== Backtrace with the debug-symbol bundle ==="
+run_gdb "${work_dir}/symbolized-backtrace.txt"
+
+for symbol in \
+  "PythonUdfExec" \
+  "ScalarExecutor::Execute" \
+  "FunctionExecutorImpl::Execute" \
+  "ExecuteInternal"; do
+  if ! grep -F "${symbol}" "${work_dir}/symbolized-backtrace.txt"; then
+    echo "Missing expected symbol in GDB backtrace: ${symbol}" >&2
+    cat "${work_dir}/symbolized-backtrace.txt" >&2
+    exit 1
+  fi
+done
+
+echo "The stripped wheel and separate debug-symbol bundle produced the expected backtrace."

ci/scripts/python_wheel_xlinux_build.sh

@@ -51,6 +51,8 @@ rm -rf /tmp/arrow-build
 rm -rf /arrow/python/dist
 rm -rf /arrow/python/build
 rm -rf /arrow/python/repaired_wheels
+rm -rf /arrow/python/debug_symbols
+rm -rf /tmp/pyarrow-debug-symbols
 rm -rf /arrow/python/pyarrow/*.so
 rm -rf /arrow/python/pyarrow/*.so.*
 
@@ -193,8 +195,10 @@ if command -v sccache &> /dev/null; then
   sccache --show-stats
 fi
 
-echo "=== Strip symbols from wheel ==="
+echo "=== Extract debug symbols and strip wheel binaries ==="
 mkdir -p dist/temp-fix-wheel
+mkdir -p debug_symbols
+mkdir -p /tmp/pyarrow-debug-symbols/pyarrow/.debug
 mv dist/pyarrow-*.whl dist/temp-fix-wheel
 
 pushd dist/temp-fix-wheel
@@ -203,16 +207,50 @@ wheel_name=$(ls pyarrow-*.whl)
 unzip "$wheel_name"
 rm "$wheel_name"
 for filename in pyarrow/*.so pyarrow/*.so.*; do
-    echo "Stripping debug symbols from: $filename";
-    strip --strip-debug "$filename"
+    debug_file="/tmp/pyarrow-debug-symbols/pyarrow/.debug/$(basename "$filename").debug"
+    echo "Extracting debug symbols from: $filename"
+    objcopy --only-keep-debug "$filename" "$debug_file"
+    strip --strip-unneeded "$filename"
+    objcopy --add-gnu-debuglink="$debug_file" "$filename"
 done
+cat > /tmp/pyarrow-debug-symbols/README.txt <<'EOF'
+This archive contains GNU debug symbols for one specific PyArrow wheel.
+It must only be used with the wheel whose full platform-qualified name is
+included in this archive's filename.
+
+To make the symbols available to GDB, extract the archive into the Python
+environment's site-packages directory. The resulting layout must be:
+
+  site-packages/pyarrow/.debug/*.debug
+
+For example:
+
+  site_packages=$(python -c \
+    'import sysconfig; print(sysconfig.get_paths()["purelib"])')
+  tar -xzf pyarrow-*-debug-symbols.tar.gz -C "${site_packages}"
+EOF
+# Keep the debug files out of the wheel. They are published as a separate
+# optional artifact for post-mortem debugging and symbolizing core dumps.
+tar -C /tmp/pyarrow-debug-symbols \
+    -czf ../../debug_symbols/pyarrow-debug-symbols.tar.gz \
+    README.txt pyarrow/.debug
 # Zip wheel again after stripping symbols
 zip -r "$wheel_name" .
 mv "$wheel_name" ..
 popd
 
 rm -rf dist/temp-fix-wheel
+rm -rf /tmp/pyarrow-debug-symbols
 
 echo "=== (${PYTHON_VERSION}) Tag the wheel with ${LINUX_WHEEL_KIND}${LINUX_WHEEL_VERSION} ==="
 auditwheel repair dist/pyarrow-*.whl -w repaired_wheels
+
+repaired_wheels=(repaired_wheels/*.whl)
+if [ "${#repaired_wheels[@]}" -ne 1 ]; then
+  echo "Expected one repaired wheel, found ${#repaired_wheels[@]}" >&2
+  exit 1
+fi
+repaired_wheel_name=$(basename "${repaired_wheels[0]}")
+mv debug_symbols/pyarrow-debug-symbols.tar.gz \
+   "debug_symbols/${repaired_wheel_name%.whl}-debug-symbols.tar.gz"
 popd

dev/tasks/python-wheels/github.linux.yml

@@ -89,6 +89,11 @@ jobs:
           name: wheel
           path: arrow/python/repaired_wheels/*.whl
 
+      - uses: actions/upload-artifact@v4
+        with:
+          name: debug-symbols
+          path: arrow/python/debug_symbols/*.tar.gz
+
       # TODO(kszucs): auditwheel show
       - name: Test wheel
         shell: bash
@@ -143,7 +148,10 @@ jobs:
             -e TEST_WHEELS=1 \
             ubuntu-verify-rc
 
-      {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }}
+      {{ macros.github_upload_releases([
+          "arrow/python/repaired_wheels/*.whl",
+          "arrow/python/debug_symbols/*.tar.gz"
+        ])|indent }}
       {{ macros.github_upload_wheel_scientific_python("arrow/python/repaired_wheels/*.whl")|indent }}
 
       {% if arrow.is_default_branch() %}