diff --git a/.github/workflows/python_wheel_debug_symbols.yml b/.github/workflows/python_wheel_debug_symbols.yml new file mode 100644 index 000000000000..1d241240ed62 --- /dev/null +++ b/.github/workflows/python_wheel_debug_symbols.yml @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Python Wheel Debug Symbols + +on: + workflow_dispatch: + push: + paths: + - '.github/workflows/python_wheel_debug_symbols.yml' + - 'ci/scripts/python_wheel_linux_debug_symbols_test.sh' + - 'ci/scripts/python_wheel_xlinux_build.sh' + pull_request: + paths: + - '.github/workflows/python_wheel_debug_symbols.yml' + - 'ci/scripts/python_wheel_linux_debug_symbols_test.sh' + - 'ci/scripts/python_wheel_xlinux_build.sh' + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + verify: + name: ${{ matrix.kind }} backtraces + runs-on: ubuntu-latest + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - kind: manylinux + build-service: python-wheel-manylinux-2-28 + test-image: python:3.11-bookworm + install-gdb: apt-get update && apt-get install -y gdb + - kind: musllinux + build-service: python-wheel-musllinux-1-2 + test-image: python:3.11-alpine3.23 + install-gdb: apk add --no-cache bash gdb + env: + ARCHERY_DEBUG: 1 + PYTHON: '3.11' + PYTHON_ABI_TAG: cp311 + ULIMIT_CORE: 0 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + fetch-depth: 0 + submodules: recursive + - name: Free up disk space + run: ci/scripts/util_free_space.sh + - uses: actions/setup-python@v7 + with: + python-version: '3.12' + - name: Install Archery + run: python -m pip install -e 'dev/archery[docker]' + - name: Build this revision's wheel and debug-symbol bundle + # Reuse the public wheel build image, but run the checked-out build script + # against this revision's sources. No Crossbow submission or upload. + run: | + archery docker run --no-build \ + -e CMAKE_BUILD_PARALLEL_LEVEL=2 \ + ${{ matrix.build-service }} + - name: Verify stripped and symbolized backtraces + run: | + mkdir -p verification + docker run --rm \ + --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --volume "${GITHUB_WORKSPACE}:/arrow" \ + ${{ matrix.test-image }} \ + sh -ec '${{ matrix.install-gdb }} && + python -m pip install numpy && + /arrow/ci/scripts/python_wheel_linux_debug_symbols_test.sh \ + /arrow/python/repaired_wheels/*.whl \ + /arrow/python/debug_symbols/*.tar.gz' \ + 2>&1 | tee verification/backtraces.txt + - name: Preserve wheels, symbols, and verification logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: debug-symbol-verification-${{ matrix.kind }} + path: | + python/repaired_wheels/*.whl + python/debug_symbols/*.tar.gz + verification/*.txt diff --git a/ci/scripts/python_wheel_linux_debug_symbols_test.sh b/ci/scripts/python_wheel_linux_debug_symbols_test.sh new file mode 100755 index 000000000000..d6c4ded55609 --- /dev/null +++ b/ci/scripts/python_wheel_linux_debug_symbols_test.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -o pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +wheel=$(realpath "$1") +debug_symbols=$(realpath "$2") + +for command in gdb grep tar; do + if ! command -v "${command}" > /dev/null; then + echo "Required command not found: ${command}" >&2 + exit 1 + fi +done + +work_dir=$(mktemp -d) +trap 'rm -rf "${work_dir}"' EXIT +site_packages="${work_dir}/site-packages" +mkdir -p "${site_packages}" + +python -m pip install --no-deps --target "${site_packages}" "${wheel}" +python -c "import numpy" + +# GH-40749: preserve non-public frames, as in the GH-38770 backtrace. +# That bug is fixed, so abort in a scalar UDF to exercise the compute executor +# reliably on current wheels instead of relying on the old reproducer crashing. +ulimit -c 0 +cat > "${work_dir}/backtrace_reproducer.py" <<'PYTHON' +import os + +import pyarrow as pa +import pyarrow.compute as pc + + +def crash(ctx, value): + os.abort() + + +pc.register_scalar_function( + crash, + "arrow_backtrace_crash", + { + "summary": "Crash for backtrace validation", + "description": "Crash for backtrace validation", + }, + {"value": pa.int64()}, + pa.int64(), +) +pc.call_function("arrow_backtrace_crash", [pa.array([1])]) +PYTHON + +run_gdb() { + local output=$1 + # Only use the supplied bundle, not system debug files or debuginfod. + if ! DEBUGINFOD_URLS= \ + PYTHONPATH="${site_packages}${PYTHONPATH:+:${PYTHONPATH}}" \ + gdb --nx --batch --quiet \ + -ex "set debug-file-directory ${work_dir}/no-system-debug" \ + -ex run \ + -ex "thread apply all backtrace" \ + --args python "${work_dir}/backtrace_reproducer.py" \ + > "${output}" 2>&1; then + cat "${output}" >&2 + return 1 + fi + cat "${output}" + if ! grep -F "received signal SIGABRT" "${output}"; then + echo "The reproducer did not reach the expected abort" >&2 + return 1 + fi +} + +# Verify that the main wheel is stripped of non-public function symbols. +echo "=== Backtrace without the debug-symbol bundle ===" +run_gdb "${work_dir}/stripped-backtrace.txt" +if grep -F "ScalarExecutor::Execute" "${work_dir}/stripped-backtrace.txt"; then + echo "The wheel still contains non-public function symbols" >&2 + exit 1 +fi + +# GDB searches an adjacent .debug directory for GNU debuglink targets. +tar -xzf "${debug_symbols}" -C "${site_packages}" +echo "=== Backtrace with the debug-symbol bundle ===" +run_gdb "${work_dir}/symbolized-backtrace.txt" + +for symbol in \ + "PythonUdfExec" \ + "ScalarExecutor::Execute" \ + "FunctionExecutorImpl::Execute" \ + "ExecuteInternal"; do + if ! grep -F "${symbol}" "${work_dir}/symbolized-backtrace.txt"; then + echo "Missing expected symbol in GDB backtrace: ${symbol}" >&2 + cat "${work_dir}/symbolized-backtrace.txt" >&2 + exit 1 + fi +done + +echo "The stripped wheel and separate debug-symbol bundle produced the expected backtrace." diff --git a/ci/scripts/python_wheel_xlinux_build.sh b/ci/scripts/python_wheel_xlinux_build.sh index 0a5d63affc2e..61d407a6f0e4 100755 --- a/ci/scripts/python_wheel_xlinux_build.sh +++ b/ci/scripts/python_wheel_xlinux_build.sh @@ -51,6 +51,8 @@ rm -rf /tmp/arrow-build rm -rf /arrow/python/dist rm -rf /arrow/python/build rm -rf /arrow/python/repaired_wheels +rm -rf /arrow/python/debug_symbols +rm -rf /tmp/pyarrow-debug-symbols rm -rf /arrow/python/pyarrow/*.so rm -rf /arrow/python/pyarrow/*.so.* @@ -193,8 +195,10 @@ if command -v sccache &> /dev/null; then sccache --show-stats fi -echo "=== Strip symbols from wheel ===" +echo "=== Extract debug symbols and strip wheel binaries ===" mkdir -p dist/temp-fix-wheel +mkdir -p debug_symbols +mkdir -p /tmp/pyarrow-debug-symbols/pyarrow/.debug mv dist/pyarrow-*.whl dist/temp-fix-wheel pushd dist/temp-fix-wheel @@ -203,16 +207,50 @@ wheel_name=$(ls pyarrow-*.whl) unzip "$wheel_name" rm "$wheel_name" for filename in pyarrow/*.so pyarrow/*.so.*; do - echo "Stripping debug symbols from: $filename"; - strip --strip-debug "$filename" + debug_file="/tmp/pyarrow-debug-symbols/pyarrow/.debug/$(basename "$filename").debug" + echo "Extracting debug symbols from: $filename" + objcopy --only-keep-debug "$filename" "$debug_file" + strip --strip-unneeded "$filename" + objcopy --add-gnu-debuglink="$debug_file" "$filename" done +cat > /tmp/pyarrow-debug-symbols/README.txt <<'EOF' +This archive contains GNU debug symbols for one specific PyArrow wheel. +It must only be used with the wheel whose full platform-qualified name is +included in this archive's filename. + +To make the symbols available to GDB, extract the archive into the Python +environment's site-packages directory. The resulting layout must be: + + site-packages/pyarrow/.debug/*.debug + +For example: + + site_packages=$(python -c \ + 'import sysconfig; print(sysconfig.get_paths()["purelib"])') + tar -xzf pyarrow-*-debug-symbols.tar.gz -C "${site_packages}" +EOF +# Keep the debug files out of the wheel. They are published as a separate +# optional artifact for post-mortem debugging and symbolizing core dumps. +tar -C /tmp/pyarrow-debug-symbols \ + -czf ../../debug_symbols/pyarrow-debug-symbols.tar.gz \ + README.txt pyarrow/.debug # Zip wheel again after stripping symbols zip -r "$wheel_name" . mv "$wheel_name" .. popd rm -rf dist/temp-fix-wheel +rm -rf /tmp/pyarrow-debug-symbols echo "=== (${PYTHON_VERSION}) Tag the wheel with ${LINUX_WHEEL_KIND}${LINUX_WHEEL_VERSION} ===" auditwheel repair dist/pyarrow-*.whl -w repaired_wheels + +repaired_wheels=(repaired_wheels/*.whl) +if [ "${#repaired_wheels[@]}" -ne 1 ]; then + echo "Expected one repaired wheel, found ${#repaired_wheels[@]}" >&2 + exit 1 +fi +repaired_wheel_name=$(basename "${repaired_wheels[0]}") +mv debug_symbols/pyarrow-debug-symbols.tar.gz \ + "debug_symbols/${repaired_wheel_name%.whl}-debug-symbols.tar.gz" popd diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml index 16ee668bff43..c233a0d899e6 100644 --- a/dev/tasks/python-wheels/github.linux.yml +++ b/dev/tasks/python-wheels/github.linux.yml @@ -89,6 +89,11 @@ jobs: name: wheel path: arrow/python/repaired_wheels/*.whl + - uses: actions/upload-artifact@v4 + with: + name: debug-symbols + path: arrow/python/debug_symbols/*.tar.gz + # TODO(kszucs): auditwheel show - name: Test wheel shell: bash @@ -143,7 +148,10 @@ jobs: -e TEST_WHEELS=1 \ ubuntu-verify-rc - {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} + {{ macros.github_upload_releases([ + "arrow/python/repaired_wheels/*.whl", + "arrow/python/debug_symbols/*.tar.gz" + ])|indent }} {{ macros.github_upload_wheel_scientific_python("arrow/python/repaired_wheels/*.whl")|indent }} {% if arrow.is_default_branch() %}