From 4c57365ec19744cc6486c8b8545ffa98b823b969 Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Fri, 25 Sep 2026 18:17:34 +0200 Subject: [PATCH 1/3] GH-40749: [Python][Packaging] Publish separate wheel debug symbols --- ci/docker/python-wheel-manylinux.dockerfile | 2 +- ci/docker/python-wheel-musllinux.dockerfile | 1 + .../python_wheel_linux_debug_symbols_test.sh | 103 ++++++++++++++++++ ci/scripts/python_wheel_xlinux_build.sh | 44 +++++++- dev/tasks/python-wheels/github.linux.yml | 10 +- 5 files changed, 155 insertions(+), 5 deletions(-) create mode 100755 ci/scripts/python_wheel_linux_debug_symbols_test.sh diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile index aa968ad3cd0c..db0d20d5029d 100644 --- a/ci/docker/python-wheel-manylinux.dockerfile +++ b/ci/docker/python-wheel-manylinux.dockerfile @@ -26,7 +26,7 @@ ENV LINUX_WHEEL_KIND='manylinux' ENV LINUX_WHEEL_VERSION=${manylinux} # Install basic dependencies -RUN dnf install -y git flex curl autoconf zip perl-IPC-Cmd perl-Time-Piece wget +RUN dnf install -y git flex curl autoconf zip perl-IPC-Cmd perl-Time-Piece wget xz # A system Python is required for Ninja and vcpkg in this Dockerfile. # On manylinux_2_28 base images, no system Python is installed. diff --git a/ci/docker/python-wheel-musllinux.dockerfile b/ci/docker/python-wheel-musllinux.dockerfile index 0d517caac19c..c5e79eee90fa 100644 --- a/ci/docker/python-wheel-musllinux.dockerfile +++ b/ci/docker/python-wheel-musllinux.dockerfile @@ -38,6 +38,7 @@ RUN apk add --no-cache \ sccache \ unzip \ wget \ + xz \ zip && \ cert-sync /etc/ssl/certs/ca-certificates.crt diff --git a/ci/scripts/python_wheel_linux_debug_symbols_test.sh b/ci/scripts/python_wheel_linux_debug_symbols_test.sh new file mode 100755 index 000000000000..1479f63b90f3 --- /dev/null +++ b/ci/scripts/python_wheel_linux_debug_symbols_test.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -o pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +wheel=$(realpath "$1") +debug_symbols=$(realpath "$2") + +for command in gdb grep tar; do + if ! command -v "${command}" > /dev/null; then + echo "Required command not found: ${command}" >&2 + exit 1 + fi +done + +work_dir=$(mktemp -d) +trap 'rm -rf "${work_dir}"' EXIT +site_packages="${work_dir}/site-packages" +mkdir -p "${site_packages}" + +python -m pip install --no-deps --target "${site_packages}" "${wheel}" +python -c "import numpy" + +cat > "${work_dir}/backtrace_reproducer.py" <<'PYTHON' +import os + +import pyarrow as pa +import pyarrow.compute as pc + + +def crash(ctx, value): + os.abort() + + +pc.register_scalar_function( + crash, + "arrow_backtrace_crash", + { + "summary": "Crash for backtrace validation", + "description": "Crash for backtrace validation", + }, + {"value": pa.int64()}, + pa.int64(), +) +pc.call_function("arrow_backtrace_crash", [pa.array([1])]) +PYTHON + +run_gdb() { + local output=$1 + PYTHONPATH="${site_packages}${PYTHONPATH:+:${PYTHONPATH}}" \ + gdb --batch --quiet \ + -ex run \ + -ex "thread apply all backtrace" \ + --args python "${work_dir}/backtrace_reproducer.py" \ + > "${output}" 2>&1 || true +} + +# Verify that the main wheel is stripped of non-public function symbols. +run_gdb "${work_dir}/stripped-backtrace.txt" +if grep -F "ScalarExecutor::Execute" "${work_dir}/stripped-backtrace.txt"; then + echo "The wheel still contains non-public function symbols" >&2 + exit 1 +fi + +# GDB searches an adjacent .debug directory for GNU debuglink targets. +tar -xJf "${debug_symbols}" -C "${site_packages}" +run_gdb "${work_dir}/symbolized-backtrace.txt" + +for symbol in \ + "PythonUdfExec" \ + "ScalarExecutor::Execute" \ + "FunctionExecutorImpl::Execute" \ + "ExecuteInternal"; do + if ! grep -F "${symbol}" "${work_dir}/symbolized-backtrace.txt"; then + echo "Missing expected symbol in GDB backtrace: ${symbol}" >&2 + cat "${work_dir}/symbolized-backtrace.txt" >&2 + exit 1 + fi +done + +echo "The stripped wheel and separate debug-symbol bundle produced the expected backtrace." diff --git a/ci/scripts/python_wheel_xlinux_build.sh b/ci/scripts/python_wheel_xlinux_build.sh index 0a5d63affc2e..748d30c7e4b2 100755 --- a/ci/scripts/python_wheel_xlinux_build.sh +++ b/ci/scripts/python_wheel_xlinux_build.sh @@ -51,6 +51,8 @@ rm -rf /tmp/arrow-build rm -rf /arrow/python/dist rm -rf /arrow/python/build rm -rf /arrow/python/repaired_wheels +rm -rf /arrow/python/debug_symbols +rm -rf /tmp/pyarrow-debug-symbols rm -rf /arrow/python/pyarrow/*.so rm -rf /arrow/python/pyarrow/*.so.* @@ -193,8 +195,10 @@ if command -v sccache &> /dev/null; then sccache --show-stats fi -echo "=== Strip symbols from wheel ===" +echo "=== Extract debug symbols and strip wheel binaries ===" mkdir -p dist/temp-fix-wheel +mkdir -p debug_symbols +mkdir -p /tmp/pyarrow-debug-symbols/pyarrow/.debug mv dist/pyarrow-*.whl dist/temp-fix-wheel pushd dist/temp-fix-wheel @@ -203,16 +207,50 @@ wheel_name=$(ls pyarrow-*.whl) unzip "$wheel_name" rm "$wheel_name" for filename in pyarrow/*.so pyarrow/*.so.*; do - echo "Stripping debug symbols from: $filename"; - strip --strip-debug "$filename" + debug_file="/tmp/pyarrow-debug-symbols/pyarrow/.debug/$(basename "$filename").debug" + echo "Extracting debug symbols from: $filename" + objcopy --only-keep-debug "$filename" "$debug_file" + strip --strip-unneeded "$filename" + objcopy --add-gnu-debuglink="$debug_file" "$filename" done +cat > /tmp/pyarrow-debug-symbols/README.txt <<'EOF' +This archive contains GNU debug symbols for one specific PyArrow wheel. +It must only be used with the wheel whose full platform-qualified name is +included in this archive's filename. + +To make the symbols available to GDB, extract the archive into the Python +environment's site-packages directory. The resulting layout must be: + + site-packages/pyarrow/.debug/*.debug + +For example: + + site_packages=$(python -c \ + 'import sysconfig; print(sysconfig.get_paths()["purelib"])') + tar -xJf pyarrow-*-debug-symbols.tar.xz -C "${site_packages}" +EOF +# Keep the debug files out of the wheel. They are published as a separate +# optional artifact for post-mortem debugging and symbolizing core dumps. +tar -C /tmp/pyarrow-debug-symbols \ + -cJf ../../debug_symbols/pyarrow-debug-symbols.tar.xz \ + README.txt pyarrow/.debug # Zip wheel again after stripping symbols zip -r "$wheel_name" . mv "$wheel_name" .. popd rm -rf dist/temp-fix-wheel +rm -rf /tmp/pyarrow-debug-symbols echo "=== (${PYTHON_VERSION}) Tag the wheel with ${LINUX_WHEEL_KIND}${LINUX_WHEEL_VERSION} ===" auditwheel repair dist/pyarrow-*.whl -w repaired_wheels + +repaired_wheels=(repaired_wheels/*.whl) +if [ "${#repaired_wheels[@]}" -ne 1 ]; then + echo "Expected one repaired wheel, found ${#repaired_wheels[@]}" >&2 + exit 1 +fi +repaired_wheel_name=$(basename "${repaired_wheels[0]}") +mv debug_symbols/pyarrow-debug-symbols.tar.xz \ + "debug_symbols/${repaired_wheel_name%.whl}-debug-symbols.tar.xz" popd diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml index 16ee668bff43..1f427cfc7bda 100644 --- a/dev/tasks/python-wheels/github.linux.yml +++ b/dev/tasks/python-wheels/github.linux.yml @@ -89,6 +89,11 @@ jobs: name: wheel path: arrow/python/repaired_wheels/*.whl + - uses: actions/upload-artifact@v4 + with: + name: debug-symbols + path: arrow/python/debug_symbols/*.tar.xz + # TODO(kszucs): auditwheel show - name: Test wheel shell: bash @@ -143,7 +148,10 @@ jobs: -e TEST_WHEELS=1 \ ubuntu-verify-rc - {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} + {{ macros.github_upload_releases([ + "arrow/python/repaired_wheels/*.whl", + "arrow/python/debug_symbols/*.tar.xz" + ])|indent }} {{ macros.github_upload_wheel_scientific_python("arrow/python/repaired_wheels/*.whl")|indent }} {% if arrow.is_default_branch() %} From d09821f8ddd562f716941b33f6088db466f311e3 Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Fri, 25 Sep 2026 20:36:33 +0200 Subject: [PATCH 2/3] Use gzip for wheel debug-symbol bundles without adding xz --- ci/docker/python-wheel-manylinux.dockerfile | 2 +- ci/docker/python-wheel-musllinux.dockerfile | 1 - ci/scripts/python_wheel_linux_debug_symbols_test.sh | 4 ++-- ci/scripts/python_wheel_xlinux_build.sh | 8 ++++---- dev/tasks/python-wheels/github.linux.yml | 4 ++-- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile index db0d20d5029d..aa968ad3cd0c 100644 --- a/ci/docker/python-wheel-manylinux.dockerfile +++ b/ci/docker/python-wheel-manylinux.dockerfile @@ -26,7 +26,7 @@ ENV LINUX_WHEEL_KIND='manylinux' ENV LINUX_WHEEL_VERSION=${manylinux} # Install basic dependencies -RUN dnf install -y git flex curl autoconf zip perl-IPC-Cmd perl-Time-Piece wget xz +RUN dnf install -y git flex curl autoconf zip perl-IPC-Cmd perl-Time-Piece wget # A system Python is required for Ninja and vcpkg in this Dockerfile. # On manylinux_2_28 base images, no system Python is installed. diff --git a/ci/docker/python-wheel-musllinux.dockerfile b/ci/docker/python-wheel-musllinux.dockerfile index c5e79eee90fa..0d517caac19c 100644 --- a/ci/docker/python-wheel-musllinux.dockerfile +++ b/ci/docker/python-wheel-musllinux.dockerfile @@ -38,7 +38,6 @@ RUN apk add --no-cache \ sccache \ unzip \ wget \ - xz \ zip && \ cert-sync /etc/ssl/certs/ca-certificates.crt diff --git a/ci/scripts/python_wheel_linux_debug_symbols_test.sh b/ci/scripts/python_wheel_linux_debug_symbols_test.sh index 1479f63b90f3..1ef36a9655c4 100755 --- a/ci/scripts/python_wheel_linux_debug_symbols_test.sh +++ b/ci/scripts/python_wheel_linux_debug_symbols_test.sh @@ -21,7 +21,7 @@ set -e set -o pipefail if [ "$#" -ne 2 ]; then - echo "Usage: $0 " >&2 + echo "Usage: $0 " >&2 exit 1 fi @@ -85,7 +85,7 @@ if grep -F "ScalarExecutor::Execute" "${work_dir}/stripped-backtrace.txt"; then fi # GDB searches an adjacent .debug directory for GNU debuglink targets. -tar -xJf "${debug_symbols}" -C "${site_packages}" +tar -xzf "${debug_symbols}" -C "${site_packages}" run_gdb "${work_dir}/symbolized-backtrace.txt" for symbol in \ diff --git a/ci/scripts/python_wheel_xlinux_build.sh b/ci/scripts/python_wheel_xlinux_build.sh index 748d30c7e4b2..61d407a6f0e4 100755 --- a/ci/scripts/python_wheel_xlinux_build.sh +++ b/ci/scripts/python_wheel_xlinux_build.sh @@ -227,12 +227,12 @@ For example: site_packages=$(python -c \ 'import sysconfig; print(sysconfig.get_paths()["purelib"])') - tar -xJf pyarrow-*-debug-symbols.tar.xz -C "${site_packages}" + tar -xzf pyarrow-*-debug-symbols.tar.gz -C "${site_packages}" EOF # Keep the debug files out of the wheel. They are published as a separate # optional artifact for post-mortem debugging and symbolizing core dumps. tar -C /tmp/pyarrow-debug-symbols \ - -cJf ../../debug_symbols/pyarrow-debug-symbols.tar.xz \ + -czf ../../debug_symbols/pyarrow-debug-symbols.tar.gz \ README.txt pyarrow/.debug # Zip wheel again after stripping symbols zip -r "$wheel_name" . @@ -251,6 +251,6 @@ if [ "${#repaired_wheels[@]}" -ne 1 ]; then exit 1 fi repaired_wheel_name=$(basename "${repaired_wheels[0]}") -mv debug_symbols/pyarrow-debug-symbols.tar.xz \ - "debug_symbols/${repaired_wheel_name%.whl}-debug-symbols.tar.xz" +mv debug_symbols/pyarrow-debug-symbols.tar.gz \ + "debug_symbols/${repaired_wheel_name%.whl}-debug-symbols.tar.gz" popd diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml index 1f427cfc7bda..c233a0d899e6 100644 --- a/dev/tasks/python-wheels/github.linux.yml +++ b/dev/tasks/python-wheels/github.linux.yml @@ -92,7 +92,7 @@ jobs: - uses: actions/upload-artifact@v4 with: name: debug-symbols - path: arrow/python/debug_symbols/*.tar.xz + path: arrow/python/debug_symbols/*.tar.gz # TODO(kszucs): auditwheel show - name: Test wheel @@ -150,7 +150,7 @@ jobs: {{ macros.github_upload_releases([ "arrow/python/repaired_wheels/*.whl", - "arrow/python/debug_symbols/*.tar.xz" + "arrow/python/debug_symbols/*.tar.gz" ])|indent }} {{ macros.github_upload_wheel_scientific_python("arrow/python/repaired_wheels/*.whl")|indent }} From a3dc0b877cb4d8a3822143d74fb578ea5911b609 Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Fri, 25 Sep 2026 20:39:48 +0200 Subject: [PATCH 3/3] Verify Linux wheel debug-symbol backtraces on branch builds --- .../workflows/python_wheel_debug_symbols.yml | 108 ++++++++++++++++++ .../python_wheel_linux_debug_symbols_test.sh | 23 +++- 2 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/python_wheel_debug_symbols.yml diff --git a/.github/workflows/python_wheel_debug_symbols.yml b/.github/workflows/python_wheel_debug_symbols.yml new file mode 100644 index 000000000000..1d241240ed62 --- /dev/null +++ b/.github/workflows/python_wheel_debug_symbols.yml @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Python Wheel Debug Symbols + +on: + workflow_dispatch: + push: + paths: + - '.github/workflows/python_wheel_debug_symbols.yml' + - 'ci/scripts/python_wheel_linux_debug_symbols_test.sh' + - 'ci/scripts/python_wheel_xlinux_build.sh' + pull_request: + paths: + - '.github/workflows/python_wheel_debug_symbols.yml' + - 'ci/scripts/python_wheel_linux_debug_symbols_test.sh' + - 'ci/scripts/python_wheel_xlinux_build.sh' + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + verify: + name: ${{ matrix.kind }} backtraces + runs-on: ubuntu-latest + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - kind: manylinux + build-service: python-wheel-manylinux-2-28 + test-image: python:3.11-bookworm + install-gdb: apt-get update && apt-get install -y gdb + - kind: musllinux + build-service: python-wheel-musllinux-1-2 + test-image: python:3.11-alpine3.23 + install-gdb: apk add --no-cache bash gdb + env: + ARCHERY_DEBUG: 1 + PYTHON: '3.11' + PYTHON_ABI_TAG: cp311 + ULIMIT_CORE: 0 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + fetch-depth: 0 + submodules: recursive + - name: Free up disk space + run: ci/scripts/util_free_space.sh + - uses: actions/setup-python@v7 + with: + python-version: '3.12' + - name: Install Archery + run: python -m pip install -e 'dev/archery[docker]' + - name: Build this revision's wheel and debug-symbol bundle + # Reuse the public wheel build image, but run the checked-out build script + # against this revision's sources. No Crossbow submission or upload. + run: | + archery docker run --no-build \ + -e CMAKE_BUILD_PARALLEL_LEVEL=2 \ + ${{ matrix.build-service }} + - name: Verify stripped and symbolized backtraces + run: | + mkdir -p verification + docker run --rm \ + --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --volume "${GITHUB_WORKSPACE}:/arrow" \ + ${{ matrix.test-image }} \ + sh -ec '${{ matrix.install-gdb }} && + python -m pip install numpy && + /arrow/ci/scripts/python_wheel_linux_debug_symbols_test.sh \ + /arrow/python/repaired_wheels/*.whl \ + /arrow/python/debug_symbols/*.tar.gz' \ + 2>&1 | tee verification/backtraces.txt + - name: Preserve wheels, symbols, and verification logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: debug-symbol-verification-${{ matrix.kind }} + path: | + python/repaired_wheels/*.whl + python/debug_symbols/*.tar.gz + verification/*.txt diff --git a/ci/scripts/python_wheel_linux_debug_symbols_test.sh b/ci/scripts/python_wheel_linux_debug_symbols_test.sh index 1ef36a9655c4..d6c4ded55609 100755 --- a/ci/scripts/python_wheel_linux_debug_symbols_test.sh +++ b/ci/scripts/python_wheel_linux_debug_symbols_test.sh @@ -43,6 +43,10 @@ mkdir -p "${site_packages}" python -m pip install --no-deps --target "${site_packages}" "${wheel}" python -c "import numpy" +# GH-40749: preserve non-public frames, as in the GH-38770 backtrace. +# That bug is fixed, so abort in a scalar UDF to exercise the compute executor +# reliably on current wheels instead of relying on the old reproducer crashing. +ulimit -c 0 cat > "${work_dir}/backtrace_reproducer.py" <<'PYTHON' import os @@ -69,15 +73,27 @@ PYTHON run_gdb() { local output=$1 - PYTHONPATH="${site_packages}${PYTHONPATH:+:${PYTHONPATH}}" \ - gdb --batch --quiet \ + # Only use the supplied bundle, not system debug files or debuginfod. + if ! DEBUGINFOD_URLS= \ + PYTHONPATH="${site_packages}${PYTHONPATH:+:${PYTHONPATH}}" \ + gdb --nx --batch --quiet \ + -ex "set debug-file-directory ${work_dir}/no-system-debug" \ -ex run \ -ex "thread apply all backtrace" \ --args python "${work_dir}/backtrace_reproducer.py" \ - > "${output}" 2>&1 || true + > "${output}" 2>&1; then + cat "${output}" >&2 + return 1 + fi + cat "${output}" + if ! grep -F "received signal SIGABRT" "${output}"; then + echo "The reproducer did not reach the expected abort" >&2 + return 1 + fi } # Verify that the main wheel is stripped of non-public function symbols. +echo "=== Backtrace without the debug-symbol bundle ===" run_gdb "${work_dir}/stripped-backtrace.txt" if grep -F "ScalarExecutor::Execute" "${work_dir}/stripped-backtrace.txt"; then echo "The wheel still contains non-public function symbols" >&2 @@ -86,6 +102,7 @@ fi # GDB searches an adjacent .debug directory for GNU debuglink targets. tar -xzf "${debug_symbols}" -C "${site_packages}" +echo "=== Backtrace with the debug-symbol bundle ===" run_gdb "${work_dir}/symbolized-backtrace.txt" for symbol in \