From 2dddde5e8a6da9246b7322d8aba8e22d39f18fbd Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 25 Aug 2026 17:54:58 +0200 Subject: [PATCH 1/5] GH-50991: [CI] Enable sccache just like ccache on Docker-based builds --- compose.yaml | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/compose.yaml b/compose.yaml index 6fa4e6fee5b..4ae9eb81af3 100644 --- a/compose.yaml +++ b/compose.yaml @@ -507,7 +507,7 @@ services: shm_size: *shm-size ulimits: *ulimits environment: - <<: [*common, *ccache, *cpp] + <<: [*common, *ccache, *sccache, *cpp] ARROW_DEPENDENCY_SOURCE: BUNDLED ARROW_OFFLINE: ON # Apache ORC always uses external orc-format. @@ -668,7 +668,7 @@ services: shm_size: *shm-size volumes: *ubuntu-volumes environment: - <<: [*common, *ccache, *cpp] + <<: [*common, *ccache, *sccache, *cpp] CC: clang-${CLANG_TOOLS} CXX: clang++-${CLANG_TOOLS} # Avoid creating huge static libraries @@ -805,7 +805,7 @@ services: - ${REPO}:${ARCH_SHORT}-cpp-jni-${VCPKG} secrets: *vcpkg-build-secrets environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] volumes: - .:/arrow:delegated - ${DOCKER_VOLUME_PREFIX}cpp-jni-ccache:/ccache:delegated @@ -833,7 +833,7 @@ services: shm_size: *shm-size ulimits: *ulimits environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] BUILD_DOCS_C_GLIB: "ON" volumes: *debian-volumes command: &c-glib-command > @@ -862,7 +862,7 @@ services: shm_size: *shm-size ulimits: *ulimits environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] volumes: *ubuntu-volumes command: *c-glib-command @@ -893,7 +893,7 @@ services: shm_size: *shm-size ulimits: *ulimits environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] BUILD_DOCS_C_GLIB: "ON" volumes: *debian-volumes command: &ruby-command > @@ -925,7 +925,7 @@ services: shm_size: *shm-size ulimits: *ulimits environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] volumes: *ubuntu-volumes command: *ruby-command @@ -1067,7 +1067,7 @@ services: base: ${REPO}:${ARCH_SHORT}-debian-${DEBIAN}-cpp shm_size: *shm-size environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] volumes: *debian-volumes command: *python-command @@ -1090,7 +1090,7 @@ services: base: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp shm_size: *shm-size environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] volumes: *ubuntu-volumes command: *python-command @@ -1113,7 +1113,7 @@ services: base: ${REPO}:${ARCH_SHORT}-fedora-${FEDORA}-cpp shm_size: *shm-size environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] volumes: *fedora-volumes command: *python-command @@ -1157,7 +1157,7 @@ services: base: ${REPO}:${ARCH_SHORT}-ubuntu-${UBUNTU}-cpp shm_size: *shm-size environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] # Bundled build of OpenTelemetry needs a git client ARROW_WITH_OPENTELEMETRY: "OFF" PYARROW_VERSION: ${PYARROW_VERSION:-} @@ -1189,7 +1189,7 @@ services: - ${REPO}:${ARCH_SHORT}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG} secrets: *vcpkg-build-secrets environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] volumes: - .:/arrow:delegated - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux-2-28-ccache:/ccache:delegated @@ -1214,7 +1214,7 @@ services: - ${REPO}:${ARCH_SHORT}-python-${PYTHON}-wheel-musllinux-1-2-vcpkg-${VCPKG} secrets: *vcpkg-build-secrets environment: - <<: [ *common, *ccache ] + <<: [*common, *ccache, *sccache] volumes: - .:/arrow:delegated - ${DOCKER_VOLUME_PREFIX}python-wheel-musllinux-1-2-ccache:/ccache:delegated @@ -1541,7 +1541,7 @@ services: cap_add: - SYS_ADMIN environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] ARROW_SUBSTRAIT: "ON" LC_ALL: "C.UTF-8" LANG: "C.UTF-8" @@ -1583,7 +1583,7 @@ services: dask: ${DASK} shm_size: *shm-size environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] volumes: *conda-volumes command: ["/arrow/ci/scripts/cpp_build.sh /arrow /build && @@ -1610,7 +1610,7 @@ services: python: ${PYTHON} shm_size: *shm-size environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] PYTEST_ARGS: # inherit volumes: *conda-volumes command: *python-conda-command @@ -1865,7 +1865,7 @@ services: node: ${NODE} volumes: *conda-volumes environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] ARCHERY_INTEGRATION_WITH_GO: 0 ARCHERY_INTEGRATION_WITH_NANOARROW: 0 ARCHERY_INTEGRATION_WITH_RUST: 0 @@ -1901,7 +1901,7 @@ services: cap_add: - SYS_ADMIN environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] ARROW_AZURE: "ON" ARROW_CUDA: "ON" ARROW_CXX_FLAGS_DEBUG: "-g1" @@ -1979,7 +1979,7 @@ services: links: - impala:impala environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] ARROW_ENGINE: "OFF" ARROW_FLIGHT: "OFF" ARROW_FLIGHT_SQL: "OFF" @@ -2021,7 +2021,7 @@ services: numpy: ${NUMPY} shm_size: *shm-size environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] volumes: *conda-maven-volumes command: ["/arrow/ci/scripts/cpp_build.sh /arrow /build && @@ -2037,7 +2037,7 @@ services: - ${DOCKER_VOLUME_PREFIX}conda-ccache:/ccache:delegated shm_size: '1gb' environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] CMAKE_GENERATOR: Ninja DEBIAN_FRONTEND: "noninteractive" DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 1 @@ -2070,7 +2070,7 @@ services: - ${DOCKER_VOLUME_PREFIX}almalinux-ccache:/ccache:delegated shm_size: '1gb' environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] CMAKE_GENERATOR: Ninja SSL_CERT_FILE: /etc/ssl/certs/ca-bundle.crt # required due to issue in RHEL-8 based images. See GH-49757 TEST_APT: 0 # would require docker-in-docker @@ -2100,7 +2100,7 @@ services: - ${DOCKER_VOLUME_PREFIX}ubuntu-ccache:/ccache:delegated shm_size: '1gb' environment: - <<: [*common, *ccache] + <<: [*common, *ccache, *sccache] CMAKE_GENERATOR: Ninja TEST_APT: 0 # would require docker-in-docker TEST_YUM: 0 From df7c4796586af0727a3f24b45c4a5f757eaf6f1a Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 25 Aug 2026 18:30:31 +0200 Subject: [PATCH 2/5] Try to enable sccache on manylinux --- ci/docker/python-wheel-manylinux.dockerfile | 4 ++++ ci/docker/python-wheel-musllinux.dockerfile | 1 + ci/scripts/python_wheel_xlinux_build.sh | 10 ++++++++++ dev/tasks/python-wheels/github.linux.yml | 1 + 4 files changed, 16 insertions(+) diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile index 57814b11ec8..aa968ad3cd0 100644 --- a/ci/docker/python-wheel-manylinux.dockerfile +++ b/ci/docker/python-wheel-manylinux.dockerfile @@ -50,6 +50,10 @@ ARG ccache=4.13.6 COPY ci/scripts/install_ccache.sh arrow/ci/scripts/ RUN /arrow/ci/scripts/install_ccache.sh ${ccache} /usr/local +# Install sccache +COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin + # Install bison (> 3.7 required for building thrift) ARG bison=3.7.6 COPY ci/scripts/install_bison.sh arrow/ci/scripts/ diff --git a/ci/docker/python-wheel-musllinux.dockerfile b/ci/docker/python-wheel-musllinux.dockerfile index c931d319446..2138b277bfc 100644 --- a/ci/docker/python-wheel-musllinux.dockerfile +++ b/ci/docker/python-wheel-musllinux.dockerfile @@ -34,6 +34,7 @@ RUN apk add --no-cache \ flex \ git \ ninja \ + sccache \ unzip \ wget \ zip diff --git a/ci/scripts/python_wheel_xlinux_build.sh b/ci/scripts/python_wheel_xlinux_build.sh index b0e53a05321..0a5d63affc2 100755 --- a/ci/scripts/python_wheel_xlinux_build.sh +++ b/ci/scripts/python_wheel_xlinux_build.sh @@ -183,6 +183,16 @@ python -m build --sdist --wheel . --no-isolation \ -C cmake.build-type="${CMAKE_BUILD_TYPE:-Debug}" \ -C cmake.args="-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=${CMAKE_INTERPROCEDURAL_OPTIMIZATION}" +if command -v ccache &> /dev/null; then + echo "=== ccache stats after the build ===" + ccache -sv 2>/dev/null || ccache -s +fi + +if command -v sccache &> /dev/null; then + echo "=== sccache stats after the build ===" + sccache --show-stats +fi + echo "=== Strip symbols from wheel ===" mkdir -p dist/temp-fix-wheel mv dist/pyarrow-*.whl dist/temp-fix-wheel diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml index b1a2e0de5cc..e2a74eb841e 100644 --- a/dev/tasks/python-wheels/github.linux.yml +++ b/dev/tasks/python-wheels/github.linux.yml @@ -68,6 +68,7 @@ jobs: - name: Build wheel shell: bash env: + {{ macros.github_set_sccache_envvars()|indent(8) }} GITHUB_TOKEN: {{ '${{ secrets.GITHUB_TOKEN }}' }} run: | if [ "{{ linux_wheel_kind }}" = "musllinux" ] && [ "{{ arch }}" = "arm64" ]; then From 5a5686abda4576dbeb8c1b9ee16c78422d100052 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 25 Aug 2026 20:11:21 +0200 Subject: [PATCH 3/5] Also install sccache on Debian-based R builds --- ci/docker/alpine-linux-3.22-r.dockerfile | 2 ++ ci/docker/linux-apt-r.dockerfile | 3 +++ 2 files changed, 5 insertions(+) diff --git a/ci/docker/alpine-linux-3.22-r.dockerfile b/ci/docker/alpine-linux-3.22-r.dockerfile index 31ffddd29d9..c4792a1459b 100644 --- a/ci/docker/alpine-linux-3.22-r.dockerfile +++ b/ci/docker/alpine-linux-3.22-r.dockerfile @@ -29,6 +29,7 @@ RUN apk add \ R-dev \ R-doc \ bash \ + ccache \ checkbashisms \ cmake \ curl-dev \ @@ -40,6 +41,7 @@ RUN apk add \ musl-locales \ openssl-dev \ pkgconfig \ + sccache \ zlib-dev && \ rm -rf /var/cache/apk/* && \ ln -s /usr/share/zoneinfo/Etc/UTC /etc/localtime && \ diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile index 7809cf55424..f33faec70d0 100644 --- a/ci/docker/linux-apt-r.dockerfile +++ b/ci/docker/linux-apt-r.dockerfile @@ -100,6 +100,9 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_gcs_testbench.sh default +COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin + ENV \ ARROW_ACERO=ON \ ARROW_BUILD_STATIC=OFF \ From 20df7af2d3aecabf18e59a5e2a3d1230d2e37393 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 25 Aug 2026 20:43:46 +0200 Subject: [PATCH 4/5] Revert "Also install sccache on Debian-based R builds": sccache is already installed by base image This reverts commit 37d53be89641fad97c9d3612db7d66136b4455bc. --- ci/docker/alpine-linux-3.22-r.dockerfile | 2 -- ci/docker/linux-apt-r.dockerfile | 3 --- 2 files changed, 5 deletions(-) diff --git a/ci/docker/alpine-linux-3.22-r.dockerfile b/ci/docker/alpine-linux-3.22-r.dockerfile index c4792a1459b..31ffddd29d9 100644 --- a/ci/docker/alpine-linux-3.22-r.dockerfile +++ b/ci/docker/alpine-linux-3.22-r.dockerfile @@ -29,7 +29,6 @@ RUN apk add \ R-dev \ R-doc \ bash \ - ccache \ checkbashisms \ cmake \ curl-dev \ @@ -41,7 +40,6 @@ RUN apk add \ musl-locales \ openssl-dev \ pkgconfig \ - sccache \ zlib-dev && \ rm -rf /var/cache/apk/* && \ ln -s /usr/share/zoneinfo/Etc/UTC /etc/localtime && \ diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile index f33faec70d0..7809cf55424 100644 --- a/ci/docker/linux-apt-r.dockerfile +++ b/ci/docker/linux-apt-r.dockerfile @@ -100,9 +100,6 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_gcs_testbench.sh default -COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin - ENV \ ARROW_ACERO=ON \ ARROW_BUILD_STATIC=OFF \ From 3e9e9ef3c11a1f56c7e07f6943e8fa807c6de2ae Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 26 Aug 2026 10:21:11 +0200 Subject: [PATCH 5/5] Fix offline build --- compose.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index 4ae9eb81af3..00e791ccea1 100644 --- a/compose.yaml +++ b/compose.yaml @@ -507,7 +507,8 @@ services: shm_size: *shm-size ulimits: *ulimits environment: - <<: [*common, *ccache, *sccache, *cpp] + # Do not inherit sccache since network is switched off in this build + <<: [*common, *ccache, *cpp] ARROW_DEPENDENCY_SOURCE: BUNDLED ARROW_OFFLINE: ON # Apache ORC always uses external orc-format.