From 1464206219e992c129e0c9ec423aa9c7863d984d Mon Sep 17 00:00:00 2001 From: Matthew Kocher Date: Tue, 25 Aug 2026 15:44:18 -0700 Subject: [PATCH] Make base_ubuntu_warden_rosetta host-architecture agnostic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stage executed arm64 binaries while doing its work — the tar self-test, and `dpkg -x` in the chroot, which shells out to the just-swapped GNU tar — so it only ran on an Apple Silicon host. Download all the arm64 debs first, extract them from outside the chroot with the builder image's tar, then swap and statically verify (e_machine == 183) every replacement, including tar and the systemd daemons. The tar round-trip now runs only behind an exec probe and skips loudly otherwise, in the stage and in rosetta_spec.rb, so the stage can build on an x86-64 CI worker. Also cover stage_collection's rosetta insertion, which had no unit test. --- .../bosh/stemcell/stage_collection_spec.rb | 15 + bosh-stemcell/spec/stemcells/rosetta_spec.rb | 16 +- .../base_ubuntu_warden_rosetta/apply.sh | 310 ++++++++++-------- 3 files changed, 207 insertions(+), 134 deletions(-) diff --git a/bosh-stemcell/spec/bosh/stemcell/stage_collection_spec.rb b/bosh-stemcell/spec/bosh/stemcell/stage_collection_spec.rb index b7ed65cc61..dc5eb8b0c2 100644 --- a/bosh-stemcell/spec/bosh/stemcell/stage_collection_spec.rb +++ b/bosh-stemcell/spec/bosh/stemcell/stage_collection_spec.rb @@ -330,6 +330,21 @@ module Bosh::Stemcell expect(stage_collection.build_stemcell_image_stages).to eq(build_stemcell_image_stages) expect(stage_collection.package_stemcell_stages("files")).to eq(package_stemcell_stages) end + + it "does not include the rosetta stage" do + expect(stage_collection.build_stemcell_image_stages).to_not include(:base_ubuntu_warden_rosetta) + end + + context "with the rosetta variant" do + let(:operating_system) { OperatingSystem.for("ubuntu", "resolute-rosetta") } + + # The stage has to run before bosh_clean/bosh_harden, which is why its + # position rather than its mere presence is asserted. + it "inserts the rosetta stage directly after base_warden" do + stages = stage_collection.build_stemcell_image_stages + expect(stages[stages.index(:base_warden) + 1]).to eq(:base_ubuntu_warden_rosetta) + end + end end end end diff --git a/bosh-stemcell/spec/stemcells/rosetta_spec.rb b/bosh-stemcell/spec/stemcells/rosetta_spec.rb index b666043f0b..4f02ebf944 100644 --- a/bosh-stemcell/spec/stemcells/rosetta_spec.rb +++ b/bosh-stemcell/spec/stemcells/rosetta_spec.rb @@ -70,7 +70,12 @@ end # The architecture check above would still pass if the binary could not - # actually run, so exercise a real round-trip. + # actually run, so exercise a real round-trip — but only on a host whose + # kernel can run arm64. An x86-64 CI worker has no arm64 handler and every + # exec fails with "Exec format error"; there the exec path is covered + # instead by the manual build on Apple Silicon + # (docs/ci-warden-rosetta-build.md). Every other assertion in this file is + # static and runs everywhere. describe command( "set -e; " \ "rm -rf /tmp/tar-spec; mkdir -p /tmp/tar-spec/src /tmp/tar-spec/out; " \ @@ -80,7 +85,14 @@ "grep -q payload /tmp/tar-spec/out/src/probe; " \ "rm -rf /tmp/tar-spec" ) do - it("extracts an archive it just created") { expect(subject.exit_status).to eq(0) } + it "extracts an archive it just created" do + if command("/lib/ld-linux-aarch64.so.1 --help").exit_status != 0 + skip("this build host cannot execute arm64 binaries; the tar " \ + "round-trip is the only check skipped, all static assertions ran") + end + + expect(subject.exit_status).to eq(0) + end end end diff --git a/stemcell_builder/stages/base_ubuntu_warden_rosetta/apply.sh b/stemcell_builder/stages/base_ubuntu_warden_rosetta/apply.sh index 067e149042..6a7d95742f 100755 --- a/stemcell_builder/stages/base_ubuntu_warden_rosetta/apply.sh +++ b/stemcell_builder/stages/base_ubuntu_warden_rosetta/apply.sh @@ -20,12 +20,24 @@ # # Inserted into warden_stages only when the OS variant is "rosetta" # (rake ... ubuntu,resolute-rosetta,...); other builds are unaffected. +# +# The stage is host-architecture agnostic: it never executes an arm64 binary as +# part of doing its work, so it runs on an x86-64 CI worker as well as on Apple +# Silicon. Everything is downloaded and extracted first and the binaries are +# swapped in at the end; extraction uses the builder container's tar rather than +# the chroot's, because whichever architecture the chroot's tar is, it is +# unrunnable on one of the two hosts. The execution-based self-test at the end is +# the one part that needs an arm64-capable kernel, and it skips loudly when the +# host has none. set -e base_dir=$(readlink -nf $(dirname $0)/../..) source $base_dir/lib/prelude_apply.bash +debs_dir=/tmp/arm64-debs +debs_root="$chroot$debs_dir/root" + # --------------------------------------------------------------------------- # Part 1: arm64 foreign architecture and runtime libraries # --------------------------------------------------------------------------- @@ -35,11 +47,15 @@ run_in_chroot $chroot "dpkg --add-architecture arm64" run_in_chroot $chroot "apt-get update" # Install arm64 runtime libraries needed by the arm64 binaries installed below -# (systemd in Part 3; unix_chkpwd, auditd and logrotate in Part 4 — auditd needs +# (systemd in Part 4; unix_chkpwd, auditd and logrotate in Part 5 — auditd needs # libauparse, logrotate needs libpopt). # These install into /lib/aarch64-linux-gnu/ and /usr/lib/aarch64-linux-gnu/, # coexisting safely with the existing amd64 libraries. # libc6:arm64 also places /lib/ld-linux-aarch64.so.1 (the arm64 ELF interpreter). +# +# apt-get install is safe here even though nothing arm64 can run yet: dpkg +# unpacks with its own built-in tar reader, and none of these packages run arm64 +# maintainer scripts. arm64_libs="libc6:arm64 \ libselinux1:arm64 \ libcrypt1:arm64 \ @@ -64,162 +80,145 @@ arm64_libs="libc6:arm64 \ run_in_chroot $chroot "apt-get install --no-install-recommends --assume-yes $arm64_libs" # --------------------------------------------------------------------------- -# Part 2: arm64 tar +# Part 2: download every arm64 deb that gets swapped in by hand # --------------------------------------------------------------------------- -# This has to happen before the systemd debs below, because extracting those -# uses the chroot's own tar — and Ubuntu 26.04's x86-64 GNU tar -# (1.35+dfsg-4ubuntu0.4) cannot extract anything under Rosetta: every file it -# creates fails with "Cannot open: Function not implemented" (ENOSYS), because -# tar issues a syscall Rosetta does not translate. Ubuntu 24.04's tar is -# unaffected, so this is new in 26.04. The arm64 build of the same version works -# because it runs natively on the arm64 kernel. It is also the only tool in the -# base image with this problem — gzip, xz, zstd, cpio, rsync and coreutils are -# all fine. -# -# It matters at runtime too, not just during the build: the BOSH agent shells -# out to tar to unpack every compiled package and release blob it downloads, so -# a stemcell with a broken tar cannot run a single deployment. -# -# dpkg has its own built-in tar reader and does not call GNU tar to unpack debs, -# so apt keeps working regardless; only direct tar invocations are affected -# (including `dpkg -x`, which does shell out to GNU tar). -# -# `apt-get install tar:arm64` cannot be used: tar is Multi-Arch-incompatible and -# tar:amd64 declares Conflicts: tar:arm64. Download and swap the binary in by -# hand instead, the same way the systemd binaries are handled below. -run_in_chroot $chroot " - mkdir -p /tmp/arm64-tar - cd /tmp/arm64-tar - apt-get download tar:arm64 -" +# None of these packages carry Multi-Arch: same, so `apt-get install :arm64` +# conflicts with the installed amd64 package (tar:amd64 declares an explicit +# Conflicts: tar:arm64). Download the debs and swap the binaries out of them +# instead. systemd-resolved and libsystemd-shared ship separately from the main +# systemd deb; unix_chkpwd comes from libpam-modules-bin. +arm64_packages="tar \ + systemd \ + libsystemd-shared \ + systemd-resolved \ + libpam-modules-bin \ + auditd \ + logrotate" + +arm64_download_list="" +for pkg in $arm64_packages; do + arm64_download_list="$arm64_download_list ${pkg}:arm64" +done -# Unpack from outside the chroot: the chroot's own tar is the broken binary we -# are about to replace, so it cannot be used to extract its own replacement. -# The builder container's tar is known-good (see ARM64_TAR_FIX in -# ci/docker/os-image-stemcell-builder/Dockerfile). -mkdir -p "$chroot/tmp/arm64-tar/root" -dpkg-deb --fsys-tarfile "$chroot"/tmp/arm64-tar/tar_*_arm64.deb \ - | tar -x -C "$chroot/tmp/arm64-tar/root" - -# Keep the x86-64 binary as tar.amd64 for debugging and so the swap is obvious -# to anyone inspecting the stemcell. -mv "$chroot/usr/bin/tar" "$chroot/usr/bin/tar.amd64" -install -m 0755 "$chroot/tmp/arm64-tar/root/usr/bin/tar" "$chroot/usr/bin/tar" -rm -rf "$chroot/tmp/arm64-tar" - -# Fail loudly if the replacement cannot actually extract. Everything below -# depends on it, and a silently broken tar would otherwise only surface much -# later as an unexplained deployment failure. run_in_chroot $chroot " - set -e - rm -rf /tmp/tar-selftest - mkdir -p /tmp/tar-selftest/src /tmp/tar-selftest/out - echo rosetta > /tmp/tar-selftest/src/probe - tar -czf /tmp/tar-selftest/probe.tgz -C /tmp/tar-selftest src - tar -xzf /tmp/tar-selftest/probe.tgz -C /tmp/tar-selftest/out - grep -q rosetta /tmp/tar-selftest/out/src/probe - rm -rf /tmp/tar-selftest + rm -rf $debs_dir + mkdir -p $debs_dir + cd $debs_dir + apt-get download $arm64_download_list " # --------------------------------------------------------------------------- -# Part 3: arm64 systemd binaries +# Part 3: extract them, from outside the chroot # --------------------------------------------------------------------------- -# Download arm64 systemd debs without installing. -# `apt-get install systemd:arm64` would conflict with systemd:amd64 because -# systemd does not carry Multi-Arch: same. Download and extract manually instead. -# systemd-resolved ships in a separate Ubuntu package from the main systemd deb. -run_in_chroot $chroot " - mkdir -p /tmp/arm64-debs - cd /tmp/arm64-debs - apt-get download systemd:arm64 libsystemd-shared:arm64 systemd-resolved:arm64 -" - -# Extract the debs into staging directories. -run_in_chroot $chroot " - dpkg -x /tmp/arm64-debs/systemd_*_arm64.deb /tmp/arm64-debs/systemd - dpkg -x /tmp/arm64-debs/libsystemd-shared_*_arm64.deb /tmp/arm64-debs/libsystemd-shared - dpkg -x /tmp/arm64-debs/systemd-resolved_*_arm64.deb /tmp/arm64-debs/systemd-resolved -" - -# Install arm64 private shared libraries into the arm64-specific path. -# The arm64 systemd binary's RPATH points to /usr/lib/aarch64-linux-gnu/systemd/. -run_in_chroot $chroot " - mkdir -p /usr/lib/aarch64-linux-gnu/systemd - cp /tmp/arm64-debs/libsystemd-shared/usr/lib/aarch64-linux-gnu/systemd/libsystemd-*.so \ - /usr/lib/aarch64-linux-gnu/systemd/ -" +# `dpkg -x` inside the chroot shells out to GNU tar, so it would run whichever +# tar the chroot has — the x86-64 one that cannot extract under Rosetta, or the +# arm64 one that cannot execute on an x86-64 worker. dpkg-deb + the builder +# container's tar is correct on both hosts. +extract_deb() { + local pkg="$1" + local dest="$debs_root/$pkg" -# Replace x86_64 systemd service daemons with arm64 equivalents. -# Only daemons in /usr/lib/systemd/ are replaced; user-facing CLI tools in -# /usr/bin/ (systemctl, journalctl, etc.) are deliberately left as x86-64. -# Those tools communicate with PID1 over D-Bus and never call pidfd themselves, -# so they work correctly under Rosetta. Keeping them x86-64 also allows the -# build-time RSpec suite to execute them inside the x86-64 chroot environment. -# Symlinks in /usr/lib/systemd/ (e.g. systemd-udevd@ -> ../../bin/udevadm) are -# skipped by -type f so they continue to point at their existing targets. -run_in_chroot $chroot " - # All regular-file daemons from the main systemd package (includes PID1). - # On Ubuntu 22.04+ (UsrMerge), /lib -> usr/lib, so the deb ships binaries - # at usr/lib/systemd/ rather than lib/systemd/. - find /tmp/arm64-debs/systemd/usr/lib/systemd/ -maxdepth 1 -type f \ - | xargs -I{} cp {} /usr/lib/systemd/ - - # systemd-resolved is packaged separately on Ubuntu. - cp /tmp/arm64-debs/systemd-resolved/usr/lib/systemd/systemd-resolved \ - /usr/lib/systemd/systemd-resolved -" + mkdir -p "$dest" + dpkg-deb --fsys-tarfile "$chroot$debs_dir/${pkg}"_*_arm64.deb | tar -x -C "$dest" +} -# Clean up staging area. -run_in_chroot $chroot "rm -rf /tmp/arm64-debs" +for pkg in $arm64_packages; do + extract_deb "$pkg" +done # --------------------------------------------------------------------------- -# Part 4: arm64 userland binaries +# Part 4: swap in the arm64 binaries # --------------------------------------------------------------------------- -# swap_arm64_binary [group] +# A swap that silently did not land would only resurface much later as an +# unexplained runtime failure, so assert the architecture of every replacement. +# e_machine at offset 18 of the ELF header is 183 (EM_AARCH64) for arm64, 62 for +# x86-64. This is a static read, so it works on any build host. +assert_arm64() { + local path="$1" machine + + is_elf "$path" || { echo "ERROR: $path is not an ELF binary" >&2; exit 1; } + + machine=$(od -An -tu1 -j18 -N1 "$path" | tr -d ' ') + if [ "$machine" != "183" ]; then + echo "ERROR: $path is not arm64 (e_machine=$machine)" >&2 + exit 1 + fi +} + +is_elf() { + head -c 4 "$1" | grep -qa 'ELF' +} + +# swap_arm64_binary [group] # -# None of these packages carry Multi-Arch: same, so `apt-get install :arm64` -# would conflict with the amd64 package; download and swap by hand instead. The -# x86-64 build is kept as .amd64, matching tar.amd64 above. +# The x86-64 build is kept as .amd64, for debugging and so the swap is +# obvious to anyone inspecting the stemcell. swap_arm64_binary() { local pkg="$1" path="$2" group="${3:-}" - local stage="/tmp/arm64-swap" + local src="$debs_root/$pkg$path" - run_in_chroot $chroot " - set -e - rm -rf $stage - mkdir -p $stage - cd $stage - apt-get download ${pkg}:arm64 - dpkg -x $stage/${pkg}_*_arm64.deb $stage/root - " - - if [ ! -f "$chroot$stage/root$path" ]; then + if [ ! -f "$src" ]; then echo "ERROR: ${pkg}:arm64 does not ship $path" >&2 exit 1 fi mv "$chroot$path" "$chroot${path}.amd64" - install -m 0755 "$chroot$stage/root$path" "$chroot$path" - [ -n "$group" ] && run_in_chroot $chroot "chown root:$group $path" - run_in_chroot $chroot "rm -rf $stage" - - # Fail loudly if the swap did not land. A silently-still-x86-64 binary would - # only resurface much later as an unexplained runtime failure. - run_in_chroot $chroot " - set -e - head -c 20 $path | grep -qa 'ELF' || { echo '$path is not an ELF binary' >&2; exit 1; } - # e_machine at offset 18 is 183 (EM_AARCH64) for arm64, 62 for x86-64. - arch_byte=\$(od -An -tu1 -j18 -N1 $path | tr -d ' ') - if [ \"\$arch_byte\" != '183' ]; then - echo \"ERROR: $path is not arm64 (e_machine=\$arch_byte)\" >&2 - exit 1 - fi - " + install -m 0755 "$src" "$chroot$path" + if [ -n "$group" ]; then + run_in_chroot $chroot "chown root:$group $path" + fi + assert_arm64 "$chroot$path" } +# Ubuntu 26.04's x86-64 GNU tar (1.35+dfsg-4ubuntu0.4) cannot extract anything +# under Rosetta: every file it creates fails with "Cannot open: Function not +# implemented" (ENOSYS), because tar issues a syscall Rosetta does not translate. +# Ubuntu 24.04's tar is unaffected, so this is new in 26.04. It is also the only +# tool in the base image with this problem — gzip, xz, zstd, cpio, rsync and +# coreutils are all fine. +# +# It matters at runtime, not just during the build: the BOSH agent shells out to +# tar to unpack every compiled package and release blob it downloads, so a +# stemcell with a broken tar cannot run a single deployment. +# +# dpkg has its own built-in tar reader and does not call GNU tar to unpack debs, +# so apt keeps working regardless; only direct tar invocations are affected +# (including `dpkg -x`, which does shell out to GNU tar). +swap_arm64_binary tar /usr/bin/tar + +# Install arm64 private shared libraries into the arm64-specific path. +# The arm64 systemd binary's RPATH points to /usr/lib/aarch64-linux-gnu/systemd/. +mkdir -p "$chroot/usr/lib/aarch64-linux-gnu/systemd" +cp "$debs_root"/libsystemd-shared/usr/lib/aarch64-linux-gnu/systemd/libsystemd-*.so \ + "$chroot/usr/lib/aarch64-linux-gnu/systemd/" + +# Replace x86_64 systemd service daemons with arm64 equivalents. +# Only daemons in /usr/lib/systemd/ are replaced; user-facing CLI tools in +# /usr/bin/ (systemctl, journalctl, etc.) are deliberately left as x86-64. +# Those tools communicate with PID1 over D-Bus and never call pidfd themselves, +# so they work correctly under Rosetta. Keeping them x86-64 also allows the +# build-time RSpec suite to execute them inside the x86-64 chroot environment. +# +# On Ubuntu 22.04+ (UsrMerge), /lib -> usr/lib, so the deb ships binaries at +# usr/lib/systemd/ rather than lib/systemd/. Symlinks there (e.g. +# systemd-udevd@ -> ../../bin/udevadm) are skipped so they keep pointing at +# their existing targets, and the package's few non-ELF helpers are left alone. +for daemon in "$debs_root"/systemd/usr/lib/systemd/*; do + if [ -L "$daemon" ] || [ ! -f "$daemon" ] || ! is_elf "$daemon"; then + continue + fi + + cp "$daemon" "$chroot/usr/lib/systemd/" + assert_arm64 "$chroot/usr/lib/systemd/$(basename "$daemon")" +done + +cp "$debs_root/systemd-resolved/usr/lib/systemd/systemd-resolved" \ + "$chroot/usr/lib/systemd/systemd-resolved" +assert_arm64 "$chroot/usr/lib/systemd/systemd-resolved" + # pam_unix forks /usr/sbin/unix_chkpwd to read /etc/shadow. An AppArmor profile # attached by path to /{,usr/}{,s}bin/unix_chkpwd — loaded on the Lima VM and # enforced by the shared kernel, so it applies inside containers too — grants no @@ -248,8 +247,55 @@ swap_arm64_binary auditd /usr/sbin/auditd # binary outright (Result=signal). swap_arm64_binary logrotate /usr/sbin/logrotate +run_in_chroot $chroot "rm -rf $debs_dir" + +# --------------------------------------------------------------------------- +# Part 5: execution self-test, where execution is possible +# --------------------------------------------------------------------------- + +# The static assertions above cannot tell a working arm64 binary from one that +# will not load, so exercise the swapped tar for real — but only on a host whose +# kernel can run arm64 at all. An x86-64 CI worker has no arm64 handler and every +# exec fails with "Exec format error"; that is expected there, and the exec path +# is covered instead by the manual build on Apple Silicon +# (docs/ci-warden-rosetta-build.md). +# +# The probe writes a marker file rather than being tested with `if +# run_in_chroot ...`: run_in_chroot's own exit status is that of its cleanup, not +# of the chroot command, so a failure is only visible through errexit — which an +# `if` condition suppresses. +probe_marker=/tmp/arm64-exec-probe +run_in_chroot $chroot " + rm -f $probe_marker + /lib/ld-linux-aarch64.so.1 --help >/dev/null 2>&1 && : > $probe_marker + true +" + +if [ -f "$chroot$probe_marker" ]; then + run_in_chroot $chroot " + set -e + /usr/bin/tar --version >/dev/null + rm -rf /tmp/tar-selftest + mkdir -p /tmp/tar-selftest/src /tmp/tar-selftest/out + echo rosetta > /tmp/tar-selftest/src/probe + tar -czf /tmp/tar-selftest/probe.tgz -C /tmp/tar-selftest src + tar -xzf /tmp/tar-selftest/probe.tgz -C /tmp/tar-selftest/out + grep -q rosetta /tmp/tar-selftest/out/src/probe + rm -rf /tmp/tar-selftest + " + echo "base_ubuntu_warden_rosetta: arm64 tar round-trip self-test passed." +else + echo "base_ubuntu_warden_rosetta: SKIPPED all execution-based checks (arm64" \ + "tar --version and round-trip) — this build host cannot execute arm64" \ + "binaries at all, so not even the ELF interpreter runs. The swapped" \ + "binaries were verified statically only; run the build on Apple Silicon" \ + "to exercise them." +fi + +run_in_chroot $chroot "rm -f $probe_marker" + # --------------------------------------------------------------------------- -# Part 5: binfmt_misc +# Part 6: binfmt_misc # --------------------------------------------------------------------------- # systemd-binfmt rewrites /proc/sys/fs/binfmt_misc, where Lima registers the