perf(release): faster linux-x64 tsc (memfast, BOLT)

.github/workflows/release.yml

@@ -14,11 +14,15 @@
 # The tsc is a PGO build of the shipped profile (fat LTO), trained on each platform:
 # crates/ts_goport/scripts/build-pgo.sh with the training set of scripts/goport/pgo-train.sh (7
 # projects at fixed commits). The job also builds the plain tsc, and fails unless the PGO tsc gives
-# the same stdout, stderr, exit code and emitted files as the plain tsc on that set. No BOLT
-# (build-release.sh has it): BOLT refuses the static Linux bin (build-bolt.sh), its Mach-O support
-# is experimental, macOS has no perf branch sampling for its profile, and BOLT has no PE support.
+# the same stdout, stderr, exit code and emitted files as the plain tsc on that set. linux-x64 also
+# gets BOLT: crates/ts_goport/scripts/bolt-musl.sh, with a profile from a counting copy that runs
+# the same set, and no -split-eh (it can break the unwinding of a caught panic with musl). The PGO
+# check then compares the BOLT tsc with the plain tsc. The others have no BOLT: linux-arm64 is not
+# tried yet, BOLT's Mach-O support is experimental, and BOLT has no PE support.
 # Linux is static musl and non-PIE, so it starts on any x86-64 or arm64 Linux, Alpine too
 # (npm/README.md, "Linux libc"). Each Linux tsc builds natively on a Blacksmith runner of its arch.
+# The linux-x64 PGO tsc links lust's memfast.c in place of musl's memcpy, memmove, memcmp, bcmp and
+# memchr (crates/ts_goport/scripts/memfast.sh): musl's memcmp and bcmp are byte loops.
 # Windows (x86_64-pc-windows-msvc, aarch64-pc-windows-msvc) builds in Git Bash, with the C runtime
 # linked in (+crt-static), so tsc.exe needs no Visual C++ redistributable. jemalloc does not build
 # there: it uses the system allocator. Each PGO tsc builds natively, so its training runs on its
@@ -42,6 +46,9 @@ on:
       - "scripts/goport/pgo-train.sh"
       - "crates/ts_goport/scripts/copy-libs.sh"
       - "crates/ts_goport/scripts/build-pgo.sh"
+      - "crates/ts_goport/scripts/memfast.*"
+      - "crates/ts_goport/scripts/bolt-musl.sh"
+      - "crates/ts_goport/scripts/check-unwind.py"
       - "LICENSE"
       - "NOTICE.md"
       - "licenses/**"
@@ -134,6 +141,15 @@ jobs:
       - run: rustup toolchain install "$RUSTUP_TOOLCHAIN" --profile minimal --target ${{ matrix.target }} --component llvm-tools
       - if: runner.os == 'Linux'
         run: sudo apt-get update -q && sudo apt-get install -yq musl-tools
+      # linux-x64: llvm-bolt, merge-fdata, llvm-objcopy and llvm-dwarfdump of LLVM 23 for bolt-musl.sh.
+      - name: BOLT tools
+        if: matrix.platform == 'linux-x64'
+        run: |
+          curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null
+          codename=$(. /etc/os-release && echo "$VERSION_CODENAME")
+          echo "deb https://apt.llvm.org/$codename/ llvm-toolchain-$codename-23 main" | sudo tee /etc/apt/sources.list.d/llvm-23.list
+          sudo apt-get update -q && sudo apt-get install -yq bolt-23 llvm-23
+          /usr/lib/llvm-23/bin/llvm-bolt --version | grep -m1 'LLVM version'
       # darwin-x64 on an arm64 Mac: the check runs the x86-64 tsc, so Rosetta 2 must be there.
       - name: Rosetta
         if: matrix.rosetta
@@ -176,6 +192,13 @@ jobs:
           # jemalloc stops at start when the kernel page is bigger than its build page. arm64 kernels
           # use 4, 16 or 64 KiB pages; 2^16 covers all three.
           [[ $target != aarch64-* ]] || echo "JEMALLOC_SYS_WITH_LG_PAGE=16" >> "$GITHUB_ENV"
+          # linux-x64: lust's memfast.c in place of musl's memcpy, memmove, memcmp, bcmp and memchr
+          # (memfast.sh), in the PGO build only. The plain tsc keeps musl's, so the PGO check below
+          # also checks memfast against them.
+          if [[ $target == x86_64-unknown-linux-musl ]]; then
+            flags=$(crates/ts_goport/scripts/memfast.sh "$RUNNER_TEMP/memfast-wrap.o")
+            echo "PGO_RUSTFLAGS=$flags" | tee -a "$GITHUB_ENV"
+          fi
       # The plain tsc: the reference of the PGO check below, or the shipped tsc with no-pgo. PGO_LINK is
       # the link mode of both builds (build-pgo.sh): static on Linux, crt-static on Windows.
       - name: Build
@@ -201,10 +224,19 @@ jobs:
           PGO_BINS: tsgo
           PGO_CARGO: cargo
           CARGO_FEATURES: --features noembed
+          # The BOLT input needs the static relocations.
+          PGO_EMIT_RELOCS: ${{ matrix.platform == 'linux-x64' && '1' || '' }}
         run: |
           PGO_TRAIN="scripts/goport/pgo-train.sh run $RUNNER_TEMP/train" crates/ts_goport/scripts/build-pgo.sh "$RUNNER_TEMP/pgo"
           mkdir -p "$RUNNER_TEMP/out"
           cp "$RUNNER_TEMP/pgo/target-use/goport/tsgo${{ matrix.exe }}" "$RUNNER_TEMP/out/tsc${{ matrix.exe }}"
+      # linux-x64: BOLT of the PGO tsc. bolt-musl.sh also checks that the BOLT tsc still unwinds a
+      # caught panic (check-unwind.py).
+      - name: BOLT
+        if: matrix.platform == 'linux-x64'
+        run: |
+          crates/ts_goport/scripts/bolt-musl.sh "$RUNNER_TEMP/out/tsc" "$RUNNER_TEMP/train" "$RUNNER_TEMP/bolt"
+          cp "$RUNNER_TEMP/bolt/bin/tsc" "$RUNNER_TEMP/out/tsc"
       - name: Check the PGO tsc against the plain tsc
         if: ${{ !matrix.no-pgo }}
         run: scripts/goport/pgo-train.sh compare "$RUNNER_TEMP/train" "$RUNNER_TEMP/plain/tsc${{ matrix.exe }}" "$RUNNER_TEMP/out/tsc${{ matrix.exe }}"
@@ -233,6 +265,12 @@ jobs:
           [[ $rc == 2 ]] && grep -q "error TS2322" check.txt
           if [[ $RUNNER_OS == Linux ]]; then
             file "$dir/tsc" | tee /dev/stderr | grep -q "statically linked"
+            # memfast (linux-x64): the five __wrap_ functions, and none of musl's.
+            if [[ ${PGO_RUSTFLAGS:-} == *--wrap=* ]]; then
+              syms=$(nm "$dir/tsc" | grep -E ' [Tt] (__wrap_)?(memcpy|memmove|memcmp|bcmp|memchr)$' | awk '{print $3}' | sort | xargs)
+              echo "mem functions: $syms"
+              [[ $syms == "__wrap_bcmp __wrap_memchr __wrap_memcmp __wrap_memcpy __wrap_memmove" ]]
+            fi
             # Static: it starts on musl (Alpine) too.
             docker run --rm -v "$dir:/t:ro" alpine:3 /t/tsc --version
           fi

NOTICE.md

@@ -51,3 +51,14 @@ The collation and normalization tables in `crates/ts_goport/src/gostd/data` and
 packages above.
 
 License: [licenses/Unicode-LICENSE.txt](licenses/Unicode-LICENSE.txt).
+
+## memfast
+
+Copyright (c) 2026 Theo Browne. Licensed under the MIT License.
+
+`crates/ts_goport/scripts/memfast.c` is `src/runtime/memfast.c` of
+[t3dotgg/lust-lang](https://github.com/t3dotgg/lust-lang) (commit 6ced2b86fd), without change
+below its header. The linux-x64 tsc links it in place of musl's memcpy, memmove, memcmp, bcmp and
+memchr.
+
+License: [licenses/memfast-LICENSE.txt](licenses/memfast-LICENSE.txt).

crates/ts_goport/scripts/bolt-musl.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# BOLT of the static musl linux-x64 tsc (release.yml). BOLT puts the hot blocks of each function
+# together, moves the cold blocks out and orders the functions. The program does not change.
+# The profile comes from a counting copy (llvm-bolt -instrument) that runs the training set of
+# scripts/goport/pgo-train.sh: the release runner has no perf branch sampling.
+#
+# Usage: bolt-musl.sh <tsc> <train-dir> <out-dir>
+#   tsc        the PGO tsc, linked with --emit-relocs (build-pgo.sh PGO_EMIT_RELOCS=1)
+#   train-dir  a pgo-train.sh setup dir
+#   out-dir    new dir; the BOLT tsc is <out-dir>/bin/tsc (with the lib files of a noembed tsc),
+#              the logs and counts stay in <out-dir>
+#   LLVM_BIN   dir of llvm-bolt, merge-fdata, llvm-objcopy and llvm-dwarfdump
+#              (default /usr/lib/llvm-23/bin)
+#
+# Static bins differ from build-release.sh's dynamic ones in three ways (relmusl1):
+#   - A static bin runs no BOLT code at exit, so the counting copy writes its counts from a
+#     watcher process every second (-instrumentation-sleep-time=1), each time all of them
+#     (-instrumentation-no-counters-clear). Some watchers spin after their last write and block
+#     every signal but SIGKILL: the script kills them.
+#   - No -split-eh. It moves landing pads to the cold part and gives their LSDA a start address
+#     (LPStart). The musl target unwinds with LLVM libunwind, which logs "lsda at ... does not
+#     start with 0xFF" for such an LSDA. In some cases a caught panic through it then aborts
+#     (relmusl1's test program: "failed to initiate panic"); in others it works.
+#   - The relocation mode needs the static relocations (--emit-relocs).
+# Checks of the BOLT tsc: no BOLT runtime code (no __bolt symbols), one RW PT_GNU_STACK, and
+# check-unwind.py (two caught panics in the tsc with no libunwind line, and every LSDA starts with
+# 0xFF: a cautious guard that can reject a tsc that would work). release.yml then compares its
+# output with the plain tsc on the training set.
+set -euo pipefail
+
+case "${1:-}" in
+  help | -h | --help)
+    sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed -e '$d' -e 's/^# \{0,1\}//'
+    exit 0
+    ;;
+esac
+(($# == 3)) || { echo "usage: bolt-musl.sh <tsc> <train-dir> <out-dir>" >&2; exit 2; }
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+repo="$(cd -- "$script_dir/../../.." && pwd)"
+tsc=$1
+train="$(cd -- "$2" && pwd)"
+out=$3
+llvm="${LLVM_BIN:-/usr/lib/llvm-23/bin}"
+[[ ! -e $out ]] || { echo "error: $out exists" >&2; exit 2; }
+mkdir -p "$out/counts" "$out/bin"
+out="$(cd -- "$out" && pwd)"
+# grep -c, not -q: with pipefail, a grep that stops early can fail the pipe (SIGPIPE).
+[[ $(readelf -SW "$tsc" | grep -c ' \.rela\.text ') != 0 ]] ||
+  { echo "error: $tsc has no .rela.text (link it with --emit-relocs: build-pgo.sh PGO_EMIT_RELOCS=1)" >&2; exit 1; }
+
+# BOLT reads a symbol with ".cold" or ".warm" in it as a split part of another function and stops
+# ("parent function not found"). Rust paths make such names ("Session..warm_auto_import_cache").
+# Rename them in a copy: "..warm" to "..Warm" (same length, only .symtab changes).
+nm "$tsc" | awk '$2 ~ /^[tT]$/ && $3 ~ /\.\.(cold|warm)/ {r = $3; gsub(/\.\.cold/, "..Cold", r); gsub(/\.\.warm/, "..Warm", r); print $3, r}' > "$out/rename.txt"
+"$llvm/llvm-objcopy" --redefine-syms="$out/rename.txt" "$tsc" "$out/input"
+
+# The counting copy. pgo-train.sh runs its copy at <train-dir>/bin/tsc.
+"$llvm/llvm-bolt" "$out/input" -o "$out/instr" -instrument -instrumentation-file="$out/counts/tsc" \
+  -instrumentation-file-append-pid -instrumentation-binpath="$train/bin/tsc" \
+  -instrumentation-sleep-time=1 -instrumentation-no-counters-clear > "$out/instr.log" 2>&1 ||
+  { tail -20 "$out/instr.log" >&2; exit 1; }
+# One process per run: tsgo on 4 KiB pages would start a worker (bin/tsgo.rs `launch`).
+GOPORT_LAUNCH=0 "$repo/scripts/goport/pgo-train.sh" run "$train" "$out/instr" > "$out/train.log" 2>&1 ||
+  { tail -20 "$out/train.log" >&2; exit 1; }
+# The watchers write up to about 1 s after their tsc ends.
+sleep 5
+left=$(pgrep -fc "^$train/bin/tsc" || true)
+pkill -9 -f "^$train/bin/tsc" || true
+echo "bolt training: $(find "$out/counts" -name 'tsc.*.fdata' | wc -l) count files, $left watchers killed"
+"$llvm/merge-fdata" "$out"/counts/tsc.*.fdata > "$out/tsc.fdata" 2> "$out/merge.log" ||
+  { tail -5 "$out/merge.log" >&2; exit 1; }
+
+"$llvm/llvm-bolt" "$out/input" -o "$out/bin/tsc" -data="$out/tsc.fdata" \
+  -reorder-blocks=ext-tsp -reorder-functions=cdsort -split-functions -split-all-cold > "$out/bolt.log" 2>&1 ||
+  { tail -20 "$out/bolt.log" >&2; exit 1; }
+echo "bolt: $(grep -m1 -o '[0-9]* out of [0-9]* functions in the binary ([0-9.]*%) have non-empty execution profile' "$out/bolt.log" || true)"
+
+if [[ $(nm "$out/bin/tsc" | grep -c '__bolt') != 0 ]]; then
+  echo "error: $out/bin/tsc has BOLT runtime symbols (the counting copy?)" >&2
+  exit 1
+fi
+stack="$(readelf -lW "$out/bin/tsc" | awk '$1 == "GNU_STACK" { s = ""; for (i = 7; i < NF; i++) s = s $i; print s }')"
+[[ $stack == RW ]] || { echo "error: $out/bin/tsc has PT_GNU_STACK '$stack', not one RW" >&2; exit 1; }
+"$script_dir/copy-libs.sh" "$out/bin" > /dev/null
+python3 "$script_dir/check-unwind.py" "$out/bin/tsc" "$llvm/llvm-dwarfdump"
+echo "BOLT tsc: $out/bin/tsc"

crates/ts_goport/scripts/build-pgo.sh

@@ -27,8 +27,9 @@
 #       the layout when BOLT is not used.
 #   --emit-relocs  keeps the static relocations in the file (non-alloc
 #       sections, not mapped at run time). BOLT needs them for relocation
-#       mode, where it also reorders functions. Not with PGO_LINK=static:
-#       build-bolt.sh refuses static bins, so they only make the file bigger.
+#       mode, where it also reorders functions. Not with PGO_LINK=static
+#       unless PGO_EMIT_RELOCS=1 (bolt-musl.sh, release.yml's linux-x64
+#       tsc): else they only make the file bigger.
 # The build prints the .text and .rela.text sections of tsgo
 # (readelf -S). Expect .text.hot, .text.unlikely and .rela.text. GNU ld (the
 # linker of the musl target) ignores keep-text-section-prefix, but its default
@@ -53,6 +54,10 @@
 #                     x86_64-unknown-linux-musl. Default: the host, with
 #                     --target only when PGO_LINK is set.
 #   PGO_BINS          the bins to build, for example "tsgo" (default: all)
+#   PGO_RUSTFLAGS     extra rustc flags of both builds. release.yml sets the
+#                     memfast link flags of the linux-x64 tsc (memfast.sh).
+#   PGO_EMIT_RELOCS=1 link the use build with --emit-relocs also when
+#                     PGO_LINK=static (the input of bolt-musl.sh)
 #   PGO_TRAIN         training command, in place of the default training
 #                     (which needs the project inputs and the corpus of
 #                     GOPORT_DATA_ROOT). The script runs `$PGO_TRAIN <dir>`,
@@ -189,7 +194,7 @@ triple="${PGO_TARGET:-}"
 # Each step needs its own target (other RUSTFLAGS). sccache is off: it could
 # reuse an object built with an older profile at the same path.
 build() { # build <target-subdir> <rustflags>
-  local target="$out/$1" flags="$2 $link_flags" target_args=""
+  local target="$out/$1" flags="$2 $link_flags${PGO_RUSTFLAGS:+ $PGO_RUSTFLAGS}" target_args=""
   # Both builds get the same link mode, so the profile of the generate build
   # matches the code of the use build.
   [[ -n $triple ]] && target_args="--target $triple"
@@ -301,7 +306,7 @@ echo "merged $(du -h "$merged" | cut -f1) -> $merged"
 # The Linux link args are the r3-link layout (see the header).
 use_flags="-Cprofile-use=$merged"
 if [[ $(uname -s) == Linux ]]; then
-  [[ $link == static ]] || use_flags+=" -Clink-arg=-Wl,--emit-relocs"
+  [[ $link == static && ${PGO_EMIT_RELOCS:-0} != 1 ]] || use_flags+=" -Clink-arg=-Wl,--emit-relocs"
   use_flags+=" -Clink-arg=-Wl,-z,keep-text-section-prefix"
 fi
 build target-use "$use_flags"