From 51d613d51b4a98355a437c6a4ad13461fdaa8e4f Mon Sep 17 00:00:00 2001 From: Theo Browne <6751787+t3dotgg@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:04:57 -0700 Subject: [PATCH 1/5] perf(release): link memfast into the linux-x64 tsc musl's memcmp and bcmp are byte loops, and its memcpy and memmove use rep movs. They took 4.4% of the cycles of a VS Code check and 9.2% on TypeORM (relmusl1). The linux-x64 PGO tsc now links lust's memfast.c (MIT, Theo Browne; byte-equal below its new header) through -Wl,--wrap for memcpy, memmove, memcmp, bcmp and memchr. - memfast.sh compiles it with musl-gcc, checks the object (no undefined symbol, no AVX or BMI) and prints the link flags. - build-pgo.sh: PGO_RUSTFLAGS adds rustc flags to both builds. - release.yml: the flags go to the PGO build only, so the plain tsc (musl's functions) stays the reference of the PGO check. The tsc check reads the five __wrap_ symbols. - NOTICE.md, licenses/memfast-LICENSE.txt and pack.mjs carry the notice into NOTICE.txt. Other targets do not change. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 16 ++ NOTICE.md | 11 ++ crates/ts_goport/scripts/build-pgo.sh | 4 +- crates/ts_goport/scripts/memfast.c | 223 ++++++++++++++++++++++++++ crates/ts_goport/scripts/memfast.sh | 49 ++++++ licenses/memfast-LICENSE.txt | 21 +++ npm/README.md | 4 +- npm/pack.mjs | 1 + 8 files changed, 327 insertions(+), 2 deletions(-) create mode 100644 crates/ts_goport/scripts/memfast.c create mode 100755 crates/ts_goport/scripts/memfast.sh create mode 100644 licenses/memfast-LICENSE.txt diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b2008c93f..451773e35 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,6 +19,8 @@ # is experimental, macOS has no perf branch sampling for its profile, and BOLT has no PE support. # Linux is static musl and non-PIE, so it starts on any x86-64 or arm64 Linux, Alpine too # (npm/README.md, "Linux libc"). Each Linux tsc builds natively on a Blacksmith runner of its arch. +# The linux-x64 PGO tsc links lust's memfast.c in place of musl's memcpy, memmove, memcmp, bcmp and +# memchr (crates/ts_goport/scripts/memfast.sh): musl's memcmp and bcmp are byte loops. # Windows (x86_64-pc-windows-msvc, aarch64-pc-windows-msvc) builds in Git Bash, with the C runtime # linked in (+crt-static), so tsc.exe needs no Visual C++ redistributable. jemalloc does not build # there: it uses the system allocator. Each PGO tsc builds natively, so its training runs on its @@ -42,6 +44,7 @@ on: - "scripts/goport/pgo-train.sh" - "crates/ts_goport/scripts/copy-libs.sh" - "crates/ts_goport/scripts/build-pgo.sh" + - "crates/ts_goport/scripts/memfast.*" - "LICENSE" - "NOTICE.md" - "licenses/**" @@ -176,6 +179,13 @@ jobs: # jemalloc stops at start when the kernel page is bigger than its build page. arm64 kernels # use 4, 16 or 64 KiB pages; 2^16 covers all three. [[ $target != aarch64-* ]] || echo "JEMALLOC_SYS_WITH_LG_PAGE=16" >> "$GITHUB_ENV" + # linux-x64: lust's memfast.c in place of musl's memcpy, memmove, memcmp, bcmp and memchr + # (memfast.sh), in the PGO build only. The plain tsc keeps musl's, so the PGO check below + # also checks memfast against them. + if [[ $target == x86_64-unknown-linux-musl ]]; then + flags=$(crates/ts_goport/scripts/memfast.sh "$RUNNER_TEMP/memfast-wrap.o") + echo "PGO_RUSTFLAGS=$flags" | tee -a "$GITHUB_ENV" + fi # The plain tsc: the reference of the PGO check below, or the shipped tsc with no-pgo. PGO_LINK is # the link mode of both builds (build-pgo.sh): static on Linux, crt-static on Windows. - name: Build @@ -231,6 +241,12 @@ jobs: [[ $rc == 2 ]] && grep -q "error TS2322" check.txt if [[ $RUNNER_OS == Linux ]]; then file "$dir/tsc" | tee /dev/stderr | grep -q "statically linked" + # memfast (linux-x64): the five __wrap_ functions, and none of musl's. + if [[ ${PGO_RUSTFLAGS:-} == *--wrap=* ]]; then + syms=$(nm "$dir/tsc" | grep -E ' [Tt] (__wrap_)?(memcpy|memmove|memcmp|bcmp|memchr)$' | awk '{print $3}' | sort | xargs) + echo "mem functions: $syms" + [[ $syms == "__wrap_bcmp __wrap_memchr __wrap_memcmp __wrap_memcpy __wrap_memmove" ]] + fi # Static: it starts on musl (Alpine) too. docker run --rm -v "$dir:/t:ro" alpine:3 /t/tsc --version fi diff --git a/NOTICE.md b/NOTICE.md index e8b97ee8d..26af75582 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -51,3 +51,14 @@ The collation and normalization tables in `crates/ts_goport/src/gostd/data` and packages above. License: [licenses/Unicode-LICENSE.txt](licenses/Unicode-LICENSE.txt). + +## memfast + +Copyright (c) 2026 Theo Browne. Licensed under the MIT License. + +`crates/ts_goport/scripts/memfast.c` is `src/runtime/memfast.c` of +[t3dotgg/lust-lang](https://github.com/t3dotgg/lust-lang) (commit 6ced2b86fd), without change +below its header. The linux-x64 tsc links it in place of musl's memcpy, memmove, memcmp, bcmp and +memchr. + +License: [licenses/memfast-LICENSE.txt](licenses/memfast-LICENSE.txt). diff --git a/crates/ts_goport/scripts/build-pgo.sh b/crates/ts_goport/scripts/build-pgo.sh index 7e83314a3..7a4a48cad 100755 --- a/crates/ts_goport/scripts/build-pgo.sh +++ b/crates/ts_goport/scripts/build-pgo.sh @@ -53,6 +53,8 @@ # x86_64-unknown-linux-musl. Default: the host, with # --target only when PGO_LINK is set. # PGO_BINS the bins to build, for example "tsgo" (default: all) +# PGO_RUSTFLAGS extra rustc flags of both builds. release.yml sets the +# memfast link flags of the linux-x64 tsc (memfast.sh). # PGO_TRAIN training command, in place of the default training # (which needs the project inputs and the corpus of # GOPORT_DATA_ROOT). The script runs `$PGO_TRAIN `, @@ -189,7 +191,7 @@ triple="${PGO_TARGET:-}" # Each step needs its own target (other RUSTFLAGS). sccache is off: it could # reuse an object built with an older profile at the same path. build() { # build - local target="$out/$1" flags="$2 $link_flags" target_args="" + local target="$out/$1" flags="$2 $link_flags${PGO_RUSTFLAGS:+ $PGO_RUSTFLAGS}" target_args="" # Both builds get the same link mode, so the profile of the generate build # matches the code of the use build. [[ -n $triple ]] && target_args="--target $triple" diff --git a/crates/ts_goport/scripts/memfast.c b/crates/ts_goport/scripts/memfast.c new file mode 100644 index 000000000..4aa6290bf --- /dev/null +++ b/crates/ts_goport/scripts/memfast.c @@ -0,0 +1,223 @@ +/* memfast.c from lust-lang (https://github.com/t3dotgg/lust-lang), the file + src/runtime/memfast.c at commit 6ced2b86fd66280cf262b6c8d6b0ff61c625d26b. The text + after this header is that file without change; its comment describes lust's build. + + Copyright (c) 2026 Theo Browne + SPDX-License-Identifier: MIT + License: licenses/memfast-LICENSE.txt (NOTICE.md, "memfast"). + + ts-rust links it into the linux-x64 tsc (static musl) in place of musl's memcpy, + memmove, memcmp, bcmp and memchr: crates/ts_goport/scripts/memfast.sh. */ +/* Fast memcpy, memmove, memcmp, bcmp and memchr for musl release builds (src/cc.rs + `memfast_unit`; the release driver links this unit next to mimalloc). musl's memcmp + and memchr are byte loops, and zig's compiler_rt memcpy and memmove (which a zig musl + link uses) move a word at a time: a release tsgo spent 2.3 G instructions in them on + the TypeORM check, where glibc's vector forms do the same work in about a tenth. + These use 16-byte vectors on x86_64 (SSE2, which every x86_64 has) with unaligned head + and tail blocks, and 8-byte words elsewhere. Results are libc's (memcmp returns the + difference of the first two bytes that differ), and no function reads a byte outside + the ranges it is given. The C compiler must not turn a loop here back into a call of + the same function: `MF_FN` asks clang for no builtins and gcc for no loop patterns. + The unit has no includes (a release unit names its own types). */ +typedef __SIZE_TYPE__ mf_size; +typedef __UINTPTR_TYPE__ mf_uptr; +typedef __UINT64_TYPE__ mf_u64; +typedef __UINT32_TYPE__ mf_u32; +typedef unsigned char mf_u8; +/* Unaligned word access through types of alignment 1 (units build with + -fno-strict-aliasing). */ +typedef mf_u64 mf_u64u __attribute__((aligned(1))); +typedef mf_u32 mf_u32u __attribute__((aligned(1))); +#define MF_LD8(p) (*(const mf_u64u *)(const void *)(p)) +#define MF_ST8(p, v) (*(mf_u64u *)(void *)(p) = (v)) +#define MF_LD4(p) (*(const mf_u32u *)(const void *)(p)) +#define MF_ST4(p, v) (*(mf_u32u *)(void *)(p) = (v)) + +#if defined(__clang__) +#define MF_FN __attribute__((no_builtin)) +#elif defined(__GNUC__) +#define MF_FN __attribute__((optimize("no-tree-loop-distribute-patterns"))) +#else +#define MF_FN +#endif + +#if defined(__x86_64__) && defined(__SSE2__) && (defined(__clang__) || defined(__GNUC__)) && !defined(MF_NO_V16) +#define MF_V16 1 +typedef char mf_v16 __attribute__((vector_size(16))); +typedef char mf_v16u __attribute__((vector_size(16), aligned(1))); +#define MF_LDV(p) (*(const mf_v16u *)(const void *)(p)) +#define MF_STV(p, v) (*(mf_v16u *)(void *)(p) = (v)) +/* Bit i set when byte i of `v` (a comparison result, bytes 0 or -1) is set. */ +#define MF_MASK(v) ((unsigned)__builtin_ia32_pmovmskb128((v))) +#endif + +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define MF_LE 1 +#endif + +/* Up to 64 bytes: every load before any store, so memmove takes it for any overlap. */ +MF_FN static inline __attribute__((always_inline)) void mf_copy_small(mf_u8 *d, const mf_u8 *s, mf_size n) { + if (n <= 16) { + if (n >= 8) { + mf_u64 a = MF_LD8(s), b = MF_LD8(s + n - 8); + MF_ST8(d, a); + MF_ST8(d + n - 8, b); + } else if (n >= 4) { + mf_u32 a = MF_LD4(s), b = MF_LD4(s + n - 4); + MF_ST4(d, a); + MF_ST4(d + n - 4, b); + } else if (n) { + mf_u8 a = s[0], b = s[n >> 1], c = s[n - 1]; + d[0] = a; + d[n >> 1] = b; + d[n - 1] = c; + } + return; + } +#ifdef MF_V16 + if (n <= 32) { + mf_v16 a = MF_LDV(s), b = MF_LDV(s + n - 16); + MF_STV(d, a); + MF_STV(d + n - 16, b); + return; + } + mf_v16 a = MF_LDV(s), b = MF_LDV(s + 16), c = MF_LDV(s + n - 32), e = MF_LDV(s + n - 16); + MF_STV(d, a); + MF_STV(d + 16, b); + MF_STV(d + n - 32, c); + MF_STV(d + n - 16, e); +#else + mf_u64 w[8]; + mf_size k = 0; + for (; k + 8 <= n; k += 8) w[k >> 3] = MF_LD8(s + k); + mf_u64 t = MF_LD8(s + n - 8); + for (mf_size j = 0; j < k; j += 8) MF_ST8(d + j, w[j >> 3]); + MF_ST8(d + n - 8, t); +#endif +} + +/* More than 64 bytes, front to back: right for disjoint ranges and for d below s. The + last 64 bytes are read first, as the loop may overwrite them when d is below s. */ +MF_FN static inline __attribute__((always_inline)) void mf_copy_fwd(mf_u8 *d, const mf_u8 *s, mf_size n) { +#ifdef MF_V16 + mf_v16 t0 = MF_LDV(s + n - 64), t1 = MF_LDV(s + n - 48), t2 = MF_LDV(s + n - 32), t3 = MF_LDV(s + n - 16); + for (mf_size i = 0; n - i > 64; i += 64) { + mf_v16 a = MF_LDV(s + i), b = MF_LDV(s + i + 16), c = MF_LDV(s + i + 32), e = MF_LDV(s + i + 48); + MF_STV(d + i, a); + MF_STV(d + i + 16, b); + MF_STV(d + i + 32, c); + MF_STV(d + i + 48, e); + } + MF_STV(d + n - 64, t0); + MF_STV(d + n - 48, t1); + MF_STV(d + n - 32, t2); + MF_STV(d + n - 16, t3); +#else + mf_u64 t = MF_LD8(s + n - 8); + for (mf_size i = 0; n - i > 8; i += 8) MF_ST8(d + i, MF_LD8(s + i)); + MF_ST8(d + n - 8, t); +#endif +} + +/* More than 64 bytes, back to front, for d above s: the first 64 bytes are read first. */ +MF_FN static inline __attribute__((always_inline)) void mf_copy_bwd(mf_u8 *d, const mf_u8 *s, mf_size n) { +#ifdef MF_V16 + mf_v16 h0 = MF_LDV(s), h1 = MF_LDV(s + 16), h2 = MF_LDV(s + 32), h3 = MF_LDV(s + 48); + for (mf_size i = n; i > 64; i -= 64) { + mf_v16 a = MF_LDV(s + i - 64), b = MF_LDV(s + i - 48), c = MF_LDV(s + i - 32), e = MF_LDV(s + i - 16); + MF_STV(d + i - 64, a); + MF_STV(d + i - 48, b); + MF_STV(d + i - 32, c); + MF_STV(d + i - 16, e); + } + MF_STV(d, h0); + MF_STV(d + 16, h1); + MF_STV(d + 32, h2); + MF_STV(d + 48, h3); +#else + mf_u64 h = MF_LD8(s); + for (mf_size i = n; i > 8; i -= 8) MF_ST8(d + i - 8, MF_LD8(s + i - 8)); + MF_ST8(d, h); +#endif +} + +MF_FN void *memcpy(void *d, const void *s, mf_size n) { + if (n <= 64) mf_copy_small((mf_u8 *)d, (const mf_u8 *)s, n); + else mf_copy_fwd((mf_u8 *)d, (const mf_u8 *)s, n); + return d; +} + +MF_FN void *memmove(void *d, const void *s, mf_size n) { + if (n <= 64) mf_copy_small((mf_u8 *)d, (const mf_u8 *)s, n); + else if ((mf_uptr)d - (mf_uptr)s >= (mf_uptr)n) mf_copy_fwd((mf_u8 *)d, (const mf_u8 *)s, n); + else mf_copy_bwd((mf_u8 *)d, (const mf_u8 *)s, n); + return d; +} + +/* The first index below n where a and b differ, from index i on, or n. */ +MF_FN static inline __attribute__((always_inline)) mf_size mf_diff(const mf_u8 *a, const mf_u8 *b, mf_size i, mf_size n) { +#ifdef MF_V16 + for (; n - i >= 16; i += 16) { + unsigned m = MF_MASK(MF_LDV(a + i) == MF_LDV(b + i)) ^ 0xFFFFu; + if (m) return i + (mf_size)__builtin_ctz(m); + } + if (i < n && n >= 16) { + /* The last 16 bytes; the ones below i are equal. */ + mf_size t = n - 16; + unsigned m = MF_MASK(MF_LDV(a + t) == MF_LDV(b + t)) ^ 0xFFFFu; + return m ? t + (mf_size)__builtin_ctz(m) : n; + } +#endif +#ifdef MF_LE + for (; n - i >= 8; i += 8) { + mf_u64 x = MF_LD8(a + i) ^ MF_LD8(b + i); + if (x) return i + (mf_size)(__builtin_ctzll(x) >> 3); + } +#endif + for (; i < n; i++) + if (a[i] != b[i]) return i; + return n; +} + +MF_FN int memcmp(const void *a, const void *b, mf_size n) { + const mf_u8 *x = (const mf_u8 *)a, *y = (const mf_u8 *)b; + mf_size i = mf_diff(x, y, 0, n); + return i < n ? (int)x[i] - (int)y[i] : 0; +} + +/* bcmp(a, b, n): 0 when equal, else not 0 (the C compiler emits it for memcmp() == 0). */ +MF_FN int bcmp(const void *a, const void *b, mf_size n) { + return mf_diff((const mf_u8 *)a, (const mf_u8 *)b, 0, n) < n; +} + +MF_FN void *memchr(const void *s, int c, mf_size n) { + const mf_u8 *p = (const mf_u8 *)s; + mf_u8 b = (mf_u8)c; + mf_size i = 0; +#ifdef MF_V16 + if (n >= 16) { + mf_v16 v = (mf_v16){0} + (char)b; + for (; n - i >= 64; i += 64) { + mf_v16 e0 = MF_LDV(p + i) == v, e1 = MF_LDV(p + i + 16) == v, e2 = MF_LDV(p + i + 32) == v, e3 = MF_LDV(p + i + 48) == v; + if (MF_MASK(e0 | e1 | e2 | e3)) { + mf_u64 m = (mf_u64)MF_MASK(e0) | (mf_u64)MF_MASK(e1) << 16 | (mf_u64)MF_MASK(e2) << 32 | (mf_u64)MF_MASK(e3) << 48; + return (void *)(p + i + (mf_size)__builtin_ctzll(m)); + } + } + for (; n - i >= 16; i += 16) { + unsigned m = MF_MASK(MF_LDV(p + i) == v); + if (m) return (void *)(p + i + (mf_size)__builtin_ctz(m)); + } + if (i < n) { + /* The last 16 bytes, without the ones below i (already checked). */ + mf_size t = n - 16; + unsigned m = MF_MASK(MF_LDV(p + t) == v) & (0xFFFFu << (unsigned)(i - t)); + if (m) return (void *)(p + t + (mf_size)__builtin_ctz(m)); + } + return 0; + } +#endif + for (; i < n; i++) + if (p[i] == b) return (void *)(p + i); + return 0; +} diff --git a/crates/ts_goport/scripts/memfast.sh b/crates/ts_goport/scripts/memfast.sh new file mode 100755 index 000000000..059aba827 --- /dev/null +++ b/crates/ts_goport/scripts/memfast.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Compiles memfast.c (lust's memcpy, memmove, memcmp, bcmp and memchr; MIT, see its header) for +# the static musl linux-x64 tsc, checks the object, and prints the rustc flags that link it in +# place of musl's functions. release.yml passes them to build-pgo.sh (PGO_RUSTFLAGS). +# +# Usage: memfast.sh +# CC the C compiler (default musl-gcc) +# +# Why: musl's memcmp and bcmp are byte loops and its memcpy and memmove use rep movs. They took +# 4.4% of the cycles of a VS Code check and 9.2% on TypeORM; with memfast 2.0% and 3.3% (relmusl1, +# cycles:u --singleThreaded). Results are libc's: the tsc output does not change. +# The functions get the names __wrap_, and the link wraps the five symbols +# (-Wl,--wrap=). Plain names would clash with musl's in libc.a, because rustc puts +# -C link-arg after it. --wrap sends every call to these, musl's own calls too. memset stays +# musl's (asm). +# Checks: no undefined symbol (a loop that the compiler turns back into a memcpy call would call +# itself through the wrap), and no AVX or BMI code (the tsc runs on any x86-64). +set -euo pipefail + +case "${1:-}" in + "" | help | -h | --help) + sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed -e '$d' -e 's/^# \{0,1\}//' + exit 2 + ;; +esac +out=$1 +src="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/memfast.c" +names=(memcpy memmove memcmp bcmp memchr) +defs=() wraps="" +for n in "${names[@]}"; do + defs+=("-D$n=__wrap_$n") + wraps+="${wraps:+,}--wrap=$n" +done +# lust builds the unit with -fno-strict-aliasing (see the comment in memfast.c). +"${CC:-musl-gcc}" -O2 -fno-strict-aliasing "${defs[@]}" -c "$src" -o "$out" +undefined="$(nm -u "$out")" +if [[ -n $undefined ]]; then + echo "error: $out has undefined symbols: $undefined" >&2 + exit 1 +fi +if objdump -d --no-show-raw-insn "$out" | + grep -qE $'\t(v[a-z0-9]+|andn|bextr|blsi|blsmsk|blsr|bzhi|pdep|pext|mulx|rorx|sarx|shlx|shrx) '; then + echo "error: $out has AVX or BMI code" >&2 + exit 1 +fi +for n in "${names[@]}"; do + nm "$out" | grep -qE " T __wrap_$n\$" || { echo "error: $out has no __wrap_$n" >&2; exit 1; } +done +echo "-Clink-arg=-Wl,$wraps -Clink-arg=$out" diff --git a/licenses/memfast-LICENSE.txt b/licenses/memfast-LICENSE.txt new file mode 100644 index 000000000..eccbd9f1c --- /dev/null +++ b/licenses/memfast-LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Theo Browne + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/npm/README.md b/npm/README.md index 1825e3f10..26bb4b76f 100644 --- a/npm/README.md +++ b/npm/README.md @@ -121,7 +121,9 @@ package. The CI builds (but darwin-x64 and win32-arm64) are PGO builds of the shipped profile (fat LTO): `build-pgo.sh` trains each platform's tsc on the set of `scripts/goport/pgo-train.sh`, and the job -fails unless the PGO tsc gives the same output as a plain build on that set. They have no BOLT: BOLT +fails unless the PGO tsc gives the same output as a plain build on that set. The Linux x64 PGO tsc +links `crates/ts_goport/scripts/memfast.c` in place of musl's memcpy, memmove, memcmp, bcmp and +memchr (`memfast.sh`); the plain build keeps musl's. They have no BOLT: BOLT refuses the static Linux bin, macOS has no perf branch sampling for its profile, and BOLT has no PE (Windows) support. To make the Linux x64 tsc by hand on zbook, use `RELEASE_FEATURES=noembed RELEASE_LIBC=musl RELEASE_PIE=0 crates/ts_goport/scripts/build-release.sh /linux-x64` and pack diff --git a/npm/pack.mjs b/npm/pack.mjs index 93af24ead..e8b4b38c3 100644 --- a/npm/pack.mjs +++ b/npm/pack.mjs @@ -179,6 +179,7 @@ function portNotice() { ...part("TypeScript third-party notices", tsNoticeFile), ...part("Go license", path.join(repoRoot, "licenses", "Go-LICENSE.txt")), ...part("Unicode license", path.join(repoRoot, "licenses", "Unicode-LICENSE.txt")), + ...part("memfast license", path.join(repoRoot, "licenses", "memfast-LICENSE.txt")), ].join("\n"); } function writeLicense(dir) { From 81e39a400b4f9f41f3dd384505da99ebc53f4f1c Mon Sep 17 00:00:00 2001 From: Theo Browne <6751787+t3dotgg@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:09:37 -0700 Subject: [PATCH 2/5] perf(release): BOLT the linux-x64 tsc The linux-x64 tsc had no BOLT: build-bolt.sh refuses static bins, because BOLT breaks static glibc. Static musl works (relmusl1) with two changes against build-release.sh's recipe: - A static bin runs no BOLT code at exit, so the counting copy writes its counts from a watcher every second (-instrumentation-sleep-time=1 -instrumentation-no-counters-clear). Some watchers spin after their last write; the script kills them. - No -split-eh. It gives moved landing pads an LPStart, and LLVM libunwind (the musl target's unwinder) aborts on such an LSDA, so a caught panic would end the tsc. bolt-musl.sh trains a counting copy on pgo-train.sh's set and rewrites the PGO tsc. check-unwind.py then checks the BOLT tsc: a real caught panic (API transpileDeclaration of a .d.ts, which Go answers with a recovered assert) must answer and the process must go on, and every LSDA must start with 0xFF. build-pgo.sh keeps --emit-relocs for the static link with PGO_EMIT_RELOCS=1. release.yml installs LLVM 23 BOLT on linux-x64 and runs the step before the PGO check, which then compares the BOLT tsc with the plain tsc. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 28 +++++- crates/ts_goport/scripts/bolt-musl.sh | 83 ++++++++++++++++++ crates/ts_goport/scripts/build-pgo.sh | 9 +- crates/ts_goport/scripts/check-unwind.py | 105 +++++++++++++++++++++++ npm/README.md | 11 +-- 5 files changed, 225 insertions(+), 11 deletions(-) create mode 100755 crates/ts_goport/scripts/bolt-musl.sh create mode 100755 crates/ts_goport/scripts/check-unwind.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 451773e35..4f1fb39f1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,9 +14,11 @@ # The tsc is a PGO build of the shipped profile (fat LTO), trained on each platform: # crates/ts_goport/scripts/build-pgo.sh with the training set of scripts/goport/pgo-train.sh (7 # projects at fixed commits). The job also builds the plain tsc, and fails unless the PGO tsc gives -# the same stdout, stderr, exit code and emitted files as the plain tsc on that set. No BOLT -# (build-release.sh has it): BOLT refuses the static Linux bin (build-bolt.sh), its Mach-O support -# is experimental, macOS has no perf branch sampling for its profile, and BOLT has no PE support. +# the same stdout, stderr, exit code and emitted files as the plain tsc on that set. linux-x64 also +# gets BOLT: crates/ts_goport/scripts/bolt-musl.sh, with a profile from a counting copy that runs +# the same set, and no -split-eh (it breaks the unwinding of a caught panic with musl). The PGO +# check then compares the BOLT tsc with the plain tsc. The others have no BOLT: linux-arm64 is not +# tried yet, BOLT's Mach-O support is experimental, and BOLT has no PE support. # Linux is static musl and non-PIE, so it starts on any x86-64 or arm64 Linux, Alpine too # (npm/README.md, "Linux libc"). Each Linux tsc builds natively on a Blacksmith runner of its arch. # The linux-x64 PGO tsc links lust's memfast.c in place of musl's memcpy, memmove, memcmp, bcmp and @@ -45,6 +47,8 @@ on: - "crates/ts_goport/scripts/copy-libs.sh" - "crates/ts_goport/scripts/build-pgo.sh" - "crates/ts_goport/scripts/memfast.*" + - "crates/ts_goport/scripts/bolt-musl.sh" + - "crates/ts_goport/scripts/check-unwind.py" - "LICENSE" - "NOTICE.md" - "licenses/**" @@ -137,6 +141,15 @@ jobs: - run: rustup toolchain install "$RUSTUP_TOOLCHAIN" --profile minimal --target ${{ matrix.target }} --component llvm-tools - if: runner.os == 'Linux' run: sudo apt-get update -q && sudo apt-get install -yq musl-tools + # linux-x64: llvm-bolt, merge-fdata, llvm-objcopy and llvm-dwarfdump of LLVM 23 for bolt-musl.sh. + - name: BOLT tools + if: matrix.platform == 'linux-x64' + run: | + curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null + codename=$(. /etc/os-release && echo "$VERSION_CODENAME") + echo "deb https://apt.llvm.org/$codename/ llvm-toolchain-$codename-23 main" | sudo tee /etc/apt/sources.list.d/llvm-23.list + sudo apt-get update -q && sudo apt-get install -yq bolt-23 llvm-23 + /usr/lib/llvm-23/bin/llvm-bolt --version | grep -m1 'LLVM version' # darwin-x64 on an arm64 Mac: the check runs the x86-64 tsc, so Rosetta 2 must be there. - name: Rosetta if: matrix.rosetta @@ -211,10 +224,19 @@ jobs: PGO_BINS: tsgo PGO_CARGO: cargo CARGO_FEATURES: --features noembed + # The BOLT input needs the static relocations. + PGO_EMIT_RELOCS: ${{ matrix.platform == 'linux-x64' && '1' || '' }} run: | PGO_TRAIN="scripts/goport/pgo-train.sh run $RUNNER_TEMP/train" crates/ts_goport/scripts/build-pgo.sh "$RUNNER_TEMP/pgo" mkdir -p "$RUNNER_TEMP/out" cp "$RUNNER_TEMP/pgo/target-use/goport/tsgo${{ matrix.exe }}" "$RUNNER_TEMP/out/tsc${{ matrix.exe }}" + # linux-x64: BOLT of the PGO tsc. bolt-musl.sh also checks that the BOLT tsc still unwinds a + # caught panic (check-unwind.py). + - name: BOLT + if: matrix.platform == 'linux-x64' + run: | + crates/ts_goport/scripts/bolt-musl.sh "$RUNNER_TEMP/out/tsc" "$RUNNER_TEMP/train" "$RUNNER_TEMP/bolt" + cp "$RUNNER_TEMP/bolt/bin/tsc" "$RUNNER_TEMP/out/tsc" - name: Check the PGO tsc against the plain tsc if: ${{ !matrix.no-pgo }} run: scripts/goport/pgo-train.sh compare "$RUNNER_TEMP/train" "$RUNNER_TEMP/plain/tsc${{ matrix.exe }}" "$RUNNER_TEMP/out/tsc${{ matrix.exe }}" diff --git a/crates/ts_goport/scripts/bolt-musl.sh b/crates/ts_goport/scripts/bolt-musl.sh new file mode 100755 index 000000000..4ee276288 --- /dev/null +++ b/crates/ts_goport/scripts/bolt-musl.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# BOLT of the static musl linux-x64 tsc (release.yml). BOLT puts the hot blocks of each function +# together, moves the cold blocks out and orders the functions. The program does not change. +# The profile comes from a counting copy (llvm-bolt -instrument) that runs the training set of +# scripts/goport/pgo-train.sh: the release runner has no perf branch sampling. +# +# Usage: bolt-musl.sh +# tsc the PGO tsc, linked with --emit-relocs (build-pgo.sh PGO_EMIT_RELOCS=1) +# train-dir a pgo-train.sh setup dir +# out-dir new dir; the BOLT tsc is /bin/tsc (with the lib files of a noembed tsc), +# the logs and counts stay in +# LLVM_BIN dir of llvm-bolt, merge-fdata, llvm-objcopy and llvm-dwarfdump +# (default /usr/lib/llvm-23/bin) +# +# Static bins differ from build-release.sh's dynamic ones in three ways (relmusl1): +# - A static bin runs no BOLT code at exit, so the counting copy writes its counts from a +# watcher process every second (-instrumentation-sleep-time=1), each time all of them +# (-instrumentation-no-counters-clear). Some watchers spin after their last write and block +# every signal but SIGKILL: the script kills them. +# - No -split-eh. It moves landing pads to the cold part and gives their LSDA a start address +# (LPStart). The musl target unwinds with LLVM libunwind, which aborts on such an LSDA +# ("lsda at ... does not start with 0xFF"), so a caught panic would end the tsc. +# - The relocation mode needs the static relocations (--emit-relocs). +# Checks of the BOLT tsc: no BOLT runtime code (no __bolt symbols), one RW PT_GNU_STACK, and +# check-unwind.py (a caught panic in the tsc, and every LSDA starts with 0xFF). release.yml then +# compares its output with the plain tsc on the training set. +set -euo pipefail + +case "${1:-}" in + help | -h | --help) + sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed -e '$d' -e 's/^# \{0,1\}//' + exit 0 + ;; +esac +(($# == 3)) || { echo "usage: bolt-musl.sh " >&2; exit 2; } +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo="$(cd -- "$script_dir/../../.." && pwd)" +tsc=$1 +train="$(cd -- "$2" && pwd)" +out=$3 +llvm="${LLVM_BIN:-/usr/lib/llvm-23/bin}" +[[ ! -e $out ]] || { echo "error: $out exists" >&2; exit 2; } +mkdir -p "$out/counts" "$out/bin" +out="$(cd -- "$out" && pwd)" +readelf -SW "$tsc" | grep -q ' \.rela\.text ' || + { echo "error: $tsc has no .rela.text (link it with --emit-relocs: build-pgo.sh PGO_EMIT_RELOCS=1)" >&2; exit 1; } + +# BOLT reads a symbol with ".cold" or ".warm" in it as a split part of another function and stops +# ("parent function not found"). Rust paths make such names ("Session..warm_auto_import_cache"). +# Rename them in a copy: "..warm" to "..Warm" (same length, only .symtab changes). +nm "$tsc" | awk '$2 ~ /^[tT]$/ && $3 ~ /\.\.(cold|warm)/ {r = $3; gsub(/\.\.cold/, "..Cold", r); gsub(/\.\.warm/, "..Warm", r); print $3, r}' > "$out/rename.txt" +"$llvm/llvm-objcopy" --redefine-syms="$out/rename.txt" "$tsc" "$out/input" + +# The counting copy. pgo-train.sh runs its copy at /bin/tsc. +"$llvm/llvm-bolt" "$out/input" -o "$out/instr" -instrument -instrumentation-file="$out/counts/tsc" \ + -instrumentation-file-append-pid -instrumentation-binpath="$train/bin/tsc" \ + -instrumentation-sleep-time=1 -instrumentation-no-counters-clear > "$out/instr.log" 2>&1 || + { tail -20 "$out/instr.log" >&2; exit 1; } +# One process per run: tsgo on 4 KiB pages would start a worker (bin/tsgo.rs `launch`). +GOPORT_LAUNCH=0 "$repo/scripts/goport/pgo-train.sh" run "$train" "$out/instr" > "$out/train.log" 2>&1 || + { tail -20 "$out/train.log" >&2; exit 1; } +# The watchers write up to about 1 s after their tsc ends. +sleep 5 +left=$(pgrep -fc "^$train/bin/tsc" || true) +pkill -9 -f "^$train/bin/tsc" || true +echo "bolt training: $(find "$out/counts" -name 'tsc.*.fdata' | wc -l) count files, $left watchers killed" +"$llvm/merge-fdata" "$out"/counts/tsc.*.fdata > "$out/tsc.fdata" 2> "$out/merge.log" || + { tail -5 "$out/merge.log" >&2; exit 1; } + +"$llvm/llvm-bolt" "$out/input" -o "$out/bin/tsc" -data="$out/tsc.fdata" \ + -reorder-blocks=ext-tsp -reorder-functions=cdsort -split-functions -split-all-cold > "$out/bolt.log" 2>&1 || + { tail -20 "$out/bolt.log" >&2; exit 1; } +echo "bolt: $(grep -m1 -o '[0-9]* out of [0-9]* functions in the binary ([0-9.]*%) have non-empty execution profile' "$out/bolt.log" || true)" + +if nm "$out/bin/tsc" | grep -q '__bolt'; then + echo "error: $out/bin/tsc has BOLT runtime symbols (the counting copy?)" >&2 + exit 1 +fi +stack="$(readelf -lW "$out/bin/tsc" | awk '$1 == "GNU_STACK" { s = ""; for (i = 7; i < NF; i++) s = s $i; print s }')" +[[ $stack == RW ]] || { echo "error: $out/bin/tsc has PT_GNU_STACK '$stack', not one RW" >&2; exit 1; } +"$script_dir/copy-libs.sh" "$out/bin" > /dev/null +python3 "$script_dir/check-unwind.py" "$out/bin/tsc" "$llvm/llvm-dwarfdump" +echo "BOLT tsc: $out/bin/tsc" diff --git a/crates/ts_goport/scripts/build-pgo.sh b/crates/ts_goport/scripts/build-pgo.sh index 7a4a48cad..5499c91ec 100755 --- a/crates/ts_goport/scripts/build-pgo.sh +++ b/crates/ts_goport/scripts/build-pgo.sh @@ -27,8 +27,9 @@ # the layout when BOLT is not used. # --emit-relocs keeps the static relocations in the file (non-alloc # sections, not mapped at run time). BOLT needs them for relocation -# mode, where it also reorders functions. Not with PGO_LINK=static: -# build-bolt.sh refuses static bins, so they only make the file bigger. +# mode, where it also reorders functions. Not with PGO_LINK=static +# unless PGO_EMIT_RELOCS=1 (bolt-musl.sh, release.yml's linux-x64 +# tsc): else they only make the file bigger. # The build prints the .text and .rela.text sections of tsgo # (readelf -S). Expect .text.hot, .text.unlikely and .rela.text. GNU ld (the # linker of the musl target) ignores keep-text-section-prefix, but its default @@ -55,6 +56,8 @@ # PGO_BINS the bins to build, for example "tsgo" (default: all) # PGO_RUSTFLAGS extra rustc flags of both builds. release.yml sets the # memfast link flags of the linux-x64 tsc (memfast.sh). +# PGO_EMIT_RELOCS=1 link the use build with --emit-relocs also when +# PGO_LINK=static (the input of bolt-musl.sh) # PGO_TRAIN training command, in place of the default training # (which needs the project inputs and the corpus of # GOPORT_DATA_ROOT). The script runs `$PGO_TRAIN `, @@ -303,7 +306,7 @@ echo "merged $(du -h "$merged" | cut -f1) -> $merged" # The Linux link args are the r3-link layout (see the header). use_flags="-Cprofile-use=$merged" if [[ $(uname -s) == Linux ]]; then - [[ $link == static ]] || use_flags+=" -Clink-arg=-Wl,--emit-relocs" + [[ $link == static && ${PGO_EMIT_RELOCS:-0} != 1 ]] || use_flags+=" -Clink-arg=-Wl,--emit-relocs" use_flags+=" -Clink-arg=-Wl,-z,keep-text-section-prefix" fi build target-use "$use_flags" diff --git a/crates/ts_goport/scripts/check-unwind.py b/crates/ts_goport/scripts/check-unwind.py new file mode 100755 index 000000000..bc691e0c9 --- /dev/null +++ b/crates/ts_goport/scripts/check-unwind.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Checks that a tsc still unwinds a caught panic. release.yml runs it on the BOLT copy of the +static musl linux-x64 tsc (bolt-musl.sh). The port catches panics as Go recovers them, so a tsc +that aborts on a panic gives a crash where Go gives an answer. + +Usage: check-unwind.py [] + +1. A caught panic in the tsc itself: `tsc --api --async` gets transpileDeclaration of a .d.ts + file. Go's transpile.go asserts that the emit wrote a file ("Output generation failed"), and + the API answers the panic as an error response (Go's request recover; the port's go_recover in + ipc/conn_async.rs). The same process must then answer transpileModule, and exit 0 when its + stdin closes. A tsc that cannot unwind aborts at the panic. +2. With : every LSDA (the exception table of a function, named by its FDE in + .eh_frame) starts with 0xFF (DW_EH_PE_omit: no LPStart). The musl target unwinds with LLVM + libunwind, which aborts on any other first byte ("lsda at ... does not start with 0xFF"). + llvm-bolt -split-eh writes such tables (2,180 in a relmusl1 test bin), so a panic through one + of those functions aborts. This checks every function, not only the path of 1. +""" +import json +import re +import signal +import struct +import subprocess +import sys +import tempfile + + +def fail(message): + sys.exit(f"check-unwind: {message}") + + +def caught_panic(tsc): + signal.alarm(120) + with tempfile.TemporaryDirectory() as cwd: + proc = subprocess.Popen([tsc, "--api", "--async", "--cwd", cwd], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + def ended(method): + fail(f"{method}: the tsc ended (exit {proc.wait()}): {proc.stderr.read()[-400:]!r}") + + # request: one JSON-RPC call (Content-Length framing), and its response. + def request(id, method, params): + body = json.dumps({"jsonrpc": "2.0", "id": id, "method": method, "params": params}).encode() + proc.stdin.write(b"Content-Length: %d\r\n\r\n" % len(body) + body) + proc.stdin.flush() + while True: + length = None + while (line := proc.stdout.readline()) != b"\r\n" or length is None: + if not line: + ended(method) + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":")[1]) + body = proc.stdout.read(length) + if len(body) < length: + ended(method) + message = json.loads(body) + if message.get("id") == id: + return message + + request(1, "initialize", None) + answer = request(2, "transpileDeclaration", + {"input": "declare const x: number;\n", "options": {"fileName": "a.d.ts"}}) + error = answer.get("error", {}).get("message", "") + if not error.startswith("panic: ") or "Output generation failed" not in error: + fail(f"transpileDeclaration of a .d.ts: no caught panic: {json.dumps(answer)[:400]}") + answer = request(3, "transpileModule", {"input": "const a = 1;", "options": {}}) + if "const a = 1;" not in answer.get("result", {}).get("outputText", ""): + fail(f"transpileModule after the panic: {json.dumps(answer)[:400]}") + proc.stdin.close() + code = proc.wait() + if code != 0: + fail(f"the tsc exited {code} after the panic: {proc.stderr.read()[-400:]!r}") + signal.alarm(0) + print(f"caught panic: {error.splitlines()[0]}; the next request answered, exit 0") + + +def lsda_starts(tsc, dwarfdump): + data = open(tsc, "rb").read() + phoff, = struct.unpack_from("/linux-x64` and pack -and test it: +memchr (`memfast.sh`); the plain build keeps musl's. It also gets BOLT (`bolt-musl.sh`: a counting +copy trains on the same set; no `-split-eh`, which breaks the unwinding of a caught panic with musl), +and the check compares the BOLT tsc with the plain build. The others have no BOLT: linux-arm64 is not +tried yet, BOLT's Mach-O support is experimental, and BOLT has no PE (Windows) support. To make +the Linux x64 tsc by hand on zbook, use `RELEASE_FEATURES=noembed RELEASE_LIBC=musl RELEASE_PIE=0 +crates/ts_goport/scripts/build-release.sh /linux-x64` and pack and test it: ```sh GOPORT_PIN= scripts/goport/npm-pack.sh --name tsc-rs --package-version \ From 6699eb7420ab3c456fdee2f32cec4b2f3375f716 Mon Sep 17 00:00:00 2001 From: Theo Browne <6751787+t3dotgg@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:52:23 -0700 Subject: [PATCH 3/5] fix(release): count matches in the memfast and BOLT checks With pipefail, `cmd | grep -q` can fail when grep stops early and cmd gets SIGPIPE, so a check could miss AVX code or BOLT runtime symbols. Count the matches instead (grep -c reads all input). The built bins do not change. Co-Authored-By: Claude Opus 5.5 --- crates/ts_goport/scripts/bolt-musl.sh | 5 +++-- crates/ts_goport/scripts/memfast.sh | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/ts_goport/scripts/bolt-musl.sh b/crates/ts_goport/scripts/bolt-musl.sh index 4ee276288..4e2799d02 100755 --- a/crates/ts_goport/scripts/bolt-musl.sh +++ b/crates/ts_goport/scripts/bolt-musl.sh @@ -42,7 +42,8 @@ llvm="${LLVM_BIN:-/usr/lib/llvm-23/bin}" [[ ! -e $out ]] || { echo "error: $out exists" >&2; exit 2; } mkdir -p "$out/counts" "$out/bin" out="$(cd -- "$out" && pwd)" -readelf -SW "$tsc" | grep -q ' \.rela\.text ' || +# grep -c, not -q: with pipefail, a grep that stops early can fail the pipe (SIGPIPE). +[[ $(readelf -SW "$tsc" | grep -c ' \.rela\.text ') != 0 ]] || { echo "error: $tsc has no .rela.text (link it with --emit-relocs: build-pgo.sh PGO_EMIT_RELOCS=1)" >&2; exit 1; } # BOLT reads a symbol with ".cold" or ".warm" in it as a split part of another function and stops @@ -72,7 +73,7 @@ echo "bolt training: $(find "$out/counts" -name 'tsc.*.fdata' | wc -l) count fil { tail -20 "$out/bolt.log" >&2; exit 1; } echo "bolt: $(grep -m1 -o '[0-9]* out of [0-9]* functions in the binary ([0-9.]*%) have non-empty execution profile' "$out/bolt.log" || true)" -if nm "$out/bin/tsc" | grep -q '__bolt'; then +if [[ $(nm "$out/bin/tsc" | grep -c '__bolt') != 0 ]]; then echo "error: $out/bin/tsc has BOLT runtime symbols (the counting copy?)" >&2 exit 1 fi diff --git a/crates/ts_goport/scripts/memfast.sh b/crates/ts_goport/scripts/memfast.sh index 059aba827..51818976d 100755 --- a/crates/ts_goport/scripts/memfast.sh +++ b/crates/ts_goport/scripts/memfast.sh @@ -38,12 +38,13 @@ if [[ -n $undefined ]]; then echo "error: $out has undefined symbols: $undefined" >&2 exit 1 fi -if objdump -d --no-show-raw-insn "$out" | - grep -qE $'\t(v[a-z0-9]+|andn|bextr|blsi|blsmsk|blsr|bzhi|pdep|pext|mulx|rorx|sarx|shlx|shrx) '; then +# grep -c, not -q: with pipefail, a grep that stops early can fail the pipe (SIGPIPE). +if [[ $(objdump -d --no-show-raw-insn "$out" | + grep -cE $'\t(v[a-z0-9]+|andn|bextr|blsi|blsmsk|blsr|bzhi|pdep|pext|mulx|rorx|sarx|shlx|shrx) ') != 0 ]]; then echo "error: $out has AVX or BMI code" >&2 exit 1 fi for n in "${names[@]}"; do - nm "$out" | grep -qE " T __wrap_$n\$" || { echo "error: $out has no __wrap_$n" >&2; exit 1; } + [[ $(nm "$out" | grep -cE " T __wrap_$n\$") == 1 ]] || { echo "error: $out has no __wrap_$n" >&2; exit 1; } done echo "-Clink-arg=-Wl,$wraps -Clink-arg=$out" From 4137e9913997e24d6dcaadd52e1950b2091830cf Mon Sep 17 00:00:00 2001 From: Theo Browne <6751787+t3dotgg@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:09:41 -0700 Subject: [PATCH 4/5] fix(release): no SIGPIPE in the tsc check `tsc --listFilesOnly | grep -qx` failed on darwin-x64 with exit 141: grep -q ended at the first match, and the tsc died of SIGPIPE under pipefail. grep without -q reads all of the output. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4f1fb39f1..3f3847adf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -254,7 +254,9 @@ jobs: [[ $RUNNER_OS != Windows ]] || listed=$(cygpath -m "$listed") cd "$RUNNER_TEMP" echo 'const x: number = "a";' > a.ts - "$dir/tsc$EXE" --listFilesOnly --lib es5 a.ts | grep -qx "$listed" + # Not grep -q: it can end before the tsc does, and the tsc then dies of SIGPIPE (exit 141 with + # pipefail; seen on darwin-x64). + "$dir/tsc$EXE" --listFilesOnly --lib es5 a.ts | grep -x "$listed" > /dev/null set +e "$dir/tsc$EXE" --noEmit a.ts > check.txt rc=$? From 663cf0c240a627ee873490a6bbd876fc74f8b255 Mon Sep 17 00:00:00 2001 From: Theo Browne <6751787+t3dotgg@users.noreply.github.com> Date: Fri, 23 Oct 2026 08:22:12 -0700 Subject: [PATCH 5/5] fix(release): check-unwind crosses moved frames, fix unwind wording check-unwind.py check 1 now runs a second caught panic: --callbacks readFile:error, then createSnapshot. The tsconfig read panics inside the config load, and in the CI BOLT tsc this unwind crosses 3 frames that BOLT moved, each with a landing pad. Check 1 also fails when stderr has a "libunwind:" line. LLVM libunwind only logs "lsda at ... does not start with 0xFF". It does not always abort. The texts of check-unwind.py, bolt-musl.sh, release.yml and npm/README.md now say that -split-eh can break the unwind, and that check 2 is a cautious guard. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 2 +- crates/ts_goport/scripts/bolt-musl.sh | 10 +-- crates/ts_goport/scripts/check-unwind.py | 86 ++++++++++++++++-------- npm/README.md | 6 +- 4 files changed, 68 insertions(+), 36 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3f3847adf..d20f7be8c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ # projects at fixed commits). The job also builds the plain tsc, and fails unless the PGO tsc gives # the same stdout, stderr, exit code and emitted files as the plain tsc on that set. linux-x64 also # gets BOLT: crates/ts_goport/scripts/bolt-musl.sh, with a profile from a counting copy that runs -# the same set, and no -split-eh (it breaks the unwinding of a caught panic with musl). The PGO +# the same set, and no -split-eh (it can break the unwinding of a caught panic with musl). The PGO # check then compares the BOLT tsc with the plain tsc. The others have no BOLT: linux-arm64 is not # tried yet, BOLT's Mach-O support is experimental, and BOLT has no PE support. # Linux is static musl and non-PIE, so it starts on any x86-64 or arm64 Linux, Alpine too diff --git a/crates/ts_goport/scripts/bolt-musl.sh b/crates/ts_goport/scripts/bolt-musl.sh index 4e2799d02..423292392 100755 --- a/crates/ts_goport/scripts/bolt-musl.sh +++ b/crates/ts_goport/scripts/bolt-musl.sh @@ -18,12 +18,14 @@ # (-instrumentation-no-counters-clear). Some watchers spin after their last write and block # every signal but SIGKILL: the script kills them. # - No -split-eh. It moves landing pads to the cold part and gives their LSDA a start address -# (LPStart). The musl target unwinds with LLVM libunwind, which aborts on such an LSDA -# ("lsda at ... does not start with 0xFF"), so a caught panic would end the tsc. +# (LPStart). The musl target unwinds with LLVM libunwind, which logs "lsda at ... does not +# start with 0xFF" for such an LSDA. In some cases a caught panic through it then aborts +# (relmusl1's test program: "failed to initiate panic"); in others it works. # - The relocation mode needs the static relocations (--emit-relocs). # Checks of the BOLT tsc: no BOLT runtime code (no __bolt symbols), one RW PT_GNU_STACK, and -# check-unwind.py (a caught panic in the tsc, and every LSDA starts with 0xFF). release.yml then -# compares its output with the plain tsc on the training set. +# check-unwind.py (two caught panics in the tsc with no libunwind line, and every LSDA starts with +# 0xFF: a cautious guard that can reject a tsc that would work). release.yml then compares its +# output with the plain tsc on the training set. set -euo pipefail case "${1:-}" in diff --git a/crates/ts_goport/scripts/check-unwind.py b/crates/ts_goport/scripts/check-unwind.py index bc691e0c9..b2863fc2b 100755 --- a/crates/ts_goport/scripts/check-unwind.py +++ b/crates/ts_goport/scripts/check-unwind.py @@ -5,16 +5,24 @@ Usage: check-unwind.py [] -1. A caught panic in the tsc itself: `tsc --api --async` gets transpileDeclaration of a .d.ts - file. Go's transpile.go asserts that the emit wrote a file ("Output generation failed"), and - the API answers the panic as an error response (Go's request recover; the port's go_recover in - ipc/conn_async.rs). The same process must then answer transpileModule, and exit 0 when its - stdin closes. A tsc that cannot unwind aborts at the panic. +1. Two caught panics in the tsc itself, each in a new `tsc --api --async`. The API answers the + panic as an error response (Go's request recover; the port's go_recover in + ipc/conn_async.rs). The same process must then answer transpileModule, exit 0 when its stdin + closes, and write no "libunwind:" line to stderr. A tsc that cannot unwind aborts at the panic. + a. transpileDeclaration of a .d.ts file. Go's transpile.go asserts that the emit wrote a file + ("Output generation failed"). + b. With --callbacks readFile:error, createSnapshot of a project. The read of its tsconfig + panics (Go's callbackfs.go panicIfError) inside the config load. In the BOLT tsc of + Release run 39967282137, this unwind crosses 3 tsoptions frames that BOLT moved, each with + a landing pad. The unwind of a. crosses no moved frame there. 2. With : every LSDA (the exception table of a function, named by its FDE in - .eh_frame) starts with 0xFF (DW_EH_PE_omit: no LPStart). The musl target unwinds with LLVM - libunwind, which aborts on any other first byte ("lsda at ... does not start with 0xFF"). - llvm-bolt -split-eh writes such tables (2,180 in a relmusl1 test bin), so a panic through one - of those functions aborts. This checks every function, not only the path of 1. + .eh_frame) starts with 0xFF (DW_EH_PE_omit: no LPStart). llvm-bolt -split-eh writes LSDAs + with an LPStart (2,180 in a relmusl1 test tsc). The musl target unwinds with LLVM libunwind, + which logs "lsda at ... does not start with 0xFF" for such an LSDA. That line is only a log. + In relmusl1's small test program, a caught panic through such LSDAs aborted ("failed to + initiate panic"), but the -split-eh test tsc still caught the panic of 1b. So this check is a + cautious guard: it can reject a tsc that would work, but it does not pass a tsc that has such + an LSDA. It checks every function, not only the paths of 1. """ import json import re @@ -29,49 +37,71 @@ def fail(message): sys.exit(f"check-unwind: {message}") -def caught_panic(tsc): +# api_panic: one run of `tsc --api --async ` in . must give the caught panic +# ; then transpileModule must answer, the tsc must exit 0, and stderr has no libunwind line. +def api_panic(tsc, cwd, args, method, params, panic): signal.alarm(120) - with tempfile.TemporaryDirectory() as cwd: - proc = subprocess.Popen([tsc, "--api", "--async", "--cwd", cwd], stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + with tempfile.TemporaryFile() as stderr: + proc = subprocess.Popen([tsc, "--api", "--async", *args, "--cwd", cwd], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=stderr) + + def stderr_text(): + stderr.seek(0) + return stderr.read().decode(errors="replace") - def ended(method): - fail(f"{method}: the tsc ended (exit {proc.wait()}): {proc.stderr.read()[-400:]!r}") + def ended(name): + fail(f"{method}: {name}: the tsc ended (exit {proc.wait()}): {stderr_text()[-400:]!r}") # request: one JSON-RPC call (Content-Length framing), and its response. - def request(id, method, params): - body = json.dumps({"jsonrpc": "2.0", "id": id, "method": method, "params": params}).encode() + def request(id, name, params): + body = json.dumps({"jsonrpc": "2.0", "id": id, "method": name, "params": params}).encode() proc.stdin.write(b"Content-Length: %d\r\n\r\n" % len(body) + body) proc.stdin.flush() while True: length = None while (line := proc.stdout.readline()) != b"\r\n" or length is None: if not line: - ended(method) + ended(name) if line.lower().startswith(b"content-length:"): length = int(line.split(b":")[1]) body = proc.stdout.read(length) if len(body) < length: - ended(method) + ended(name) message = json.loads(body) - if message.get("id") == id: + if message.get("id") == id and "method" not in message: return message request(1, "initialize", None) - answer = request(2, "transpileDeclaration", - {"input": "declare const x: number;\n", "options": {"fileName": "a.d.ts"}}) + answer = request(2, method, params) error = answer.get("error", {}).get("message", "") - if not error.startswith("panic: ") or "Output generation failed" not in error: - fail(f"transpileDeclaration of a .d.ts: no caught panic: {json.dumps(answer)[:400]}") + if not error.startswith("panic: ") or panic not in error: + fail(f"{method}: no caught panic: {json.dumps(answer)[:400]}") answer = request(3, "transpileModule", {"input": "const a = 1;", "options": {}}) if "const a = 1;" not in answer.get("result", {}).get("outputText", ""): - fail(f"transpileModule after the panic: {json.dumps(answer)[:400]}") + fail(f"{method}: transpileModule after the panic: {json.dumps(answer)[:400]}") proc.stdin.close() code = proc.wait() if code != 0: - fail(f"the tsc exited {code} after the panic: {proc.stderr.read()[-400:]!r}") + fail(f"{method}: the tsc exited {code} after the panic: {stderr_text()[-400:]!r}") + logs = [line for line in stderr_text().splitlines() if "libunwind:" in line] + if logs: + fail(f"{method}: {len(logs)} libunwind lines on stderr, for example {logs[0]!r}") signal.alarm(0) - print(f"caught panic: {error.splitlines()[0]}; the next request answered, exit 0") + print(f"{method}: caught panic ({error.splitlines()[0]}); the next request answered, exit 0") + + +def caught_panics(tsc): + with tempfile.TemporaryDirectory() as cwd: + api_panic(tsc, cwd, [], "transpileDeclaration", + {"input": "declare const x: number;\n", "options": {"fileName": "a.d.ts"}}, + "Output generation failed") + with open(f"{cwd}/tsconfig.json", "w") as f: + f.write('{"compilerOptions": {"strict": true, "noEmit": true}, "files": ["a.ts"]}\n') + with open(f"{cwd}/a.ts", "w") as f: + f.write("export const x: number = 1;\n") + api_panic(tsc, cwd, ["--callbacks", "readFile:error"], "createSnapshot", + {"openProjects": [f"{cwd}/tsconfig.json"]}, + "filesystem operation configured with serverFS.error: readFile") def lsda_starts(tsc, dwarfdump): @@ -100,6 +130,6 @@ def lsda_starts(tsc, dwarfdump): if len(sys.argv) not in (2, 3): sys.exit(__doc__) -caught_panic(sys.argv[1]) +caught_panics(sys.argv[1]) if len(sys.argv) == 3: lsda_starts(sys.argv[1], sys.argv[2]) diff --git a/npm/README.md b/npm/README.md index 1e86d1867..dc65b2b3b 100644 --- a/npm/README.md +++ b/npm/README.md @@ -124,9 +124,9 @@ The CI builds (but darwin-x64 and win32-arm64) are PGO builds of the shipped pro fails unless the PGO tsc gives the same output as a plain build on that set. The Linux x64 PGO tsc links `crates/ts_goport/scripts/memfast.c` in place of musl's memcpy, memmove, memcmp, bcmp and memchr (`memfast.sh`); the plain build keeps musl's. It also gets BOLT (`bolt-musl.sh`: a counting -copy trains on the same set; no `-split-eh`, which breaks the unwinding of a caught panic with musl), -and the check compares the BOLT tsc with the plain build. The others have no BOLT: linux-arm64 is not -tried yet, BOLT's Mach-O support is experimental, and BOLT has no PE (Windows) support. To make +copy trains on the same set; no `-split-eh`, which can break the unwinding of a caught panic with +musl), and the check compares the BOLT tsc with the plain build. The others have no BOLT: linux-arm64 +is not tried yet, BOLT's Mach-O support is experimental, and BOLT has no PE (Windows) support. To make the Linux x64 tsc by hand on zbook, use `RELEASE_FEATURES=noembed RELEASE_LIBC=musl RELEASE_PIE=0 crates/ts_goport/scripts/build-release.sh /linux-x64` and pack and test it: