diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d64c698bb..014510f06 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,11 +14,15 @@ # The tsc is a PGO build of the shipped profile (fat LTO), trained on each platform: # crates/ts_goport/scripts/build-pgo.sh with the training set of scripts/goport/pgo-train.sh (7 # projects at fixed commits). The job also builds the plain tsc, and fails unless the PGO tsc gives -# the same stdout, stderr, exit code and emitted files as the plain tsc on that set. No BOLT -# (build-release.sh has it): BOLT refuses the static Linux bin (build-bolt.sh), its Mach-O support -# is experimental, macOS has no perf branch sampling for its profile, and BOLT has no PE support. +# the same stdout, stderr, exit code and emitted files as the plain tsc on that set. linux-x64 also +# gets BOLT: crates/ts_goport/scripts/bolt-musl.sh, with a profile from a counting copy that runs +# the same set, and no -split-eh (it can break the unwinding of a caught panic with musl). The PGO +# check then compares the BOLT tsc with the plain tsc. The others have no BOLT: linux-arm64 is not +# tried yet, BOLT's Mach-O support is experimental, and BOLT has no PE support. # Linux is static musl and non-PIE, so it starts on any x86-64 or arm64 Linux, Alpine too # (npm/README.md, "Linux libc"). Each Linux tsc builds natively on a Blacksmith runner of its arch. +# The linux-x64 PGO tsc links lust's memfast.c in place of musl's memcpy, memmove, memcmp, bcmp and +# memchr (crates/ts_goport/scripts/memfast.sh): musl's memcmp and bcmp are byte loops. # Windows (x86_64-pc-windows-msvc, aarch64-pc-windows-msvc) builds in Git Bash, with the C runtime # linked in (+crt-static), so tsc.exe needs no Visual C++ redistributable. jemalloc does not build # there: it uses the system allocator. Each PGO tsc builds natively, so its training runs on its @@ -42,6 +46,9 @@ on: - "scripts/goport/pgo-train.sh" - "crates/ts_goport/scripts/copy-libs.sh" - "crates/ts_goport/scripts/build-pgo.sh" + - "crates/ts_goport/scripts/memfast.*" + - "crates/ts_goport/scripts/bolt-musl.sh" + - "crates/ts_goport/scripts/check-unwind.py" - "LICENSE" - "NOTICE.md" - "licenses/**" @@ -134,6 +141,15 @@ jobs: - run: rustup toolchain install "$RUSTUP_TOOLCHAIN" --profile minimal --target ${{ matrix.target }} --component llvm-tools - if: runner.os == 'Linux' run: sudo apt-get update -q && sudo apt-get install -yq musl-tools + # linux-x64: llvm-bolt, merge-fdata, llvm-objcopy and llvm-dwarfdump of LLVM 23 for bolt-musl.sh. + - name: BOLT tools + if: matrix.platform == 'linux-x64' + run: | + curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null + codename=$(. /etc/os-release && echo "$VERSION_CODENAME") + echo "deb https://apt.llvm.org/$codename/ llvm-toolchain-$codename-23 main" | sudo tee /etc/apt/sources.list.d/llvm-23.list + sudo apt-get update -q && sudo apt-get install -yq bolt-23 llvm-23 + /usr/lib/llvm-23/bin/llvm-bolt --version | grep -m1 'LLVM version' # darwin-x64 on an arm64 Mac: the check runs the x86-64 tsc, so Rosetta 2 must be there. - name: Rosetta if: matrix.rosetta @@ -176,6 +192,13 @@ jobs: # jemalloc stops at start when the kernel page is bigger than its build page. arm64 kernels # use 4, 16 or 64 KiB pages; 2^16 covers all three. [[ $target != aarch64-* ]] || echo "JEMALLOC_SYS_WITH_LG_PAGE=16" >> "$GITHUB_ENV" + # linux-x64: lust's memfast.c in place of musl's memcpy, memmove, memcmp, bcmp and memchr + # (memfast.sh), in the PGO build only. The plain tsc keeps musl's, so the PGO check below + # also checks memfast against them. + if [[ $target == x86_64-unknown-linux-musl ]]; then + flags=$(crates/ts_goport/scripts/memfast.sh "$RUNNER_TEMP/memfast-wrap.o") + echo "PGO_RUSTFLAGS=$flags" | tee -a "$GITHUB_ENV" + fi # The plain tsc: the reference of the PGO check below, or the shipped tsc with no-pgo. PGO_LINK is # the link mode of both builds (build-pgo.sh): static on Linux, crt-static on Windows. - name: Build @@ -201,10 +224,19 @@ jobs: PGO_BINS: tsgo PGO_CARGO: cargo CARGO_FEATURES: --features noembed + # The BOLT input needs the static relocations. + PGO_EMIT_RELOCS: ${{ matrix.platform == 'linux-x64' && '1' || '' }} run: | PGO_TRAIN="scripts/goport/pgo-train.sh run $RUNNER_TEMP/train" crates/ts_goport/scripts/build-pgo.sh "$RUNNER_TEMP/pgo" mkdir -p "$RUNNER_TEMP/out" cp "$RUNNER_TEMP/pgo/target-use/goport/tsgo${{ matrix.exe }}" "$RUNNER_TEMP/out/tsc${{ matrix.exe }}" + # linux-x64: BOLT of the PGO tsc. bolt-musl.sh also checks that the BOLT tsc still unwinds a + # caught panic (check-unwind.py). + - name: BOLT + if: matrix.platform == 'linux-x64' + run: | + crates/ts_goport/scripts/bolt-musl.sh "$RUNNER_TEMP/out/tsc" "$RUNNER_TEMP/train" "$RUNNER_TEMP/bolt" + cp "$RUNNER_TEMP/bolt/bin/tsc" "$RUNNER_TEMP/out/tsc" - name: Check the PGO tsc against the plain tsc if: ${{ !matrix.no-pgo }} run: scripts/goport/pgo-train.sh compare "$RUNNER_TEMP/train" "$RUNNER_TEMP/plain/tsc${{ matrix.exe }}" "$RUNNER_TEMP/out/tsc${{ matrix.exe }}" @@ -233,6 +265,12 @@ jobs: [[ $rc == 2 ]] && grep -q "error TS2322" check.txt if [[ $RUNNER_OS == Linux ]]; then file "$dir/tsc" | tee /dev/stderr | grep -q "statically linked" + # memfast (linux-x64): the five __wrap_ functions, and none of musl's. + if [[ ${PGO_RUSTFLAGS:-} == *--wrap=* ]]; then + syms=$(nm "$dir/tsc" | grep -E ' [Tt] (__wrap_)?(memcpy|memmove|memcmp|bcmp|memchr)$' | awk '{print $3}' | sort | xargs) + echo "mem functions: $syms" + [[ $syms == "__wrap_bcmp __wrap_memchr __wrap_memcmp __wrap_memcpy __wrap_memmove" ]] + fi # Static: it starts on musl (Alpine) too. docker run --rm -v "$dir:/t:ro" alpine:3 /t/tsc --version fi diff --git a/NOTICE.md b/NOTICE.md index e8b97ee8d..26af75582 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -51,3 +51,14 @@ The collation and normalization tables in `crates/ts_goport/src/gostd/data` and packages above. License: [licenses/Unicode-LICENSE.txt](licenses/Unicode-LICENSE.txt). + +## memfast + +Copyright (c) 2026 Theo Browne. Licensed under the MIT License. + +`crates/ts_goport/scripts/memfast.c` is `src/runtime/memfast.c` of +[t3dotgg/lust-lang](https://github.com/t3dotgg/lust-lang) (commit 6ced2b86fd), without change +below its header. The linux-x64 tsc links it in place of musl's memcpy, memmove, memcmp, bcmp and +memchr. + +License: [licenses/memfast-LICENSE.txt](licenses/memfast-LICENSE.txt). diff --git a/crates/ts_goport/scripts/bolt-musl.sh b/crates/ts_goport/scripts/bolt-musl.sh new file mode 100755 index 000000000..423292392 --- /dev/null +++ b/crates/ts_goport/scripts/bolt-musl.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# BOLT of the static musl linux-x64 tsc (release.yml). BOLT puts the hot blocks of each function +# together, moves the cold blocks out and orders the functions. The program does not change. +# The profile comes from a counting copy (llvm-bolt -instrument) that runs the training set of +# scripts/goport/pgo-train.sh: the release runner has no perf branch sampling. +# +# Usage: bolt-musl.sh +# tsc the PGO tsc, linked with --emit-relocs (build-pgo.sh PGO_EMIT_RELOCS=1) +# train-dir a pgo-train.sh setup dir +# out-dir new dir; the BOLT tsc is /bin/tsc (with the lib files of a noembed tsc), +# the logs and counts stay in +# LLVM_BIN dir of llvm-bolt, merge-fdata, llvm-objcopy and llvm-dwarfdump +# (default /usr/lib/llvm-23/bin) +# +# Static bins differ from build-release.sh's dynamic ones in three ways (relmusl1): +# - A static bin runs no BOLT code at exit, so the counting copy writes its counts from a +# watcher process every second (-instrumentation-sleep-time=1), each time all of them +# (-instrumentation-no-counters-clear). Some watchers spin after their last write and block +# every signal but SIGKILL: the script kills them. +# - No -split-eh. It moves landing pads to the cold part and gives their LSDA a start address +# (LPStart). The musl target unwinds with LLVM libunwind, which logs "lsda at ... does not +# start with 0xFF" for such an LSDA. In some cases a caught panic through it then aborts +# (relmusl1's test program: "failed to initiate panic"); in others it works. +# - The relocation mode needs the static relocations (--emit-relocs). +# Checks of the BOLT tsc: no BOLT runtime code (no __bolt symbols), one RW PT_GNU_STACK, and +# check-unwind.py (two caught panics in the tsc with no libunwind line, and every LSDA starts with +# 0xFF: a cautious guard that can reject a tsc that would work). release.yml then compares its +# output with the plain tsc on the training set. +set -euo pipefail + +case "${1:-}" in + help | -h | --help) + sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed -e '$d' -e 's/^# \{0,1\}//' + exit 0 + ;; +esac +(($# == 3)) || { echo "usage: bolt-musl.sh " >&2; exit 2; } +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo="$(cd -- "$script_dir/../../.." && pwd)" +tsc=$1 +train="$(cd -- "$2" && pwd)" +out=$3 +llvm="${LLVM_BIN:-/usr/lib/llvm-23/bin}" +[[ ! -e $out ]] || { echo "error: $out exists" >&2; exit 2; } +mkdir -p "$out/counts" "$out/bin" +out="$(cd -- "$out" && pwd)" +# grep -c, not -q: with pipefail, a grep that stops early can fail the pipe (SIGPIPE). +[[ $(readelf -SW "$tsc" | grep -c ' \.rela\.text ') != 0 ]] || + { echo "error: $tsc has no .rela.text (link it with --emit-relocs: build-pgo.sh PGO_EMIT_RELOCS=1)" >&2; exit 1; } + +# BOLT reads a symbol with ".cold" or ".warm" in it as a split part of another function and stops +# ("parent function not found"). Rust paths make such names ("Session..warm_auto_import_cache"). +# Rename them in a copy: "..warm" to "..Warm" (same length, only .symtab changes). +nm "$tsc" | awk '$2 ~ /^[tT]$/ && $3 ~ /\.\.(cold|warm)/ {r = $3; gsub(/\.\.cold/, "..Cold", r); gsub(/\.\.warm/, "..Warm", r); print $3, r}' > "$out/rename.txt" +"$llvm/llvm-objcopy" --redefine-syms="$out/rename.txt" "$tsc" "$out/input" + +# The counting copy. pgo-train.sh runs its copy at /bin/tsc. +"$llvm/llvm-bolt" "$out/input" -o "$out/instr" -instrument -instrumentation-file="$out/counts/tsc" \ + -instrumentation-file-append-pid -instrumentation-binpath="$train/bin/tsc" \ + -instrumentation-sleep-time=1 -instrumentation-no-counters-clear > "$out/instr.log" 2>&1 || + { tail -20 "$out/instr.log" >&2; exit 1; } +# One process per run: tsgo on 4 KiB pages would start a worker (bin/tsgo.rs `launch`). +GOPORT_LAUNCH=0 "$repo/scripts/goport/pgo-train.sh" run "$train" "$out/instr" > "$out/train.log" 2>&1 || + { tail -20 "$out/train.log" >&2; exit 1; } +# The watchers write up to about 1 s after their tsc ends. +sleep 5 +left=$(pgrep -fc "^$train/bin/tsc" || true) +pkill -9 -f "^$train/bin/tsc" || true +echo "bolt training: $(find "$out/counts" -name 'tsc.*.fdata' | wc -l) count files, $left watchers killed" +"$llvm/merge-fdata" "$out"/counts/tsc.*.fdata > "$out/tsc.fdata" 2> "$out/merge.log" || + { tail -5 "$out/merge.log" >&2; exit 1; } + +"$llvm/llvm-bolt" "$out/input" -o "$out/bin/tsc" -data="$out/tsc.fdata" \ + -reorder-blocks=ext-tsp -reorder-functions=cdsort -split-functions -split-all-cold > "$out/bolt.log" 2>&1 || + { tail -20 "$out/bolt.log" >&2; exit 1; } +echo "bolt: $(grep -m1 -o '[0-9]* out of [0-9]* functions in the binary ([0-9.]*%) have non-empty execution profile' "$out/bolt.log" || true)" + +if [[ $(nm "$out/bin/tsc" | grep -c '__bolt') != 0 ]]; then + echo "error: $out/bin/tsc has BOLT runtime symbols (the counting copy?)" >&2 + exit 1 +fi +stack="$(readelf -lW "$out/bin/tsc" | awk '$1 == "GNU_STACK" { s = ""; for (i = 7; i < NF; i++) s = s $i; print s }')" +[[ $stack == RW ]] || { echo "error: $out/bin/tsc has PT_GNU_STACK '$stack', not one RW" >&2; exit 1; } +"$script_dir/copy-libs.sh" "$out/bin" > /dev/null +python3 "$script_dir/check-unwind.py" "$out/bin/tsc" "$llvm/llvm-dwarfdump" +echo "BOLT tsc: $out/bin/tsc" diff --git a/crates/ts_goport/scripts/build-pgo.sh b/crates/ts_goport/scripts/build-pgo.sh index 7e83314a3..5499c91ec 100755 --- a/crates/ts_goport/scripts/build-pgo.sh +++ b/crates/ts_goport/scripts/build-pgo.sh @@ -27,8 +27,9 @@ # the layout when BOLT is not used. # --emit-relocs keeps the static relocations in the file (non-alloc # sections, not mapped at run time). BOLT needs them for relocation -# mode, where it also reorders functions. Not with PGO_LINK=static: -# build-bolt.sh refuses static bins, so they only make the file bigger. +# mode, where it also reorders functions. Not with PGO_LINK=static +# unless PGO_EMIT_RELOCS=1 (bolt-musl.sh, release.yml's linux-x64 +# tsc): else they only make the file bigger. # The build prints the .text and .rela.text sections of tsgo # (readelf -S). Expect .text.hot, .text.unlikely and .rela.text. GNU ld (the # linker of the musl target) ignores keep-text-section-prefix, but its default @@ -53,6 +54,10 @@ # x86_64-unknown-linux-musl. Default: the host, with # --target only when PGO_LINK is set. # PGO_BINS the bins to build, for example "tsgo" (default: all) +# PGO_RUSTFLAGS extra rustc flags of both builds. release.yml sets the +# memfast link flags of the linux-x64 tsc (memfast.sh). +# PGO_EMIT_RELOCS=1 link the use build with --emit-relocs also when +# PGO_LINK=static (the input of bolt-musl.sh) # PGO_TRAIN training command, in place of the default training # (which needs the project inputs and the corpus of # GOPORT_DATA_ROOT). The script runs `$PGO_TRAIN `, @@ -189,7 +194,7 @@ triple="${PGO_TARGET:-}" # Each step needs its own target (other RUSTFLAGS). sccache is off: it could # reuse an object built with an older profile at the same path. build() { # build - local target="$out/$1" flags="$2 $link_flags" target_args="" + local target="$out/$1" flags="$2 $link_flags${PGO_RUSTFLAGS:+ $PGO_RUSTFLAGS}" target_args="" # Both builds get the same link mode, so the profile of the generate build # matches the code of the use build. [[ -n $triple ]] && target_args="--target $triple" @@ -301,7 +306,7 @@ echo "merged $(du -h "$merged" | cut -f1) -> $merged" # The Linux link args are the r3-link layout (see the header). use_flags="-Cprofile-use=$merged" if [[ $(uname -s) == Linux ]]; then - [[ $link == static ]] || use_flags+=" -Clink-arg=-Wl,--emit-relocs" + [[ $link == static && ${PGO_EMIT_RELOCS:-0} != 1 ]] || use_flags+=" -Clink-arg=-Wl,--emit-relocs" use_flags+=" -Clink-arg=-Wl,-z,keep-text-section-prefix" fi build target-use "$use_flags" diff --git a/crates/ts_goport/scripts/check-unwind.py b/crates/ts_goport/scripts/check-unwind.py new file mode 100755 index 000000000..b2863fc2b --- /dev/null +++ b/crates/ts_goport/scripts/check-unwind.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Checks that a tsc still unwinds a caught panic. release.yml runs it on the BOLT copy of the +static musl linux-x64 tsc (bolt-musl.sh). The port catches panics as Go recovers them, so a tsc +that aborts on a panic gives a crash where Go gives an answer. + +Usage: check-unwind.py [] + +1. Two caught panics in the tsc itself, each in a new `tsc --api --async`. The API answers the + panic as an error response (Go's request recover; the port's go_recover in + ipc/conn_async.rs). The same process must then answer transpileModule, exit 0 when its stdin + closes, and write no "libunwind:" line to stderr. A tsc that cannot unwind aborts at the panic. + a. transpileDeclaration of a .d.ts file. Go's transpile.go asserts that the emit wrote a file + ("Output generation failed"). + b. With --callbacks readFile:error, createSnapshot of a project. The read of its tsconfig + panics (Go's callbackfs.go panicIfError) inside the config load. In the BOLT tsc of + Release run 39967282137, this unwind crosses 3 tsoptions frames that BOLT moved, each with + a landing pad. The unwind of a. crosses no moved frame there. +2. With : every LSDA (the exception table of a function, named by its FDE in + .eh_frame) starts with 0xFF (DW_EH_PE_omit: no LPStart). llvm-bolt -split-eh writes LSDAs + with an LPStart (2,180 in a relmusl1 test tsc). The musl target unwinds with LLVM libunwind, + which logs "lsda at ... does not start with 0xFF" for such an LSDA. That line is only a log. + In relmusl1's small test program, a caught panic through such LSDAs aborted ("failed to + initiate panic"), but the -split-eh test tsc still caught the panic of 1b. So this check is a + cautious guard: it can reject a tsc that would work, but it does not pass a tsc that has such + an LSDA. It checks every function, not only the paths of 1. +""" +import json +import re +import signal +import struct +import subprocess +import sys +import tempfile + + +def fail(message): + sys.exit(f"check-unwind: {message}") + + +# api_panic: one run of `tsc --api --async ` in . must give the caught panic +# ; then transpileModule must answer, the tsc must exit 0, and stderr has no libunwind line. +def api_panic(tsc, cwd, args, method, params, panic): + signal.alarm(120) + with tempfile.TemporaryFile() as stderr: + proc = subprocess.Popen([tsc, "--api", "--async", *args, "--cwd", cwd], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=stderr) + + def stderr_text(): + stderr.seek(0) + return stderr.read().decode(errors="replace") + + def ended(name): + fail(f"{method}: {name}: the tsc ended (exit {proc.wait()}): {stderr_text()[-400:]!r}") + + # request: one JSON-RPC call (Content-Length framing), and its response. + def request(id, name, params): + body = json.dumps({"jsonrpc": "2.0", "id": id, "method": name, "params": params}).encode() + proc.stdin.write(b"Content-Length: %d\r\n\r\n" % len(body) + body) + proc.stdin.flush() + while True: + length = None + while (line := proc.stdout.readline()) != b"\r\n" or length is None: + if not line: + ended(name) + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":")[1]) + body = proc.stdout.read(length) + if len(body) < length: + ended(name) + message = json.loads(body) + if message.get("id") == id and "method" not in message: + return message + + request(1, "initialize", None) + answer = request(2, method, params) + error = answer.get("error", {}).get("message", "") + if not error.startswith("panic: ") or panic not in error: + fail(f"{method}: no caught panic: {json.dumps(answer)[:400]}") + answer = request(3, "transpileModule", {"input": "const a = 1;", "options": {}}) + if "const a = 1;" not in answer.get("result", {}).get("outputText", ""): + fail(f"{method}: transpileModule after the panic: {json.dumps(answer)[:400]}") + proc.stdin.close() + code = proc.wait() + if code != 0: + fail(f"{method}: the tsc exited {code} after the panic: {stderr_text()[-400:]!r}") + logs = [line for line in stderr_text().splitlines() if "libunwind:" in line] + if logs: + fail(f"{method}: {len(logs)} libunwind lines on stderr, for example {logs[0]!r}") + signal.alarm(0) + print(f"{method}: caught panic ({error.splitlines()[0]}); the next request answered, exit 0") + + +def caught_panics(tsc): + with tempfile.TemporaryDirectory() as cwd: + api_panic(tsc, cwd, [], "transpileDeclaration", + {"input": "declare const x: number;\n", "options": {"fileName": "a.d.ts"}}, + "Output generation failed") + with open(f"{cwd}/tsconfig.json", "w") as f: + f.write('{"compilerOptions": {"strict": true, "noEmit": true}, "files": ["a.ts"]}\n') + with open(f"{cwd}/a.ts", "w") as f: + f.write("export const x: number = 1;\n") + api_panic(tsc, cwd, ["--callbacks", "readFile:error"], "createSnapshot", + {"openProjects": [f"{cwd}/tsconfig.json"]}, + "filesystem operation configured with serverFS.error: readFile") + + +def lsda_starts(tsc, dwarfdump): + data = open(tsc, "rb").read() + phoff, = struct.unpack_from("= 8) { + mf_u64 a = MF_LD8(s), b = MF_LD8(s + n - 8); + MF_ST8(d, a); + MF_ST8(d + n - 8, b); + } else if (n >= 4) { + mf_u32 a = MF_LD4(s), b = MF_LD4(s + n - 4); + MF_ST4(d, a); + MF_ST4(d + n - 4, b); + } else if (n) { + mf_u8 a = s[0], b = s[n >> 1], c = s[n - 1]; + d[0] = a; + d[n >> 1] = b; + d[n - 1] = c; + } + return; + } +#ifdef MF_V16 + if (n <= 32) { + mf_v16 a = MF_LDV(s), b = MF_LDV(s + n - 16); + MF_STV(d, a); + MF_STV(d + n - 16, b); + return; + } + mf_v16 a = MF_LDV(s), b = MF_LDV(s + 16), c = MF_LDV(s + n - 32), e = MF_LDV(s + n - 16); + MF_STV(d, a); + MF_STV(d + 16, b); + MF_STV(d + n - 32, c); + MF_STV(d + n - 16, e); +#else + mf_u64 w[8]; + mf_size k = 0; + for (; k + 8 <= n; k += 8) w[k >> 3] = MF_LD8(s + k); + mf_u64 t = MF_LD8(s + n - 8); + for (mf_size j = 0; j < k; j += 8) MF_ST8(d + j, w[j >> 3]); + MF_ST8(d + n - 8, t); +#endif +} + +/* More than 64 bytes, front to back: right for disjoint ranges and for d below s. The + last 64 bytes are read first, as the loop may overwrite them when d is below s. */ +MF_FN static inline __attribute__((always_inline)) void mf_copy_fwd(mf_u8 *d, const mf_u8 *s, mf_size n) { +#ifdef MF_V16 + mf_v16 t0 = MF_LDV(s + n - 64), t1 = MF_LDV(s + n - 48), t2 = MF_LDV(s + n - 32), t3 = MF_LDV(s + n - 16); + for (mf_size i = 0; n - i > 64; i += 64) { + mf_v16 a = MF_LDV(s + i), b = MF_LDV(s + i + 16), c = MF_LDV(s + i + 32), e = MF_LDV(s + i + 48); + MF_STV(d + i, a); + MF_STV(d + i + 16, b); + MF_STV(d + i + 32, c); + MF_STV(d + i + 48, e); + } + MF_STV(d + n - 64, t0); + MF_STV(d + n - 48, t1); + MF_STV(d + n - 32, t2); + MF_STV(d + n - 16, t3); +#else + mf_u64 t = MF_LD8(s + n - 8); + for (mf_size i = 0; n - i > 8; i += 8) MF_ST8(d + i, MF_LD8(s + i)); + MF_ST8(d + n - 8, t); +#endif +} + +/* More than 64 bytes, back to front, for d above s: the first 64 bytes are read first. */ +MF_FN static inline __attribute__((always_inline)) void mf_copy_bwd(mf_u8 *d, const mf_u8 *s, mf_size n) { +#ifdef MF_V16 + mf_v16 h0 = MF_LDV(s), h1 = MF_LDV(s + 16), h2 = MF_LDV(s + 32), h3 = MF_LDV(s + 48); + for (mf_size i = n; i > 64; i -= 64) { + mf_v16 a = MF_LDV(s + i - 64), b = MF_LDV(s + i - 48), c = MF_LDV(s + i - 32), e = MF_LDV(s + i - 16); + MF_STV(d + i - 64, a); + MF_STV(d + i - 48, b); + MF_STV(d + i - 32, c); + MF_STV(d + i - 16, e); + } + MF_STV(d, h0); + MF_STV(d + 16, h1); + MF_STV(d + 32, h2); + MF_STV(d + 48, h3); +#else + mf_u64 h = MF_LD8(s); + for (mf_size i = n; i > 8; i -= 8) MF_ST8(d + i - 8, MF_LD8(s + i - 8)); + MF_ST8(d, h); +#endif +} + +MF_FN void *memcpy(void *d, const void *s, mf_size n) { + if (n <= 64) mf_copy_small((mf_u8 *)d, (const mf_u8 *)s, n); + else mf_copy_fwd((mf_u8 *)d, (const mf_u8 *)s, n); + return d; +} + +MF_FN void *memmove(void *d, const void *s, mf_size n) { + if (n <= 64) mf_copy_small((mf_u8 *)d, (const mf_u8 *)s, n); + else if ((mf_uptr)d - (mf_uptr)s >= (mf_uptr)n) mf_copy_fwd((mf_u8 *)d, (const mf_u8 *)s, n); + else mf_copy_bwd((mf_u8 *)d, (const mf_u8 *)s, n); + return d; +} + +/* The first index below n where a and b differ, from index i on, or n. */ +MF_FN static inline __attribute__((always_inline)) mf_size mf_diff(const mf_u8 *a, const mf_u8 *b, mf_size i, mf_size n) { +#ifdef MF_V16 + for (; n - i >= 16; i += 16) { + unsigned m = MF_MASK(MF_LDV(a + i) == MF_LDV(b + i)) ^ 0xFFFFu; + if (m) return i + (mf_size)__builtin_ctz(m); + } + if (i < n && n >= 16) { + /* The last 16 bytes; the ones below i are equal. */ + mf_size t = n - 16; + unsigned m = MF_MASK(MF_LDV(a + t) == MF_LDV(b + t)) ^ 0xFFFFu; + return m ? t + (mf_size)__builtin_ctz(m) : n; + } +#endif +#ifdef MF_LE + for (; n - i >= 8; i += 8) { + mf_u64 x = MF_LD8(a + i) ^ MF_LD8(b + i); + if (x) return i + (mf_size)(__builtin_ctzll(x) >> 3); + } +#endif + for (; i < n; i++) + if (a[i] != b[i]) return i; + return n; +} + +MF_FN int memcmp(const void *a, const void *b, mf_size n) { + const mf_u8 *x = (const mf_u8 *)a, *y = (const mf_u8 *)b; + mf_size i = mf_diff(x, y, 0, n); + return i < n ? (int)x[i] - (int)y[i] : 0; +} + +/* bcmp(a, b, n): 0 when equal, else not 0 (the C compiler emits it for memcmp() == 0). */ +MF_FN int bcmp(const void *a, const void *b, mf_size n) { + return mf_diff((const mf_u8 *)a, (const mf_u8 *)b, 0, n) < n; +} + +MF_FN void *memchr(const void *s, int c, mf_size n) { + const mf_u8 *p = (const mf_u8 *)s; + mf_u8 b = (mf_u8)c; + mf_size i = 0; +#ifdef MF_V16 + if (n >= 16) { + mf_v16 v = (mf_v16){0} + (char)b; + for (; n - i >= 64; i += 64) { + mf_v16 e0 = MF_LDV(p + i) == v, e1 = MF_LDV(p + i + 16) == v, e2 = MF_LDV(p + i + 32) == v, e3 = MF_LDV(p + i + 48) == v; + if (MF_MASK(e0 | e1 | e2 | e3)) { + mf_u64 m = (mf_u64)MF_MASK(e0) | (mf_u64)MF_MASK(e1) << 16 | (mf_u64)MF_MASK(e2) << 32 | (mf_u64)MF_MASK(e3) << 48; + return (void *)(p + i + (mf_size)__builtin_ctzll(m)); + } + } + for (; n - i >= 16; i += 16) { + unsigned m = MF_MASK(MF_LDV(p + i) == v); + if (m) return (void *)(p + i + (mf_size)__builtin_ctz(m)); + } + if (i < n) { + /* The last 16 bytes, without the ones below i (already checked). */ + mf_size t = n - 16; + unsigned m = MF_MASK(MF_LDV(p + t) == v) & (0xFFFFu << (unsigned)(i - t)); + if (m) return (void *)(p + t + (mf_size)__builtin_ctz(m)); + } + return 0; + } +#endif + for (; i < n; i++) + if (p[i] == b) return (void *)(p + i); + return 0; +} diff --git a/crates/ts_goport/scripts/memfast.sh b/crates/ts_goport/scripts/memfast.sh new file mode 100755 index 000000000..51818976d --- /dev/null +++ b/crates/ts_goport/scripts/memfast.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Compiles memfast.c (lust's memcpy, memmove, memcmp, bcmp and memchr; MIT, see its header) for +# the static musl linux-x64 tsc, checks the object, and prints the rustc flags that link it in +# place of musl's functions. release.yml passes them to build-pgo.sh (PGO_RUSTFLAGS). +# +# Usage: memfast.sh +# CC the C compiler (default musl-gcc) +# +# Why: musl's memcmp and bcmp are byte loops and its memcpy and memmove use rep movs. They took +# 4.4% of the cycles of a VS Code check and 9.2% on TypeORM; with memfast 2.0% and 3.3% (relmusl1, +# cycles:u --singleThreaded). Results are libc's: the tsc output does not change. +# The functions get the names __wrap_, and the link wraps the five symbols +# (-Wl,--wrap=). Plain names would clash with musl's in libc.a, because rustc puts +# -C link-arg after it. --wrap sends every call to these, musl's own calls too. memset stays +# musl's (asm). +# Checks: no undefined symbol (a loop that the compiler turns back into a memcpy call would call +# itself through the wrap), and no AVX or BMI code (the tsc runs on any x86-64). +set -euo pipefail + +case "${1:-}" in + "" | help | -h | --help) + sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed -e '$d' -e 's/^# \{0,1\}//' + exit 2 + ;; +esac +out=$1 +src="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/memfast.c" +names=(memcpy memmove memcmp bcmp memchr) +defs=() wraps="" +for n in "${names[@]}"; do + defs+=("-D$n=__wrap_$n") + wraps+="${wraps:+,}--wrap=$n" +done +# lust builds the unit with -fno-strict-aliasing (see the comment in memfast.c). +"${CC:-musl-gcc}" -O2 -fno-strict-aliasing "${defs[@]}" -c "$src" -o "$out" +undefined="$(nm -u "$out")" +if [[ -n $undefined ]]; then + echo "error: $out has undefined symbols: $undefined" >&2 + exit 1 +fi +# grep -c, not -q: with pipefail, a grep that stops early can fail the pipe (SIGPIPE). +if [[ $(objdump -d --no-show-raw-insn "$out" | + grep -cE $'\t(v[a-z0-9]+|andn|bextr|blsi|blsmsk|blsr|bzhi|pdep|pext|mulx|rorx|sarx|shlx|shrx) ') != 0 ]]; then + echo "error: $out has AVX or BMI code" >&2 + exit 1 +fi +for n in "${names[@]}"; do + [[ $(nm "$out" | grep -cE " T __wrap_$n\$") == 1 ]] || { echo "error: $out has no __wrap_$n" >&2; exit 1; } +done +echo "-Clink-arg=-Wl,$wraps -Clink-arg=$out" diff --git a/licenses/memfast-LICENSE.txt b/licenses/memfast-LICENSE.txt new file mode 100644 index 000000000..eccbd9f1c --- /dev/null +++ b/licenses/memfast-LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Theo Browne + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/npm/README.md b/npm/README.md index 1825e3f10..dc65b2b3b 100644 --- a/npm/README.md +++ b/npm/README.md @@ -121,11 +121,14 @@ package. The CI builds (but darwin-x64 and win32-arm64) are PGO builds of the shipped profile (fat LTO): `build-pgo.sh` trains each platform's tsc on the set of `scripts/goport/pgo-train.sh`, and the job -fails unless the PGO tsc gives the same output as a plain build on that set. They have no BOLT: BOLT -refuses the static Linux bin, macOS has no perf branch sampling for its profile, and BOLT has no PE -(Windows) support. To make the Linux x64 tsc by hand on zbook, use `RELEASE_FEATURES=noembed -RELEASE_LIBC=musl RELEASE_PIE=0 crates/ts_goport/scripts/build-release.sh /linux-x64` and pack -and test it: +fails unless the PGO tsc gives the same output as a plain build on that set. The Linux x64 PGO tsc +links `crates/ts_goport/scripts/memfast.c` in place of musl's memcpy, memmove, memcmp, bcmp and +memchr (`memfast.sh`); the plain build keeps musl's. It also gets BOLT (`bolt-musl.sh`: a counting +copy trains on the same set; no `-split-eh`, which can break the unwinding of a caught panic with +musl), and the check compares the BOLT tsc with the plain build. The others have no BOLT: linux-arm64 +is not tried yet, BOLT's Mach-O support is experimental, and BOLT has no PE (Windows) support. To make +the Linux x64 tsc by hand on zbook, use `RELEASE_FEATURES=noembed RELEASE_LIBC=musl RELEASE_PIE=0 +crates/ts_goport/scripts/build-release.sh /linux-x64` and pack and test it: ```sh GOPORT_PIN= scripts/goport/npm-pack.sh --name tsc-rs --package-version \ diff --git a/npm/pack.mjs b/npm/pack.mjs index 93af24ead..e8b4b38c3 100644 --- a/npm/pack.mjs +++ b/npm/pack.mjs @@ -179,6 +179,7 @@ function portNotice() { ...part("TypeScript third-party notices", tsNoticeFile), ...part("Go license", path.join(repoRoot, "licenses", "Go-LICENSE.txt")), ...part("Unicode license", path.join(repoRoot, "licenses", "Unicode-LICENSE.txt")), + ...part("memfast license", path.join(repoRoot, "licenses", "memfast-LICENSE.txt")), ].join("\n"); } function writeLicense(dir) {