From aed1e14e1ac09344f3f8dbfe8bc9b7da31450481 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 17:25:25 +0000 Subject: [PATCH 1/2] Benchmarks: a changed benchmark runs only the benchmarks of its modules ci/bench_arches.py treated every change under bench/ as shared, running every benchmark on every architecture. A pull request that only gates an algorithm's benchmark on another architecture (as adding one does, since check_arch_gates.py requires it) then ran the whole suite in each of the nine configurations, past the 15-minute timeout on the slower runners. A changed bench/benches/primitives/.rs now narrows to the modules its USES lists, whose benchmarks include it; main.rs, and a benchmark whose USES cannot be read (e.g. a deleted one), still run everything. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WkLN6tAYk76HACiEWLbAMD --- ci/bench_arches.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/ci/bench_arches.py b/ci/bench_arches.py index 2cd598bb4..073881981 100644 --- a/ci/bench_arches.py +++ b/ci/bench_arches.py @@ -13,13 +13,16 @@ whose own files changed: `src/asm//.rs`, or the Rust API's `src/.rs` or `src/hashes/.rs` (for every architecture), or `src//.rs`, whose module is `_` (e.g. -`src/hmac/sha256.rs` is `hmac_sha256`, as in `src/asm/`). The +`src/hmac/sha256.rs` is `hmac_sha256`, as in `src/asm/`), or the +modules a changed benchmark `bench/benches/primitives/.rs` lists in +its `USES` (on every architecture), whose benchmarks include it. The benchmarks decide which of them run (each lists the modules it `USES`, see bench/benches/primitives/main.rs), and run everything for a module none of them uses (e.g. `cpu`, `lib`, or `hashes/mod.rs`'s `mod`). Any other change -it benchmarks (e.g. the benchmarks themselves) runs every benchmark, and -`modules` is empty. The generated `src/asm//mod.rs` only declares the -modules, so it narrows nothing either way. +it benchmarks (e.g. the benchmarks' `main.rs`, or a benchmark whose `USES` +it cannot read) runs every benchmark, and `modules` is empty. The generated +`src/asm//mod.rs` only declares the modules, so it narrows nothing +either way. An architecture with primitives that choose among implementations by CPU feature is benchmarked once with every feature the runner has, and once @@ -68,10 +71,25 @@ ASM = re.compile(r"src/asm/([a-z0-9_]+)/([a-z0-9_]+)\.rs$") API = re.compile(r"src/(?:hashes/)?([a-z0-9_]+)\.rs$") FAMILY = re.compile(r"src/(?!asm/|hashes/)([a-z0-9_]+)/([a-z0-9_]+)\.rs$") +# One algorithm's benchmark, and the modules it lists in its `USES`. +BENCH = re.compile(r"bench/benches/primitives/(?!main\.rs$)[a-z0-9_]+\.rs$") +USES = re.compile(r"pub const USES: &\[&str\] = &\[([^\]]*)\];") ALL = None +def bench_uses(path): + """The modules the benchmark at `path` lists in its `USES`, or None if + it cannot be read (e.g. a deleted benchmark) or lists none.""" + try: + with open(path) as f: + m = USES.search(f.read()) + except OSError: + return None + modules = re.findall(r'"([a-z0-9_]+)"', m[1]) if m else [] + return modules or None + + def arches(changed): # The modules to benchmark on each architecture that needs it, or ALL. needed = {} @@ -94,6 +112,10 @@ def need(arch, module): name = family[1] if family[2] == "mod" else f"{family[1]}_{family[2]}" for a in PLATFORMS: need(a, name) + elif BENCH.match(path) and (uses := bench_uses(path)): + for a in PLATFORMS: + for m in uses: + need(a, m) elif SHARED.match(path): for a in PLATFORMS: needed[a] = ALL From de56f7ba4fe6dd811992ad3edcae337bfb41a74f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 17:43:41 +0000 Subject: [PATCH 2/2] Benchmarks: allow 30 minutes for a run of every benchmark Every benchmark, which a shared change (or a run by hand) still runs, now takes about 14 minutes on the aarch64 runners and more than the 15-minute timeout on the x86-64, x86 and ARMv7 ones, which were cancelled on this pull request's own run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WkLN6tAYk76HACiEWLbAMD --- .github/workflows/bench.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 925649754..85e909f5d 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -68,7 +68,9 @@ jobs: include: ${{ fromJSON(needs.arches.outputs.matrix) }} name: Compare with base (${{ matrix.arch }}${{ matrix.cpu-features && format(', VG_CPU_FEATURES={0}', matrix.cpu-features) || '' }}) runs-on: ${{ matrix.os }} - timeout-minutes: 15 + # Running every benchmark (for a shared change, or by hand) takes about + # 14 minutes on the aarch64 runners and longer on the others. + timeout-minutes: 30 # With an empty image the job runs directly on the runner. container: image: ${{ matrix.image || '' }}