From 56faf7cff0b49dfd1b914e0703d6f4bc7ee48e63 Mon Sep 17 00:00:00 2001
From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com>
Date: Mon, 16 Nov 2026 16:35:41 -0500
Subject: [PATCH] fix(scripts): fail checks that checked nothing and record the
hook timings
A check that ran over no input exited 0, which reads as a clean run.
- check-cant-fail-scan.sh : an empty diff while HEAD is not the
base exits 2, as check-affected.sh does. A diff with no test file keeps
exit 0 and now counts the files it looked at.
- affected-tests.sh --shard: a leg that draws none of a non-empty
selection exits 2 (more legs than suites). No workflow calls it with
--shard; CI legs use run-plugin-tests.sh --suites-from.
- lefthook.yml pre-push: a merge-base copy of the hook that cannot be
extracted, or a failed mktemp, blocks the push instead of passing. No
origin/main merge-base still skips with its notice.
The check-placement contract gets the commit-set budget (10 s, judgment
from the measured set) and a per-check timing table for both stages.
Change-scoped tests stay in CI: they add 91 s on a one-file script edit
and 146 s on a 28-file change, over the 60 s push budget. The
plugin-imports boundary check already runs at pre-push.
Co-Authored-By: Claude Opus 5.5
---
.claude/hooks/check-affected-pre-push.test.sh | 20 +++++++++
docs/contracts/check-placement/README.md | 41 ++++++++++++++++++-
lefthook.yml | 18 +++++---
scripts/README.md | 7 +++-
scripts/affected-tests.sh | 13 +++---
scripts/affected-tests.test.sh | 19 +++++----
scripts/check-cant-fail-scan.sh | 20 +++++++--
scripts/check-cant-fail-scan.test.sh | 18 +++++++-
8 files changed, 132 insertions(+), 24 deletions(-)
diff --git a/.claude/hooks/check-affected-pre-push.test.sh b/.claude/hooks/check-affected-pre-push.test.sh
index f653de31244..1b0e6f095f7 100755
--- a/.claude/hooks/check-affected-pre-push.test.sh
+++ b/.claude/hooks/check-affected-pre-push.test.sh
@@ -408,6 +408,26 @@ if command -v lefthook >/dev/null 2>&1; then
fi
run_claude "$lh"
expect "after lefthook install the Claude Code hook stays quiet" 0 0
+
+ # A merge-base that holds no check-affected hook leaves the job nothing to
+ # run, so the push fails instead of passing unchecked.
+ nohook="$tmp/lh-nohook"
+ mkdir -p "$nohook"
+ git_init_test_repo "$nohook" || exit 2
+ cp "$repo_root/lefthook.yml" "$nohook/"
+ git_test_config "$nohook" add -A
+ git_test_config "$nohook" commit -q -m base
+ git_test_config "$nohook" update-ref refs/remotes/origin/main HEAD
+ git init -q --bare "$tmp/remote-nohook.git"
+ git_test_config "$nohook" remote add origin "$tmp/remote-nohook.git"
+ lh_install="$(cd "$nohook" && lefthook install 2>&1)" || bad "lefthook install in the no-hook fixture" "$lh_install"
+ RC=0
+ ERR="$(cd "$nohook" && git_test_config "$nohook" push -q origin HEAD:refs/heads/x 2>&1)" || RC=$?
+ if [[ "$RC" != 0 && "$ERR" == *"nothing was checked"* ]]; then
+ ok "lefthook blocks a push whose merge-base has no check-affected hook to run"
+ else
+ bad "lefthook blocks a push whose merge-base has no check-affected hook to run" "exit $RC; output: $ERR"
+ fi
else
ok "SKIP lefthook end-to-end cases (lefthook not installed)"
fi
diff --git a/docs/contracts/check-placement/README.md b/docs/contracts/check-placement/README.md
index 31858473f09..b248f3c9832 100644
--- a/docs/contracts/check-placement/README.md
+++ b/docs/contracts/check-placement/README.md
@@ -379,7 +379,7 @@ marketplace comes from these rows.
| Budget | Value | Basis |
|---|---|---|
| Per-batch formatter (`PostToolBatch`) | 1 s median per fire | judgment from the measured spread; revisit after the rollout's per-batch measurements |
-| Commit (`pre-commit` set) | set from one staged-file run of every shipped `pre-commit` entry together on the same 44-file set, measured when the rollout of the remaining check plugins lands | until then the per-entry rows below stand |
+| Commit (`pre-commit` set) | 10 s for all 11 shipped `pre-commit` entries run in a row | judgment from the measured set: 0.91 s on 1 file, 1.91 s on 10, 9.6 s on a 45-file mixed set and 96.3 s on a 44-file set (per-check timings below); 10 s keeps the mixed set inside and `reference-verify`'s 12.2 s on 44 files outside, as its move to `pre-push` decided |
| Push (`pre-push`) | 60 s | judgment from the measured spread; beyond it, CI only |
### Measured rows
@@ -405,3 +405,42 @@ The per-edit rows are the `PostToolUse` rows each plugin's `hooks/hooks.json` ra
timed with a sample that needs work (hyperfine, 2 warmups, 10 runs; a 15 ms spawn baseline is
included). They are the baseline the git-hook entries and `PostToolBatch` rows replaced; each new
entry adds its own row here when it ships.
+
+### Per-check timings
+
+Method: hyperfine 1.21.0, 1 warmup, 10 runs for `pre-commit` and 3 for `pre-push`, median with
+min-max. A `pre-commit` sample is a real `origin/main` commit staged on its parent, re-staged before
+every run, with each entry script run directly from a copy of `origin/main`. A `pre-push` sample is
+`scripts/check-affected.sh --no-fetch --base ` at a real commit, or at a one-line edit to
+`scripts/check-cost-figures.sh` on `origin/main`. Host: Linux, WSL2, 24 CPUs; shellcheck 0.11.0,
+shfmt 3.14.1, typos 1.52.2, markdownlint-cli2 0.24.0, actionlint 1.7.12. Samples: 1 file
+(`55c8ce6fea`, one `.sh`), 10 files (`dedc4fd990`), 44 files (`83fb17c179`, 24 `.sh`, 22 of them
+copies of `hook-utils.sh`), 45 files (`5fdcd660b4`, 24 `.mjs`, 12 workflows); 28 files
+(`4e73c874b5`).
+
+| Check | Stage | Median (min-max) | Sample | Host | Date |
+|---|---|---|---|---|---|
+| all 11 entries in a row | pre-commit | 0.91 s; 1.91 s; 96.3 s (93.1-102.8); 9.6 s (9.5-9.7) | 1; 10; 44; 45 files | Linux, WSL2, 24 CPUs | 2026-11-16 |
+| bash-format `shell-check` | pre-commit | 0.32 s; 0.66 s; 98.9 s (93.8-108.7); 6.3 s | 1; 10; 44; 45 files | Linux, WSL2, 24 CPUs | 2026-11-16 |
+| code-tidying `comment-lint` | pre-commit | 0.37 s; 0.45 s; 0.17 s; 1.72 s | 1; 10; 44; 45 files | Linux, WSL2, 24 CPUs | 2026-11-16 |
+| spelling `typos-check` | pre-commit | 0.06 s; 0.13 s; 0.86 s; 0.36 s | 1; 10; 44; 45 files | Linux, WSL2, 24 CPUs | 2026-11-16 |
+| markdown-format `markdown-check` | pre-commit | 0.01 s; 0.40 s; 0.35 s; 0.55 s | 1; 10; 44; 45 files | Linux, WSL2, 24 CPUs | 2026-11-16 |
+| guardrails `pre-commit-content` | pre-commit | 0.06 s; 0.14 s; 0.39 s; 0.37 s | 1; 10; 44; 45 files | Linux, WSL2, 24 CPUs | 2026-11-16 |
+| actionlint `actionlint-check` | pre-commit | 0.01 s; 0.01 s; 0.01 s; 0.10 s | 1; 10; 44; 45 files | Linux, WSL2, 24 CPUs | 2026-11-16 |
+| instruction-placement `index-check` | pre-commit | 0.03 s on every sample | 1; 10; 44; 45 files | Linux, WSL2, 24 CPUs | 2026-11-16 |
+| javascript-format, python-format, go-format, powershell-format entries | pre-commit | 0.01-0.02 s each on every sample (no file of their types, or a few `.mjs`) | 1; 10; 44; 45 files | Linux, WSL2, 24 CPUs | 2026-11-16 |
+| `scripts/check-affected.sh` lint tier, which includes the `plugin-imports` boundary check | pre-push | 4.0 s (4.0-4.1); 18.6 s (18.2-18.6) | 1 file; 28 files | Linux, WSL2, 24 CPUs | 2026-11-16 |
+| `scripts/check-plugin-imports.sh` alone | pre-push | 1.24 s (1.21-1.30), 10 runs | whole tree | Linux, WSL2, 24 CPUs | 2026-11-16 |
+| guardrails `pre-push-content` | pre-push | 0.17 s (0.15-0.19), 5 runs | 28 files, one unpushed commit | Linux, WSL2, 24 CPUs | 2026-11-16 |
+| guardrails `reference-verify` | pre-push | 1.25 s (1.18-1.27), 5 runs | 28 files, one unpushed commit | Linux, WSL2, 24 CPUs | 2026-11-16 |
+| `scripts/check-affected.sh --tests` | CI only | 4.5 s, 0 suites; 94.7 s (89.5-96.5), 10 suites; 164.3 s (159.4-165.8), 24 suites | 1 file (`151a7b0c0c`, data); 1 file (script edit); 28 files | Linux, WSL2, 24 CPUs | 2026-11-16 |
+
+The `pre-push` set on the 28-file sample totals 20.0 s: 18.6 s, 0.17 s and 1.25 s. Change-scoped
+tests stay in CI: `--tests` adds 1.0 s on a change that selects no suite, 90.7 s on a one-file
+script edit and 145.7 s on the 28-file change, so a typical push goes over the 60 s budget. The
+boundary check already runs at `pre-push`, inside the lint tier.
+
+The shipped `pre-commit` set took 96.3 s on the 44-file sample, a real `origin/main` commit, over
+the commit budget and the push budget. bash-format's `shell-check` alone measured 98.9 s:
+shellcheck takes 1.7 s on one copy of the 5,174-line `hook-utils.sh`, and the commit changes 22 of
+them.
diff --git a/lefthook.yml b/lefthook.yml
index 6d71c967485..b61977d5518 100644
--- a/lefthook.yml
+++ b/lefthook.yml
@@ -13,8 +13,10 @@ pre-push:
# controls this job: read a branch's diff to lefthook.yml, lefthook-local.yml, .lefthook/ and
# .lefthook-local/ before committing or pushing on it, since the guardrails hooks below fire at
# commit too.
- # This job reads no stdin, so it judges HEAD rather than the pushed refs. Exit 1 is a failing
- # check and blocks the push; anything else passes, and CI checks the pushed commit either way.
+ # This job reads no stdin, so it judges HEAD rather than the pushed refs. A failing check
+ # exits 1 and blocks the push. So does a merge-base copy of the hook that cannot be extracted:
+ # the job then checked nothing, and passing would read as a checked push. With no origin/main
+ # merge-base the job says it skipped and passes; CI checks the pushed commit either way.
# Keep double quotes out of `run`: on Windows lefthook wraps it in double quotes without
# escaping them, and sh gets a cut-off script. mktemp paths hold no spaces, so $tmp needs none.
- name: check-affected
@@ -23,10 +25,16 @@ pre-push:
echo 'check-affected pre-push: skipped, no origin/main merge-base to take the check from; CI checks it.' >&2
exit 0
fi
- tmp=$(mktemp -d) || exit 0
+ if ! tmp=$(mktemp -d); then
+ echo 'check-affected pre-push: failed, nothing was checked: mktemp could not make a directory.' >&2
+ exit 1
+ fi
trap 'rm -rf $tmp' EXIT
- git archive $mb -- .claude/hooks/check-affected-pre-push.sh lib/hook-utils.sh 2>/dev/null | tar -x -C $tmp 2>/dev/null
- [ -f $tmp/.claude/hooks/check-affected-pre-push.sh ] || exit 0
+ git archive $mb -- .claude/hooks/check-affected-pre-push.sh lib/hook-utils.sh | tar -x -C $tmp
+ if [ ! -f $tmp/.claude/hooks/check-affected-pre-push.sh ]; then
+ echo 'check-affected pre-push: failed, nothing was checked: merge-base' $mb 'has no .claude/hooks/check-affected-pre-push.sh with lib/hook-utils.sh; rebase onto origin/main.' >&2
+ exit 1
+ fi
bash $tmp/.claude/hooks/check-affected-pre-push.sh --git-hook ` exits `0` with a `Not applicable`
+ line counting the changed files when none of them is a test file: its corpus
+ is the changed test files, and a change that adds no test cannot add one that
+ cannot fail. An empty diff while HEAD is not the base exits `2`, since it
+ checked nothing, as `scripts/check-affected.sh` does.
The contract governs the observable interface, not the option set: `set -euo`
and `set -uo` both appear in the family and neither is required, which is why
diff --git a/scripts/affected-tests.sh b/scripts/affected-tests.sh
index 55a8faed715..fafddcc464c 100755
--- a/scripts/affected-tests.sh
+++ b/scripts/affected-tests.sh
@@ -59,8 +59,9 @@
# the sorted suite list partitioned by index modulo n. The union of legs
# 0..n-1 is exactly the unsharded selection and no two legs share a suite.
# Modulo rather than contiguous blocks, because the sorted list clusters suites
-# by directory. An EMPTY leg exits 0: "this leg had nothing to run" and
-# "nothing was affected" are the same statement about that runner.
+# by directory. A leg that draws none of a non-empty selection exits 2, a usage
+# error (more legs than selected suites): it checked nothing, so exit 0 would
+# read as a tested leg. An empty selection still exits 0 before the partition.
#
# FAIL LOUD, NOT OPEN. A changed file that maps to NO suite is an ERROR, not an
# empty selection: "zero suites" reads as "nothing to run" when it actually
@@ -1680,11 +1681,11 @@ if [[ "$shard_total" -gt 1 ]]; then
leg+=("${selected[$si]}")
done
echo "shard: leg $shard_index of $shard_total keeps ${#leg[@]} of ${#selected[@]} selected suite(s)." >&2
- selected=("${leg[@]}")
- if [[ ${#selected[@]} -eq 0 ]]; then
- echo "This leg has no suites to run; the other legs carry the selection." >&2
- exit 0
+ if [[ ${#leg[@]} -eq 0 ]]; then
+ echo "error: leg $shard_index of $shard_total draws none of the ${#selected[@]} selected suite(s); use at most ${#selected[@]} leg(s)." >&2
+ exit 2
fi
+ selected=("${leg[@]}")
fi
if [[ "$explain" -eq 1 ]]; then
diff --git a/scripts/affected-tests.test.sh b/scripts/affected-tests.test.sh
index 11a5d05410f..20dc7cee2cf 100755
--- a/scripts/affected-tests.test.sh
+++ b/scripts/affected-tests.test.sh
@@ -729,15 +729,20 @@ else
fail "--shard 0/1 changed the selection"
fi
-# The empty leg. It is the short-circuit CI depends on: a leg that draws
-# nothing must EXIT 0, because a leg that skipped would make the matrix result
-# `skipped` and ci-status is fail-closed on that.
-out="$(select_shard 999/1000)"
+# The empty leg. A leg that draws none of a non-empty selection checked
+# nothing, so it is a usage error (more legs than selected suites), never a
+# pass that reads as a tested leg.
+out="$(cd "$repo" && bash scripts/affected-tests.sh --shard 999/1000 -- "${shard_paths[@]}" 2>&1 >/dev/null)"
RC=$?
-if [[ "$RC" -eq 0 ]] && [[ -z "$(printf '%s' "$out" | grep . || true)" ]]; then
- ok "a leg with no suites exits 0 and prints no selection"
+if [[ "$RC" -eq 2 ]] && [[ "$out" == *"leg 999 of 1000 draws none of the"* ]]; then
+ ok "a leg that draws no suite exits 2 and says why"
else
- fail "empty leg should exit 0 with nothing selected (rc=$RC): $out"
+ fail "empty leg should exit 2 naming the leg (rc=$RC): $out"
+fi
+if [[ -z "$(select_shard 999/1000 | grep . || true)" ]]; then
+ ok "an empty leg prints no selection"
+else
+ fail "an empty leg printed a selection"
fi
# A spec this script cannot read must never become leg 0 of 1, which runs
diff --git a/scripts/check-cant-fail-scan.sh b/scripts/check-cant-fail-scan.sh
index c0bdf89eae1..c0eaabd58bc 100755
--- a/scripts/check-cant-fail-scan.sh
+++ b/scripts/check-cant-fail-scan.sh
@@ -13,7 +13,11 @@
# whole-tree walk prunes some directories that `--file` does not, so a file under
# one of them is skipped here too; PRUNED mirrors collect_files() in the scanner,
# and this script's suite fails when the two differ. A diff with no test file in
-# it reports the gate not applicable.
+# it exits 0 and reports the gate not applicable with the count of files it
+# looked at: a change that adds no test cannot add a test that cannot fail. An
+# empty diff while HEAD is not the base exits 2, since it checked nothing, the
+# rule scripts/check-affected.sh follows; HEAD at the base has no change and
+# exits 0 (scripts/README.md, "The check-script contract").
set -uo pipefail
SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" || exit 2
@@ -41,10 +45,20 @@ if [[ -z "${1:-}" ]]; then
fi
top="$(git rev-parse --show-toplevel)" || exit 2
-changed="$(git -C "$top" diff --name-only --diff-filter=ACMR "$1...HEAD")" || {
+all="$(git -C "$top" diff --name-only "$1...HEAD")" || {
printf 'ERROR: cannot diff against %s\n' "$1" >&2
exit 2
}
+if [[ -z "$all" ]]; then
+ if [[ "$(git -C "$top" rev-list --count "$1...HEAD")" == 0 ]]; then
+ printf '::notice::Not applicable: HEAD is %s, so there is no change to scan.\n' "$1"
+ exit 0
+ fi
+ printf 'ERROR: no file changed since %s, yet HEAD is not %s, so there was nothing to check; pass no base to scan the whole tree.\n' "$1" "$1" >&2
+ exit 2
+fi
+total="$(grep -c '' <<<"$all")"
+changed="$(git -C "$top" diff --name-only --diff-filter=ACMR "$1...HEAD")" || exit 2
scanned=0
worst=0
@@ -65,7 +79,7 @@ while IFS= read -r path; do
done <<<"$changed"
if [[ $scanned -eq 0 ]]; then
- printf '::notice::Not applicable: no test file changed since %s; check-tests scans the whole tree when there is no diff base.\n' "$1"
+ printf '::notice::Not applicable: %d file(s) changed since %s, none a test file the scan covers; check-tests scans the whole tree when there is no diff base.\n' "$total" "$1"
exit 0
fi
[[ $worst -eq 0 ]] && printf 'PASS: %d changed test file(s), no gating findings.\n' "$scanned"
diff --git a/scripts/check-cant-fail-scan.test.sh b/scripts/check-cant-fail-scan.test.sh
index 2c34d088bd0..4c0be129660 100755
--- a/scripts/check-cant-fail-scan.test.sh
+++ b/scripts/check-cant-fail-scan.test.sh
@@ -87,7 +87,23 @@ new_repo
put "$r" README.md 'changed
'
gate 0 "a diff with no test file passes" "$r" base
-has "it reports the gate not applicable" "$OUT" "Not applicable: no test file changed since base"
+has "it reports the gate not applicable, counting the files it looked at" "$OUT" \
+ "Not applicable: 1 file(s) changed since base, none a test file the scan covers"
+
+# HEAD at the base has no change to scan.
+new_repo
+gate 0 "HEAD at the base passes" "$r" base
+has "it says HEAD is the base" "$OUT" "Not applicable: HEAD is base"
+
+# Commits whose net diff is empty checked nothing, so they are never a pass.
+new_repo
+put "$r" README.md 'changed
+'
+put "$r" README.md 'base
+'
+gate 2 "an empty diff with commits on top is an environment error, not a pass" "$r" base
+has "it says nothing changed" "$ERR" "no file changed since base, yet HEAD is not base"
+if [[ -z "$OUT" ]]; then pass "an empty diff prints no clean statement"; else bad "an empty diff prints no clean statement" "$OUT"; fi
# A clean changed test file passes and is named.
new_repo