diff --git a/.changes/bash-format/perf-shell-check-dedupe-identical-blobs-66766b51.md b/.changes/bash-format/perf-shell-check-dedupe-identical-blobs-66766b51.md new file mode 100644 index 00000000000..891de9e4f56 --- /dev/null +++ b/.changes/bash-format/perf-shell-check-dedupe-identical-blobs-66766b51.md @@ -0,0 +1,7 @@ +--- +bump: patch +--- + +### Changed + +- The pre-commit check runs ShellCheck once per distinct staged script instead of once per path, and reports each finding under every path that holds the same bytes, so each copy's own added lines still decide what blocks. Copies are linted together only when nothing else ShellCheck reads can differ between them: the same extension and the same nearest `.shellcheckrc`, and, for copies in different directories, a probe that finds no `source` statement (a `SCRIPTDIR` search path resolves those from each script's own directory). A commit that changes 21 identical copies of a 5,174-line script lints that script once. diff --git a/plugins/bash-format/git-hooks/shell-check.sh b/plugins/bash-format/git-hooks/shell-check.sh index 43bc2797dce..6d1615415a9 100755 --- a/plugins/bash-format/git-hooks/shell-check.sh +++ b/plugins/bash-format/git-hooks/shell-check.sh @@ -116,6 +116,90 @@ checks_changed_lines out="$CHECKS_TMP/shellcheck" : >"$out" rc=0 + +# Identical scripts lint identically when ShellCheck reads nothing else that +# depends on the path: the extension (its dialect fallback), the .shellcheckrc +# nearest the script's directory, and, for a script that sources a file, the +# directory a SCRIPTDIR search path resolves it from. So each distinct (bytes, +# extension, nearest rc) is linted once, from its first path, into $lint; a +# group spread over directories stays whole only when a probe finds nothing it +# sources, else it splits by directory. $copies maps a linted path to each copy, +# which gets its findings under its own path, so its own added lines still decide +# what blocks. Any failure here lints every path. The path-dependent inputs are +# the ones https://github.com/koalaman/shellcheck/blob/master/shellcheck.1.md +# lists under RC FILES, source-path and --shell. +lint=(${files[@]+"${files[@]}"}) +copies="$CHECKS_TMP/copies" +: >"$copies" +# sc_rcdir : the directory of the .shellcheckrc or shellcheckrc nearest the +# path, up to the top level, in $rcdir; empty when none (the lookup above the top +# level is the same for every path). +sc_rcdir() { + local d="$1" + rcdir= + while [[ "$d" == */* ]]; do + d="${d%/*}" + [[ -e "$d/.shellcheckrc" || -e "$d/shellcheckrc" ]] && rcdir="$d" && return + done + [[ -e .shellcheckrc || -e shellcheckrc ]] && rcdir=. + return 0 +} +# sc_sources : 0 unless ShellCheck, without any rc, SHELLCHECK_OPTS or the +# script's own directives, reports no source statement (SC1090, SC1091) and no +# error, such as a parse error that hides the rest of the script. +sc_sources() { + local probe + probe="$(sed 's/#[[:space:]]*shellcheck/#_/g' -- "$1" | + (unset SHELLCHECK_OPTS && shellcheck --norc -s bash -f gcc -S info -) 2>&1)" + (($? <= 1)) || return 0 + grep -q -e '\[SC109[01]\]' -e ': error: ' <<<"$probe" +} +dedupe_files() { + local table="$CHECKS_TMP/blobs" i=0 f h ext dir key rep path + plain=() + for f in ${files[@]+"${files[@]}"}; do + [[ "$f" == *[$'\t\n']* ]] || plain+=("$f") + done + ((${#plain[@]} > 1)) || return 0 + printf '%s\n' "${plain[@]}" | git hash-object --no-filters --stdin-paths >"$CHECKS_TMP/hashes" || return 1 + : >"$table" + while IFS= read -r h; do + f="${plain[i]}" + ext="${f##*.}" + sc_rcdir "$f" + dir=. + [[ "$f" == */* ]] && dir="${f%/*}" + printf '%s %s %s\t%s\t%s\n' "$h" "$ext" "$rcdir" "$dir" "$f" >>"$table" + i=$((i + 1)) + done <"$CHECKS_TMP/hashes" + ((i == ${#plain[@]})) || return 1 + # A group whose paths sit in more than one directory: its first path, probed. + awk -F '\t' '{ if (!($1 in first)) first[$1] = $3; else if (dir[$1] != $2) spread[$1] = 1; dir[$1] = $2 } + END { for (k in spread) print k "\t" first[k] }' "$table" >"$CHECKS_TMP/spread" || return 1 + : >"$CHECKS_TMP/split" + while IFS=$'\t' read -r key rep; do + sc_sources "$rep" && printf '%s\n' "$key" >>"$CHECKS_TMP/split" + done <"$CHECKS_TMP/spread" + awk -F '\t' -v probed="$CHECKS_TMP/split" ' + BEGIN { while ((got = (getline l < probed)) > 0) apart[l] = 1; if (got < 0) exit 2 } + { k = ($1 in apart) ? $1 "\t" $2 : $1; if (!(k in rep)) rep[k] = $3; print rep[k] "\t" $3 }' "$table" >"$CHECKS_TMP/reps" || return 1 + lint=() + while IFS=$'\t' read -r rep path; do + if [[ "$rep" == "$path" ]]; then + lint+=("$path") + else + printf '%s\t%s\n' "$rep" "$path" >>"$copies" + fi + done <"$CHECKS_TMP/reps" + for f in ${files[@]+"${files[@]}"}; do + [[ "$f" == *[$'\t\n']* ]] && lint+=("$f") + done + return 0 +} +if ! dedupe_files; then + lint=(${files[@]+"${files[@]}"}) + : >"$copies" +fi # Windows caps a whole command line at 32,767 characters, and ShellCheck 0.11.0 # takes its file list only as arguments, so the paths go in batches whose # summed length, plus a separator each, stays under this budget; the rest of @@ -130,7 +214,7 @@ shellcheck_batch() { } batch=() size=0 -for f in ${files[@]+"${files[@]}"}; do +for f in ${lint[@]+"${lint[@]}"}; do if ((${#batch[@]} && size + ${#f} + 1 > sc_budget)); then shellcheck_batch "${batch[@]}" batch=() @@ -140,6 +224,25 @@ for f in ${files[@]+"${files[@]}"}; do size=$((size + ${#f} + 1)) done ((${#batch[@]})) && shellcheck_batch "${batch[@]}" +# Each finding of a linted path is repeated for every copy of it, right after. +if [[ -s "$copies" ]]; then + awk -v copies="$copies" ' + BEGIN { + while ((got = (getline l < copies)) > 0) { t = index(l, "\t"); r = substr(l, 1, t - 1); c[r] = (r in c) ? c[r] "\t" substr(l, t + 1) : substr(l, t + 1) } + if (got < 0) exit 2 + } + { print } + match($0, /:[0-9]+:[0-9]+: /) && (substr($0, 1, RSTART - 1) in c) { + n = split(c[substr($0, 1, RSTART - 1)], m, "\t") + for (i = 1; i <= n; i++) print m[i] substr($0, RSTART) + }' "$out" >"$out.copies" + copied=$? + if ((copied == 0)); then + mv "$out.copies" "$out" + else + checks_tool_failed awk "$copied" 'copying the shellcheck findings to identical scripts' + fi +fi # A script gone from the work tree or partly staged is read from the index on # stdin, named "-". for f in ${indexed[@]+"${indexed[@]}"}; do diff --git a/plugins/bash-format/git-hooks/shell-check.test.sh b/plugins/bash-format/git-hooks/shell-check.test.sh index 3abbbd5b36e..90d5c74fae1 100755 --- a/plugins/bash-format/git-hooks/shell-check.test.sh +++ b/plugins/bash-format/git-hooks/shell-check.test.sh @@ -8,6 +8,7 @@ # --worktree reports without rewriting, a merge leaves a script it takes # unchanged from the merged branch alone, a missing tool follows the skip-key rule, # many staged paths reach ShellCheck in batches under Windows' command-line cap, +# identical scripts are linted once unless their rc or sourced files could differ, # and the plugin's own vendored installer wires the entry so `git commit` runs it. # Expected values are the tools' documented output: a tab-indented body formats to # the two spaces indent_size = 2 asks for; SC2154 is " is referenced but not @@ -74,7 +75,9 @@ HINT='bash-format: fix the findings above, or adjust the repository .shellcheckr # paths must reach shellcheck in more than one call. These cases put a stub in # its place and need no real one: past the cap the stub fails to launch # with status 126, as Windows does; otherwise it logs "--" per call and each path -# it gets, prints a finding for a *-finding.sh path and exits 2 for *-broken.sh. +# it gets, prints a finding for a *-finding.sh path or one holding "stub: finding", +# and exits 2 for *-broken.sh. Identical scripts are linted once, so each fixture +# here holds its own bytes. mkdir -p "$WORK/sc-stub" cat >"$WORK/sc-stub/shellcheck" <<'STUB' #!/usr/bin/env bash @@ -91,11 +94,13 @@ rc=0 for f in "$@"; do printf '%s\n' "$f" >>"$SC_STUB_LOG" case "$f" in - *-finding.sh) - printf '%s:2:1: warning: stub finding [SC2154]\n' "$f" - ((rc < 1)) && rc=1 - ;; *-broken.sh) rc=2 ;; + *) + if [[ "$f" == *-finding.sh ]] || grep -q 'stub: finding' "$f"; then + printf '%s:2:1: warning: stub finding [SC2154]\n' "$f" + ((rc < 1)) && rc=1 + fi + ;; esac done exit "$rc" @@ -113,7 +118,7 @@ LONG="$(printf 'd%.0s' {1..60})/$(printf 'f%.0s' {1..80})" new_repo NONE mkdir -p "$REPO/${LONG%/*}" for i in {1..250}; do - printf '#!/bin/bash\necho ok\n' >"$REPO/$LONG-$i.sh" + printf '#!/bin/bash\necho ok %s\n' "$i" >"$REPO/$LONG-$i.sh" done git -C "$REPO" add -A expect "many long staged paths: the hook passes" "0|" "$(sc_stub "$REPO")" @@ -121,20 +126,38 @@ expect "many long staged paths: shellcheck ran in more than one call" yes \ "$( (($(grep -cx -- -- "$WORK/sc.log") > 1)) && echo yes)" expect "many long staged paths: each path reached shellcheck exactly once" \ "$(git -C "$REPO" diff --cached --name-only | sort)" "$(grep -vx -- -- "$WORK/sc.log" | sort)" -printf '#!/bin/bash\necho ok\n' >"$REPO/a-finding.sh" -printf '#!/bin/bash\necho ok\n' >"$REPO/z-finding.sh" +printf '#!/bin/bash\necho a\n' >"$REPO/a-finding.sh" +printf '#!/bin/bash\necho z\n' >"$REPO/z-finding.sh" git -C "$REPO" add a-finding.sh z-finding.sh expect "a finding from the first batch and one from the last both block" "1|a-finding.sh:2: warning: stub finding [SC2154] z-finding.sh:2: warning: stub finding [SC2154] $HINT" "$(sc_stub "$REPO")" git -C "$REPO" rm -q --cached a-finding.sh z-finding.sh rm "$REPO/a-finding.sh" "$REPO/z-finding.sh" -printf '#!/bin/bash\necho ok\n' >"$REPO/a-broken.sh" +printf '#!/bin/bash\necho broken\n' >"$REPO/a-broken.sh" git -C "$REPO" add a-broken.sh out="$(sc_stub "$REPO")" expect "a first batch's status above 1 still blocks after a later batch passes" \ "1|bash-format: shellcheck failed:" "${out%%$'\n'*}" +# Identical staged scripts in one directory are linted once; each copy gets the +# findings under its own path, and its own added lines decide what blocks. b.sh +# already held line 2, so its copy of the finding there does not block. +new_repo NONE +printf '#!/bin/bash\n# stub: finding\necho before\n' >"$REPO/b-copy.sh" +git -C "$REPO" add b-copy.sh +git -C "$REPO" commit -q -m b --no-verify +for f in a-copy.sh b-copy.sh c-copy.sh; do + printf '#!/bin/bash\n# stub: finding\necho same\n' >"$REPO/$f" +done +printf '#!/bin/bash\necho other\n' >"$REPO/d-other.sh" +git -C "$REPO" add a-copy.sh b-copy.sh c-copy.sh d-other.sh +expect "identical copies: every copy whose added lines hold the finding reports it" "1|a-copy.sh:2: warning: stub finding [SC2154] +c-copy.sh:2: warning: stub finding [SC2154] +$HINT" "$(sc_stub "$REPO")" +expect "identical copies are linted once, differing content still each" "a-copy.sh +d-other.sh" "$(grep -vx -- -- "$WORK/sc.log" | sort)" + for tool in shfmt shellcheck jq node; do if ! command -v "$tool" >/dev/null 2>&1; then echo "SKIP: $tool not on PATH -- the remaining shell-check tests skipped" @@ -288,6 +311,63 @@ git -C "$REPO" add a.sh expect "no .editorconfig: ShellCheck still reports" "1||a.sh:2: warning: undefined_var is referenced but not assigned. [SC2154] $HINT" "$(check "$REPO")" +# Identical bytes in different directories: ShellCheck reads the .shellcheckrc +# nearest each script, and a SCRIPTDIR search path resolves a sourced file from +# each script's own directory, so such copies are linted apart unless neither +# can change the result. A shim in front of the real shellcheck logs each call. +mkdir -p "$WORK/sc-count" +printf '#!/usr/bin/env bash\nprintf "%%s\\n" "$*" >>"$SC_COUNT_LOG"\nexec %q "$@"\n' "$(command -v shellcheck)" >"$WORK/sc-count/shellcheck" +chmod +x "$WORK/sc-count/shellcheck" +sc_count() { # -> "|"; the calls go to $WORK/count.log + : >"$WORK/count.log" + local out + out="$(cd "$1" && SC_COUNT_LOG="$WORK/count.log" PATH="$WORK/sc-count:$PATH" bash "$CHECK" 2>&1)" + printf '%s|%s' "$?" "$out" +} +linted() { # : how many lint calls named it + grep -c -- "-x .* $1\( \|\$\)" "$WORK/count.log" +} +UNDEF="$SHEBANG"$'\necho "$undefined_var"' + +new_repo NONE +mkdir -p "$REPO/x" "$REPO/y" +printf '%s\n' "$UNDEF" >"$REPO/x/a.sh" +printf '%s\n' "$UNDEF" >"$REPO/y/a.sh" +git -C "$REPO" add x y +expect "copies in two directories that source nothing: each copy reports the finding" "1|x/a.sh:2: warning: undefined_var is referenced but not assigned. [SC2154] +y/a.sh:2: warning: undefined_var is referenced but not assigned. [SC2154] +$HINT" "$(sc_count "$REPO")" +expect "copies in two directories that source nothing: linted once" "1|0" "$(linted x/a.sh)|$(linted y/a.sh)" + +new_repo NONE +mkdir -p "$REPO/pkg" +printf 'disable=SC2154\n' >"$REPO/pkg/.shellcheckrc" +printf '%s\n' "$UNDEF" >"$REPO/a.sh" +printf '%s\n' "$UNDEF" >"$REPO/pkg/a.sh" +git -C "$REPO" add a.sh pkg +expect "a copy under its own .shellcheckrc is linted apart: its rc disables the finding" "1|a.sh:2: warning: undefined_var is referenced but not assigned. [SC2154] +$HINT" "$(sc_count "$REPO")" +expect "a copy under its own .shellcheckrc: both copies linted" "1|1" "$(linted a.sh)|$(linted pkg/a.sh)" + +new_repo NONE +printf 'source-path=SCRIPTDIR\nexternal-sources=true\n' >"$REPO/.shellcheckrc" +mkdir -p "$REPO/x" "$REPO/y" +printf '%s\nsource helper.sh\necho "$v"\n' "$SHEBANG" >"$REPO/x/a.sh" +printf '%s\nsource helper.sh\necho "$v"\n' "$SHEBANG" >"$REPO/y/a.sh" +printf '# shellcheck shell=bash\nv=1\nexport v\n' >"$REPO/x/helper.sh" +printf '# shellcheck shell=bash\n:\n' >"$REPO/y/helper.sh" +git -C "$REPO" add .shellcheckrc x/a.sh y/a.sh +expect "copies that source a file from their own directory are linted apart" "1|y/a.sh:3: warning: v is referenced but not assigned. [SC2154] +$HINT" "$(sc_count "$REPO")" + +new_repo NONE +mkdir -p "$REPO/x" "$REPO/y" +printf '%s\n# shellcheck disable=SC1091\nif true; then\necho (\nsource helper.sh\n' "$SHEBANG" >"$REPO/x/a.sh" +cp "$REPO/x/a.sh" "$REPO/y/a.sh" +git -C "$REPO" add x y +sc_count "$REPO" >/dev/null +expect "copies the probe cannot read (a parse error, a disabled SC1091) are linted apart" "1|1" "$(linted x/a.sh)|$(linted y/a.sh)" + new_repo $'root = true\n\n[*.sh]\nindent_style = space\nindent_size = 2' mkdir -p "$REPO/pkg/sub" printf 'root = true\n\n[*.md]\nindent_size = 4\n' >"$REPO/pkg/.editorconfig"