Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
bump: patch
---

### Changed

- The pre-commit check runs ShellCheck once per distinct staged script instead of once per path, and reports each finding under every path that holds the same bytes, so each copy's own added lines still decide what blocks. Copies are linted together only when nothing else ShellCheck reads can differ between them: the same extension and the same nearest `.shellcheckrc`, and, for copies in different directories, a probe that finds no `source` statement (a `SCRIPTDIR` search path resolves those from each script's own directory). A commit that changes 21 identical copies of a 5,174-line script lints that script once.
105 changes: 104 additions & 1 deletion plugins/bash-format/git-hooks/shell-check.sh
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,90 @@ checks_changed_lines
out="$CHECKS_TMP/shellcheck"
: >"$out"
rc=0

# Identical scripts lint identically when ShellCheck reads nothing else that
# depends on the path: the extension (its dialect fallback), the .shellcheckrc
# nearest the script's directory, and, for a script that sources a file, the
# directory a SCRIPTDIR search path resolves it from. So each distinct (bytes,
# extension, nearest rc) is linted once, from its first path, into $lint; a
# group spread over directories stays whole only when a probe finds nothing it
# sources, else it splits by directory. $copies maps a linted path to each copy,
# which gets its findings under its own path, so its own added lines still decide
# what blocks. Any failure here lints every path. The path-dependent inputs are
# the ones https://github.com/koalaman/shellcheck/blob/master/shellcheck.1.md
# lists under RC FILES, source-path and --shell.
lint=(${files[@]+"${files[@]}"})
copies="$CHECKS_TMP/copies"
: >"$copies"
# sc_rcdir <path>: the directory of the .shellcheckrc or shellcheckrc nearest the
# path, up to the top level, in $rcdir; empty when none (the lookup above the top
# level is the same for every path).
sc_rcdir() {
local d="$1"
rcdir=
while [[ "$d" == */* ]]; do
d="${d%/*}"
[[ -e "$d/.shellcheckrc" || -e "$d/shellcheckrc" ]] && rcdir="$d" && return
done
[[ -e .shellcheckrc || -e shellcheckrc ]] && rcdir=.
return 0
}
# sc_sources <path>: 0 unless ShellCheck, without any rc, SHELLCHECK_OPTS or the
# script's own directives, reports no source statement (SC1090, SC1091) and no
# error, such as a parse error that hides the rest of the script.
sc_sources() {
local probe
probe="$(sed 's/#[[:space:]]*shellcheck/#_/g' -- "$1" |
(unset SHELLCHECK_OPTS && shellcheck --norc -s bash -f gcc -S info -) 2>&1)"
(($? <= 1)) || return 0
grep -q -e '\[SC109[01]\]' -e ': error: ' <<<"$probe"
}
dedupe_files() {
local table="$CHECKS_TMP/blobs" i=0 f h ext dir key rep path
plain=()
for f in ${files[@]+"${files[@]}"}; do
[[ "$f" == *[$'\t\n']* ]] || plain+=("$f")
done
((${#plain[@]} > 1)) || return 0
printf '%s\n' "${plain[@]}" | git hash-object --no-filters --stdin-paths >"$CHECKS_TMP/hashes" || return 1
: >"$table"
while IFS= read -r h; do
f="${plain[i]}"
ext="${f##*.}"
sc_rcdir "$f"
dir=.
[[ "$f" == */* ]] && dir="${f%/*}"
printf '%s %s %s\t%s\t%s\n' "$h" "$ext" "$rcdir" "$dir" "$f" >>"$table"
i=$((i + 1))
done <"$CHECKS_TMP/hashes"
((i == ${#plain[@]})) || return 1
# A group whose paths sit in more than one directory: its first path, probed.
awk -F '\t' '{ if (!($1 in first)) first[$1] = $3; else if (dir[$1] != $2) spread[$1] = 1; dir[$1] = $2 }
END { for (k in spread) print k "\t" first[k] }' "$table" >"$CHECKS_TMP/spread" || return 1
: >"$CHECKS_TMP/split"
while IFS=$'\t' read -r key rep; do
sc_sources "$rep" && printf '%s\n' "$key" >>"$CHECKS_TMP/split"
done <"$CHECKS_TMP/spread"
awk -F '\t' -v probed="$CHECKS_TMP/split" '
BEGIN { while ((got = (getline l < probed)) > 0) apart[l] = 1; if (got < 0) exit 2 }
{ k = ($1 in apart) ? $1 "\t" $2 : $1; if (!(k in rep)) rep[k] = $3; print rep[k] "\t" $3 }' "$table" >"$CHECKS_TMP/reps" || return 1
lint=()
while IFS=$'\t' read -r rep path; do
if [[ "$rep" == "$path" ]]; then
lint+=("$path")
else
printf '%s\t%s\n' "$rep" "$path" >>"$copies"
fi
done <"$CHECKS_TMP/reps"
for f in ${files[@]+"${files[@]}"}; do
[[ "$f" == *[$'\t\n']* ]] && lint+=("$f")
done
return 0
}
if ! dedupe_files; then
lint=(${files[@]+"${files[@]}"})
: >"$copies"
fi
# Windows caps a whole command line at 32,767 characters, and ShellCheck 0.11.0
# takes its file list only as arguments, so the paths go in batches whose
# summed length, plus a separator each, stays under this budget; the rest of
Expand All @@ -130,7 +214,7 @@ shellcheck_batch() {
}
batch=()
size=0
for f in ${files[@]+"${files[@]}"}; do
for f in ${lint[@]+"${lint[@]}"}; do
if ((${#batch[@]} && size + ${#f} + 1 > sc_budget)); then
shellcheck_batch "${batch[@]}"
batch=()
Expand All @@ -140,6 +224,25 @@ for f in ${files[@]+"${files[@]}"}; do
size=$((size + ${#f} + 1))
done
((${#batch[@]})) && shellcheck_batch "${batch[@]}"
# Each finding of a linted path is repeated for every copy of it, right after.
if [[ -s "$copies" ]]; then
awk -v copies="$copies" '
BEGIN {
while ((got = (getline l < copies)) > 0) { t = index(l, "\t"); r = substr(l, 1, t - 1); c[r] = (r in c) ? c[r] "\t" substr(l, t + 1) : substr(l, t + 1) }
if (got < 0) exit 2
}
{ print }
match($0, /:[0-9]+:[0-9]+: /) && (substr($0, 1, RSTART - 1) in c) {
n = split(c[substr($0, 1, RSTART - 1)], m, "\t")
for (i = 1; i <= n; i++) print m[i] substr($0, RSTART)
}' "$out" >"$out.copies"
copied=$?
if ((copied == 0)); then
mv "$out.copies" "$out"
else
checks_tool_failed awk "$copied" 'copying the shellcheck findings to identical scripts'
fi
fi
# A script gone from the work tree or partly staged is read from the index on
# stdin, named "-".
for f in ${indexed[@]+"${indexed[@]}"}; do
Expand Down
98 changes: 89 additions & 9 deletions plugins/bash-format/git-hooks/shell-check.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
# --worktree reports without rewriting, a merge leaves a script it takes
# unchanged from the merged branch alone, a missing tool follows the skip-key rule,
# many staged paths reach ShellCheck in batches under Windows' command-line cap,
# identical scripts are linted once unless their rc or sourced files could differ,
# and the plugin's own vendored installer wires the entry so `git commit` runs it.
# Expected values are the tools' documented output: a tab-indented body formats to
# the two spaces indent_size = 2 asks for; SC2154 is "<var> is referenced but not
Expand Down Expand Up @@ -74,7 +75,9 @@ HINT='bash-format: fix the findings above, or adjust the repository .shellcheckr
# paths must reach shellcheck in more than one call. These cases put a stub in
# its place and need no real one: past the cap the stub fails to launch
# with status 126, as Windows does; otherwise it logs "--" per call and each path
# it gets, prints a finding for a *-finding.sh path and exits 2 for *-broken.sh.
# it gets, prints a finding for a *-finding.sh path or one holding "stub: finding",
# and exits 2 for *-broken.sh. Identical scripts are linted once, so each fixture
# here holds its own bytes.
mkdir -p "$WORK/sc-stub"
cat >"$WORK/sc-stub/shellcheck" <<'STUB'
#!/usr/bin/env bash
Expand All @@ -91,11 +94,13 @@ rc=0
for f in "$@"; do
printf '%s\n' "$f" >>"$SC_STUB_LOG"
case "$f" in
*-finding.sh)
printf '%s:2:1: warning: stub finding [SC2154]\n' "$f"
((rc < 1)) && rc=1
;;
*-broken.sh) rc=2 ;;
*)
if [[ "$f" == *-finding.sh ]] || grep -q 'stub: finding' "$f"; then
printf '%s:2:1: warning: stub finding [SC2154]\n' "$f"
((rc < 1)) && rc=1
fi
;;
esac
done
exit "$rc"
Expand All @@ -113,28 +118,46 @@ LONG="$(printf 'd%.0s' {1..60})/$(printf 'f%.0s' {1..80})"
new_repo NONE
mkdir -p "$REPO/${LONG%/*}"
for i in {1..250}; do
printf '#!/bin/bash\necho ok\n' >"$REPO/$LONG-$i.sh"
printf '#!/bin/bash\necho ok %s\n' "$i" >"$REPO/$LONG-$i.sh"
done
git -C "$REPO" add -A
expect "many long staged paths: the hook passes" "0|" "$(sc_stub "$REPO")"
expect "many long staged paths: shellcheck ran in more than one call" yes \
"$( (($(grep -cx -- -- "$WORK/sc.log") > 1)) && echo yes)"
expect "many long staged paths: each path reached shellcheck exactly once" \
"$(git -C "$REPO" diff --cached --name-only | sort)" "$(grep -vx -- -- "$WORK/sc.log" | sort)"
printf '#!/bin/bash\necho ok\n' >"$REPO/a-finding.sh"
printf '#!/bin/bash\necho ok\n' >"$REPO/z-finding.sh"
printf '#!/bin/bash\necho a\n' >"$REPO/a-finding.sh"
printf '#!/bin/bash\necho z\n' >"$REPO/z-finding.sh"
git -C "$REPO" add a-finding.sh z-finding.sh
expect "a finding from the first batch and one from the last both block" "1|a-finding.sh:2: warning: stub finding [SC2154]
z-finding.sh:2: warning: stub finding [SC2154]
$HINT" "$(sc_stub "$REPO")"
git -C "$REPO" rm -q --cached a-finding.sh z-finding.sh
rm "$REPO/a-finding.sh" "$REPO/z-finding.sh"
printf '#!/bin/bash\necho ok\n' >"$REPO/a-broken.sh"
printf '#!/bin/bash\necho broken\n' >"$REPO/a-broken.sh"
git -C "$REPO" add a-broken.sh
out="$(sc_stub "$REPO")"
expect "a first batch's status above 1 still blocks after a later batch passes" \
"1|bash-format: shellcheck failed:" "${out%%$'\n'*}"

# Identical staged scripts in one directory are linted once; each copy gets the
# findings under its own path, and its own added lines decide what blocks. b.sh
# already held line 2, so its copy of the finding there does not block.
new_repo NONE
printf '#!/bin/bash\n# stub: finding\necho before\n' >"$REPO/b-copy.sh"
git -C "$REPO" add b-copy.sh
git -C "$REPO" commit -q -m b --no-verify
for f in a-copy.sh b-copy.sh c-copy.sh; do
printf '#!/bin/bash\n# stub: finding\necho same\n' >"$REPO/$f"
done
printf '#!/bin/bash\necho other\n' >"$REPO/d-other.sh"
git -C "$REPO" add a-copy.sh b-copy.sh c-copy.sh d-other.sh
expect "identical copies: every copy whose added lines hold the finding reports it" "1|a-copy.sh:2: warning: stub finding [SC2154]
c-copy.sh:2: warning: stub finding [SC2154]
$HINT" "$(sc_stub "$REPO")"
expect "identical copies are linted once, differing content still each" "a-copy.sh
d-other.sh" "$(grep -vx -- -- "$WORK/sc.log" | sort)"

for tool in shfmt shellcheck jq node; do
if ! command -v "$tool" >/dev/null 2>&1; then
echo "SKIP: $tool not on PATH -- the remaining shell-check tests skipped"
Expand Down Expand Up @@ -288,6 +311,63 @@ git -C "$REPO" add a.sh
expect "no .editorconfig: ShellCheck still reports" "1||a.sh:2: warning: undefined_var is referenced but not assigned. [SC2154]
$HINT" "$(check "$REPO")"

# Identical bytes in different directories: ShellCheck reads the .shellcheckrc
# nearest each script, and a SCRIPTDIR search path resolves a sourced file from
# each script's own directory, so such copies are linted apart unless neither
# can change the result. A shim in front of the real shellcheck logs each call.
mkdir -p "$WORK/sc-count"
printf '#!/usr/bin/env bash\nprintf "%%s\\n" "$*" >>"$SC_COUNT_LOG"\nexec %q "$@"\n' "$(command -v shellcheck)" >"$WORK/sc-count/shellcheck"
chmod +x "$WORK/sc-count/shellcheck"
sc_count() { # <repo> -> "<rc>|<stdout and stderr>"; the calls go to $WORK/count.log
: >"$WORK/count.log"
local out
out="$(cd "$1" && SC_COUNT_LOG="$WORK/count.log" PATH="$WORK/sc-count:$PATH" bash "$CHECK" 2>&1)"
printf '%s|%s' "$?" "$out"
}
linted() { # <path>: how many lint calls named it
grep -c -- "-x .* $1\( \|\$\)" "$WORK/count.log"
}
UNDEF="$SHEBANG"$'\necho "$undefined_var"'

new_repo NONE
mkdir -p "$REPO/x" "$REPO/y"
printf '%s\n' "$UNDEF" >"$REPO/x/a.sh"
printf '%s\n' "$UNDEF" >"$REPO/y/a.sh"
git -C "$REPO" add x y
expect "copies in two directories that source nothing: each copy reports the finding" "1|x/a.sh:2: warning: undefined_var is referenced but not assigned. [SC2154]
y/a.sh:2: warning: undefined_var is referenced but not assigned. [SC2154]
$HINT" "$(sc_count "$REPO")"
expect "copies in two directories that source nothing: linted once" "1|0" "$(linted x/a.sh)|$(linted y/a.sh)"

new_repo NONE
mkdir -p "$REPO/pkg"
printf 'disable=SC2154\n' >"$REPO/pkg/.shellcheckrc"
printf '%s\n' "$UNDEF" >"$REPO/a.sh"
printf '%s\n' "$UNDEF" >"$REPO/pkg/a.sh"
git -C "$REPO" add a.sh pkg
expect "a copy under its own .shellcheckrc is linted apart: its rc disables the finding" "1|a.sh:2: warning: undefined_var is referenced but not assigned. [SC2154]
$HINT" "$(sc_count "$REPO")"
expect "a copy under its own .shellcheckrc: both copies linted" "1|1" "$(linted a.sh)|$(linted pkg/a.sh)"

new_repo NONE
printf 'source-path=SCRIPTDIR\nexternal-sources=true\n' >"$REPO/.shellcheckrc"
mkdir -p "$REPO/x" "$REPO/y"
printf '%s\nsource helper.sh\necho "$v"\n' "$SHEBANG" >"$REPO/x/a.sh"
printf '%s\nsource helper.sh\necho "$v"\n' "$SHEBANG" >"$REPO/y/a.sh"
printf '# shellcheck shell=bash\nv=1\nexport v\n' >"$REPO/x/helper.sh"
printf '# shellcheck shell=bash\n:\n' >"$REPO/y/helper.sh"
git -C "$REPO" add .shellcheckrc x/a.sh y/a.sh
expect "copies that source a file from their own directory are linted apart" "1|y/a.sh:3: warning: v is referenced but not assigned. [SC2154]
$HINT" "$(sc_count "$REPO")"

new_repo NONE
mkdir -p "$REPO/x" "$REPO/y"
printf '%s\n# shellcheck disable=SC1091\nif true; then\necho (\nsource helper.sh\n' "$SHEBANG" >"$REPO/x/a.sh"
cp "$REPO/x/a.sh" "$REPO/y/a.sh"
git -C "$REPO" add x y
sc_count "$REPO" >/dev/null
expect "copies the probe cannot read (a parse error, a disabled SC1091) are linted apart" "1|1" "$(linted x/a.sh)|$(linted y/a.sh)"

new_repo $'root = true\n\n[*.sh]\nindent_style = space\nindent_size = 2'
mkdir -p "$REPO/pkg/sub"
printf 'root = true\n\n[*.md]\nindent_size = 4\n' >"$REPO/pkg/.editorconfig"
Expand Down
Loading