diff --git a/scripts/hygiene.sh b/scripts/hygiene.sh index 04fa8279..ffc9b1f2 100755 --- a/scripts/hygiene.sh +++ b/scripts/hygiene.sh @@ -15,7 +15,13 @@ # # The script is duplicated per repo on purpose: a shared copy would have to live # somewhere central, and pointing at it from a published repo is itself the kind -# of cross-reference these rules forbid. Keep the five copies identical. +# of cross-reference these rules forbid. +# +# Every copy must be byte-identical. Once they were not: they drifted for twelve +# days into several generations at once, and the rules added most recently — the +# ones written because a whole class of violation was getting through — were +# present in some copies and missing from others. Editing one copy without the +# rest is how that happens. # # Usage: hygiene_check public # this repo is published # hygiene_check infra # private-but-shared: may name vault paths @@ -25,8 +31,55 @@ # a private script mentioned with its path is caught by the infra/ rule anyway. HYGIENE_PRIVATE='(\binfra/|pickle/secrets|(^|[^a-z])secrets/|\b(deploy|smoke|apply|create|sync|provision)-[a-z][a-z-]*\.sh\b)' -# Internal process vocabulary. The trailing [A-Z]? is load-bearing: M4A, W2-B. -HYGIENE_TOKENS='\b(M[0-9]+(\.[0-9]+)?[A-Z]?|W[0-9]+(\.[0-9]+)?(-[A-Z])?|G[0-9]|B[0-9]|A[0-9]|C[0-9]|R[12]|S([1-9]|1[0-3])|O([1-9]|10)|F[0-9]|H1|WP-[A-Z0-9]+|api-[A-Z]|Lane [A-Z])\b|보안 게이트|review finding|gate finding|work package' +# Internal process vocabulary, in the case it is normally written. The trailing +# [A-Z]? is load-bearing: M4A, W2-B. +HYGIENE_TOKENS='\b(M[0-9]+(\.[0-9]+)?[A-Z]?|W[0-9]+(\.[0-9]+)?(-[A-Z])?|G[0-9]|B[0-9]|A[0-9]|C[0-9]|R[12]|S([1-9]|1[0-3])|O([1-9]|10)|F[0-9]|H1|WP-[A-Z0-9]+|api-[A-Z]|Lane [A-Z])\b' + +# The same vocabulary in lowercase — the case the pattern above can never see, +# so a token that survives into an identifier survives the gate. A plain `grep +# -i` is not the fix: lowercased, every one of these tokens is also ordinary +# code (c1 a connection, s2 a state, -f2 a cut field,

a heading, .m2 the +# maven home, w3.org in a URL), and the gate would drown in them. A lowercase +# token therefore has to look like a word in running text: it starts the line — +# or, once grep -Hn has prefixed the file and line, follows that colon — or +# follows a space or an opening bracket, never a quote, dot, slash or dash, +# which is where identifiers live; and it ends at a word end or at a camelCase +# hump, the shape that catches m7Rejects. A trailing dot counts only when a +# letter does not follow it, so `shipped in m6.` is caught while `c1.send(` is +# not. Digits are capped at two because milestones and waves are numbered in +# tens, while uncapped digits matched password fixtures (m1234567) and +# subdomains (m365). `_` deliberately does not end a token: \b does not end one +# either (it counts _ as a word character), so M7_x is not caught in capitals +# and the lowercase rule matches that reach rather than inventing its own. +# shellcheck disable=SC2016 +HYGIENE_TOKEN_BODY_LC='m[0-9]{1,2}(\.[0-9]+)?[a-z]?|w[0-9]{1,2}(\.[0-9]+)?(-[a-z])?|g[0-9]|b[0-9]|a[0-9]|c[0-9]|r[12]|s([1-9]|1[0-3])|o([1-9]|10)|f[0-9]|h1|wp-[a-z0-9]+|api-[a-z]|lane [a-z]' +# The opening context is start-of-line, the colon `grep -Hn` puts in front of +# the content, or a space — optionally followed by a bracket that a space +# opened. That last detail is what separates prose from a call: in +# `phase roles (g5)` the `(` follows a space, in `draw(g1, ctx)` it follows an +# identifier, and only the first is text. +# +# The closing context is a camelCase hump (`m7Rejects`), a sentence-ending dot, +# a comma or colon that a space or line end follows, a space followed by +# anything that is not an operator, or the line end. A bracket-opened token may +# also close on its bracket. What this deliberately excludes is the shape code +# uses: `c1 = …`, `c1;`, `foo(a1, b2)`, `s2 += 1`. A short lowercase name +# standing alone is a token in prose and a variable in code, and only the +# characters around it say which — so the rule reads them instead of listing +# the names, which would have to grow forever. +HYGIENE_TOKENS_LC='(^|:| )[([]('"$HYGIENE_TOKEN_BODY_LC"')([])]|[A-Z]|\.([^A-Za-z]|$)| [^=+*/<>&|%^~]| $|$)|(^|:| )('"$HYGIENE_TOKEN_BODY_LC"')([A-Z]|\.([^A-Za-z]|$)|[,:]( |$)| [^=+*/<>&|%^~]| $|$)' + +# A path is not running text: its word separators are / _ . and -, so the same +# vocabulary needs its own opening set there. Without it `notes/m4a-rollout.txt` +# would pass while `notes/M4A-rollout.txt` fails, because \b already treats the +# slash as a word start — the capitalised rule reaches into a path and the +# lowercase one has to reach just as far. +HYGIENE_TOKENS_LC_PATH='(^|[/_.-])('"$HYGIENE_TOKEN_BODY_LC"')([A-Z]|[^A-Za-z0-9_]|$)' + +# Process phrases. These are ordinary words rather than IDs, so nothing about +# them collides with code and they are matched case-blind: "Work Package" +# opening a sentence is the same leak as "work package" inside one. +HYGIENE_PHRASES='보안 게이트|review finding|gate finding|work package' # Substrings that merely LOOK like a token or a reference. They are REMOVED from # the line before the test, never used to suppress the line: a line-level @@ -34,12 +87,17 @@ HYGIENE_TOKENS='\b(M[0-9]+(\.[0-9]+)?[A-Z]?|W[0-9]+(\.[0-9]+)?(-[A-Z])?|G[0-9]|B # revision let `M4A … contract v0.14.1` through, and how any comment sharing a # line with an inline SVG passed. `d="…"` is stripped as a whole attribute so the # next path command (M12…) cannot survive as a lookalike. +# `SHA256:` covers an SSH key fingerprint: what follows is base64, and a base64 +# run contains every shape in the vocabulary (`SHA256:g1A4pf…` reads as g1 in +# lowercase). It is stripped by prefix rather than by blob shape so that only a +# digest, never an arbitrary long word, becomes invisible to the rules. # The `$R1`/`$R2` entries are shell variable names appearing literally in a -# regex, not expansions. `S3` and `-O2` are allowed because the storage service -# and the compiler flag are likelier in real code than the finding IDs they -# collide with. +# regex, not expansions. `S3`/`s3` and `-O2` are allowed because the storage +# service and the compiler flag are likelier in real code than the finding IDs +# they collide with — the storage service in either case, since it is written +# both ways. # shellcheck disable=SC2016 -HYGIENE_ALLOW='d="[^"]*"|sha512-[A-Za-z0-9+/=]*|"integrity"|"$err" || true) + if [ -s "$err" ]; then + echo "hygiene: scan did not complete (rule /$pattern/):" >&2 + cat "$err" >&2 + rm -f "$err" + return 2 + fi + rm -f "$err" + printf '%s' "$out" +} + hygiene_check() { - local kind="$1" rc=0 hits paths mdallow + local kind="$1" rc=0 hits raw paths mdallow local -a files # Fail closed: an empty file list means the scan did not run (not a git @@ -88,11 +176,10 @@ hygiene_check() { return 1 fi - hits=$(printf '%s\0' "${files[@]}" \ - | xargs -0 grep -HnIE "(\.\./docs|(^|[^a-z])docs/)" 2>/dev/null \ - | hygiene_match "(\.\./docs|(^|[^a-z])docs/)" || true) + if ! raw=$(hygiene_scan -HnI "(\.\./docs|(^|[^a-z])docs/)"); then rc=1; raw=""; fi + hits=$(printf '%s' "$raw" | hygiene_match "(\.\./docs|(^|[^a-z])docs/)" || true) if [ -n "$hits" ]; then - echo "hygiene: reference to the documentation repository:" >&2 + echo "hygiene: reference to a documentation path this repository does not contain:" >&2 echo "$hits" >&2 rc=1 fi @@ -101,9 +188,8 @@ hygiene_check() { # reference to an outside document — no list of outside names required, and # a document that does not exist yet is caught the day it is named. mdallow="$(hygiene_md_allow)" || return 1 - hits=$(printf '%s\0' "${files[@]}" \ - | xargs -0 grep -HnoIE '[A-Za-z0-9._-]+\.md\b' 2>/dev/null \ - | grep -vE ":(${mdallow})$" || true) + if ! raw=$(hygiene_scan -HnoI '[A-Za-z0-9._-]+\.md\b'); then rc=1; raw=""; fi + hits=$(printf '%s' "$raw" | grep -vE ":(${mdallow})$" || true) if [ -n "$hits" ]; then echo "hygiene: mention of a markdown document this repo does not contain:" >&2 echo "$hits" >&2 @@ -111,9 +197,8 @@ hygiene_check() { fi if [ "$kind" = public ]; then - hits=$(printf '%s\0' "${files[@]}" \ - | xargs -0 grep -HnIE "$HYGIENE_PRIVATE" 2>/dev/null \ - | hygiene_match "$HYGIENE_PRIVATE" || true) + if ! raw=$(hygiene_scan -HnI "$HYGIENE_PRIVATE"); then rc=1; raw=""; fi + hits=$(printf '%s' "$raw" | hygiene_match "$HYGIENE_PRIVATE" || true) if [ -n "$hits" ]; then echo "hygiene: reference to the private infrastructure repository or the secret vault:" >&2 echo "$hits" >&2 @@ -132,19 +217,26 @@ hygiene_check() { fi fi - hits=$(printf '%s\0' "${files[@]}" \ - | xargs -0 grep -HnIE "$HYGIENE_TOKENS" 2>/dev/null \ - | hygiene_match "$HYGIENE_TOKENS" || true) + if ! raw=$(hygiene_scan -HnI "$HYGIENE_TOKENS|$HYGIENE_TOKENS_LC"); then rc=1; raw=""; fi + hits=$(printf '%s' "$raw" | hygiene_match "$HYGIENE_TOKENS|$HYGIENE_TOKENS_LC" || true) if [ -n "$hits" ]; then echo "hygiene: internal process token (state the fact instead):" >&2 echo "$hits" >&2 rc=1 fi + if ! raw=$(hygiene_scan -HnIi "$HYGIENE_PHRASES"); then rc=1; raw=""; fi + hits=$(printf '%s' "$raw" | hygiene_match_i "$HYGIENE_PHRASES" || true) + if [ -n "$hits" ]; then + echo "hygiene: internal process phrase (state the fact instead):" >&2 + echo "$hits" >&2 + rc=1 + fi + # Path names carry the same rules: a directory called M7-notes/ says as much # as a comment would. paths=$(printf '%s\n' "${files[@]}" \ - | hygiene_match "$HYGIENE_TOKENS" || true) + | hygiene_match "$HYGIENE_TOKENS|$HYGIENE_TOKENS_LC_PATH|$HYGIENE_PHRASES" || true) if [ -n "$paths" ]; then echo "hygiene: file or directory name carries a process token:" >&2 echo "$paths" >&2 @@ -164,7 +256,7 @@ hygiene_check() { # shellcheck disable=SC2030,SC2031 # the checks run in subshells by design; the # variables they read are assigned here and never written back. hygiene_selftest() { - local tmp rc=0 self line + local tmp rc=0 self line bad_path self="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hygiene.sh" tmp="$(mktemp -d)" || return 1 mkdir -p "$tmp/scripts" @@ -210,6 +302,32 @@ frame protocol (Lane C agreement) M4A gate mandated in contract v0.14.1 rollout M4A gate applies to host 10.32.0.5 milestone M6 done +m6 shipped this in lowercase +m4a rollout in lowercase +w1.5 lesson applied +phase roles (w3) +launch gate g5 pending +teardown step (b1) +review finding c4 addressed in lowercase +prefill rule (r1) +s4 hardening item in lowercase +o7 operations item in lowercase +discovered as h1 +admin (wp-f3) queries +landed with the api-b merge +frame protocol (lane c agreement) +void m7Rejects() { +shipped in m6. +fingerprint SHA256:g1A4pfkmf+XmceT0lCSr03Ev landed with the api-b merge +key SHA256:g1A4pfkmf+XmceT0lCSr03Ev rotated in the M4A rollout +config value "gate g5 pending" +void m6KeysAppearInCatalog() { +w3 lesson applied here +phase roles (g5) +rolled back in m6, then re-applied +Work Package delivered +Review Finding closed +GATE FINDING still open SAMPLES # A lowercase document in the tree is refused even though tracking it would @@ -223,6 +341,52 @@ SAMPLES fi rm -f "$tmp/design-notes.md" + # A scan that cannot read the tree must fail the gate rather than report a + # clean run. A dangling symlink is the cheapest way to break grep for real + # (and it breaks it for root too, unlike a permission bit). + ln -s missing-target "$tmp/dangling" + printf 'clean line\n' > "$tmp/sample.txt" + git -C "$tmp" add -A >/dev/null 2>&1 + if ( cd "$tmp" && . scripts/hygiene.sh && hygiene_check public ) >/dev/null 2>&1; then + echo "hygiene selftest: an unreadable tree still reported clean" >&2 + rc=1 + fi + git -C "$tmp" rm -q --cached dangling >/dev/null 2>&1 + rm -f "$tmp/dangling" + + # A path carries the vocabulary as loudly as a comment does, in either case + # and at any depth: \b already reaches into a path for the capitalised form, + # so the lowercase rule is given the same reach (HYGIENE_TOKENS_LC_PATH). + printf 'clean line\n' > "$tmp/sample.txt" + while IFS= read -r bad_path; do + [ -z "$bad_path" ] && continue + mkdir -p "$tmp/$(dirname "$bad_path")" + echo "nothing interesting" > "$tmp/$bad_path" + git -C "$tmp" add -A >/dev/null 2>&1 + if ( cd "$tmp" && . scripts/hygiene.sh && hygiene_check public ) >/dev/null 2>&1; then + echo "hygiene selftest: process token in a path not refused: $bad_path" >&2 + rc=1 + fi + git -C "$tmp" rm -q --cached "$bad_path" >/dev/null 2>&1 + rm -rf "${tmp:?}/${bad_path%%/*}" + done <<'PATHS' +notes/m4a-rollout.txt +notes/M4A-rollout.txt +wp-f3/readme.txt +PATHS + + # ...but an underscore ends a token no more in a path than in running text, + # so an ordinary name keeps working. + mkdir -p "$tmp/nft" + echo "package nft" > "$tmp/nft/m11_in.go" + git -C "$tmp" add -A >/dev/null 2>&1 + if ! ( cd "$tmp" && . scripts/hygiene.sh && hygiene_check public ) >/dev/null 2>&1; then + echo "hygiene selftest: false positive on an ordinary path" >&2 + rc=1 + fi + git -C "$tmp" rm -q --cached nft/m11_in.go >/dev/null 2>&1 + rm -rf "${tmp:?}/nft" + # The opposite direction: legitimate content must pass, or the gate becomes # something people work around. cat > "$tmp/sample.txt" <<'CLEAN' @@ -232,6 +396,31 @@ the L4 forwarder sends PROXY v2 icon see README.md and CHANGELOG.md for details the local GUIDE.md covers this + c1 := connect(t, ts, "tok-1") + s2, err := Load(path) +

Heading

+grep '^KEY=' env | cut -d= -f2 +nft counter m11_new beside m11_in +MAVEN_USER_HOME="${HOME}/.m2" +http://www.w3.org/2001/XMLSchema-instance +password fixture m1234567 rejected +reserved subdomain m365 +the s3 bucket policy +answers 404, 422 and 503 without disclosing which +10.32.0.0/24 reaches 10.32.0.5:8443 and port 22 +icon + var c1 = connections.first(); + let s2 = state.next() + const b1 = buffer[1] + sheet.range(a1) + draw(g1, ctx) + r1 = re.compile(x) + return c1; + foo(a1, b2) + if (s1 && s2) { + let s2 += 1 + const [a1, b1] = pair + x = f1 * 2 CLEAN git -C "$tmp" add -A >/dev/null 2>&1 if ! ( cd "$tmp" && . scripts/hygiene.sh && hygiene_check public ) >/dev/null 2>&1; then