From 776642dae01d5e4ce6253a2d02212d41cd9f16b5 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 16 Nov 2026 00:41:38 -0500 Subject: [PATCH 1/8] feat(harness-config): generate the rule-to-enforcer inventory rule-inventory.sh writes one row per rule (rule, enforcer, rung, tier) from the repository's own sources, and rolls the rows up into a count per ladder tier and a list of guidance rules that sit lower than they could. Rung ids and tiers are read from the ladder copy beside the skill. A source it cannot find or read is named, never counted as zero. Rungs no file shows come from rule_inventory.declared in the harness-config convention file, which the schema now describes. Refs: #15711 Co-Authored-By: Claude Fable 5.2 --- .../schemas/harness-config.schema.json | 21 ++ .../scripts/rule-inventory.sh | 340 ++++++++++++++++++ .../scripts/rule-inventory.test.sh | 185 ++++++++++ 3 files changed, 546 insertions(+) create mode 100755 plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh create mode 100755 plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh diff --git a/plugins/harness-config/schemas/harness-config.schema.json b/plugins/harness-config/schemas/harness-config.schema.json index 6da58858111..6c3c8c207e0 100644 --- a/plugins/harness-config/schemas/harness-config.schema.json +++ b/plugins/harness-config/schemas/harness-config.schema.json @@ -48,6 +48,27 @@ "default": false, "description": "When true, /harness-config:audit-automation-gaps may state the size of an implementation plan in minutes, hours, days or weeks; when false it gives a relative size (Small, Medium, Large). A quoted \"true\" or \"false\" is a string, not a boolean, and is invalid." }, + "rule_inventory": { + "type": "object", + "description": "Inputs to the rule-to-enforcer inventory that /harness-config:audit-automation-gaps prints.", + "additionalProperties": false, + "properties": { + "declared": { + "type": "array", + "description": "Rules held by a mechanism no file shows the inventory, such as a type that makes the mistake impossible or a shared helper a check names. Each item is one inventory row.", + "items": { + "type": "object", + "required": ["rule", "rung"], + "additionalProperties": false, + "properties": { + "rule": { "$ref": "#/$defs/text", "description": "The rule, in one line." }, + "enforcer": { "$ref": "#/$defs/text", "description": "What holds it: a file, type, helper or check." }, + "rung": { "type": "string", "pattern": "^[a-z][a-z-]*$", "description": "A rung id of the enforcement ladder the plugin carries; an id the ladder does not list is reported and skipped." } + } + } + } + } + }, "suppressions": { "type": "object", "description": "Deliberately kept findings of /harness-config:audit and /harness-config:audit-pass, keyed by finding_id.", diff --git a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh new file mode 100755 index 00000000000..d55ccf4f77f --- /dev/null +++ b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh @@ -0,0 +1,340 @@ +#!/usr/bin/env bash +set -u + +usage() { + cat <<'EOF' +rule-inventory.sh - rule-to-enforcer inventory and its per-tier rollups. + +Usage: + rule-inventory.sh [--root DIR] [--rows] [--limit N] + + --root DIR repository to inventory (default: the git toplevel of the + current directory, else the current directory) + --rows print the inventory itself, one tab-separated row per rule: + rule, enforcer, rung, tier; no summary + --limit N how many "sits lower than it could" candidates the summary lists + (default 15; the total is always printed) + +Sources, one adapter each: + declared rule_inventory.declared in /harness-config.yaml, + each item {rule, enforcer, rung}; the only source for rungs no + file shows + settings .claude/settings.json: each permissions.deny entry is a + harness-setting; each PreToolUse or PermissionRequest handler is + an action-guard + ecosystems .claude/ecosystems/*.yaml: each command line of check-cmd is an + analyzer-pack-rule, of test-cmd a behavior-test, of build-cmd a + compiler-setting + test files tracked files named *.test.*, *.spec.*, *_test.*, test_*.* or + *Tests.*, one behavior-test row per file (a unit of files, not + of test cases) + control .claude/skills/control-*/SKILL.md, a runtime-verification each + review statements of REVIEW.md, an automated-review each + instruction statements of every tracked CLAUDE.md and AGENTS.md and of + .claude/rules/*.md, an instruction each + skills every other .claude/skills/*/SKILL.md, a skill each + human statements of CONTRIBUTING.md and checklist items of the pull + request template, a human-review each +A statement is one list item or one prose paragraph, its wrapped lines joined; +headings, tables, fences, comments and @-imports are not statements. Files +under a fixtures, evals, tests or testdata directory are not read as the +repository's own instructions. + +Sits lower than it could: a guidance-tier statement in which never, do not, +don't or must not is followed, within six words of the same clause, by a +backticked literal forbids that literal, which a pattern check could match. +It is listed unless the statement carries "Enforced by:" or the literal +appears in a static-analysis or codebase row. These are candidates for a +person to judge, not findings. + +Exit: 0 once the inventory is produced (a source it could not read is named +in the output); 2 for a usage error, a root it cannot enter, or a ladder copy +it cannot read. +EOF +} + +root="" +rows_only=0 +limit=15 +while [[ $# -gt 0 ]]; do + case "$1" in + -h | --help) usage; exit 0 ;; + --root) + [[ $# -ge 2 ]] || { echo "rule-inventory.sh: --root needs a directory" >&2; exit 2; } + root="$2"; shift ;; + --rows) rows_only=1 ;; + --limit) + [[ $# -ge 2 && "$2" =~ ^[0-9]+$ ]] || { echo "rule-inventory.sh: --limit needs a number" >&2; exit 2; } + limit="$2"; shift ;; + *) echo "rule-inventory.sh: unrecognized argument: $1" >&2; usage >&2; exit 2 ;; + esac + shift +done + +script_dir="$(cd "${BASH_SOURCE[0]%/*}" && pwd)" +plugin_root="${CLAUDE_PLUGIN_ROOT:-$(cd "$script_dir/../../.." && pwd)}" +ladder="${RULE_INVENTORY_LADDER:-$script_dir/../context/enforcement-ladder.md}" +yaml_awk="$plugin_root/lib/yaml-subset.awk" +resolve_home="$plugin_root/lib/resolve-convention-home.sh" + +if [[ -z "$root" ]]; then + root="$(git rev-parse --show-toplevel 2>/dev/null | tr -d '\r')" + root="${root:-$PWD}" +fi +if [[ ! -d "$root" ]] || ! cd -- "$root" 2>/dev/null; then + echo "rule-inventory.sh: root $root is not a directory this run could enter" >&2 + exit 2 +fi +root="$PWD" + +[[ -r "$ladder" ]] || { echo "rule-inventory.sh: ladder copy $ladder is not readable" >&2; exit 2; } +ladder_version="$(awk -F': *' '/^enforcement-ladder:/ {print $2; exit}' "$ladder")" +rungs="$(awk ' + /^## / && seen { exit } + /^### Tier [0-9]+: / { tier = $3; sub(/:$/, "", tier); name = $0; sub(/^### Tier [0-9]+: /, "", name); seen = 1; next } + !tier { next } + /^[0-9]+\. \*\*`[a-z-]+`\*\*/ { id = $0; sub(/^[0-9]+\. \*\*`/, "", id); sub(/`.*/, "", id); print id "\t" tier "\t" name; next } + /^\| `[a-z-]+` \|/ { id = $0; sub(/^\| `/, "", id); sub(/`.*/, "", id); print id "\t" tier "\t" name } +' "$ladder")" +[[ -n "$rungs" ]] || { echo "rule-inventory.sh: no rung ids found in $ladder" >&2; exit 2; } +tier_of() { awk -F'\t' -v r="$1" '$1 == r {print $2; exit}' <<<"$rungs"; } + +work="$(mktemp -d)" || exit 2 +trap 'rm -rf -- "$work"' EXIT +rows="$work/rows.tsv" +: >"$rows" +notes="$work/notes.txt" +: >"$notes" +note() { printf '%s\n' "$*" >>"$notes"; } + +clean() { tr '\t\r' ' ' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'; } +emit() { + local t + t="$(tier_of "$3")" + if [[ -z "$t" ]]; then + note "skipped: rung \`$3\` from $2 is not a rung of ladder $ladder_version" + return + fi + printf '%s\t%s\t%s\t%s\n' "$(printf '%s' "$1" | clean)" "$2" "$3" "$t" >>"$rows" +} + +if git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + git -c core.quotepath=off ls-files >"$work/files" 2>/dev/null +else + find . -path ./.git -prune -o -path ./node_modules -prune -o -type f -print | sed 's|^\./||' >"$work/files" +fi +own_files() { grep -Ev '(^|/)(fixtures|evals|tests|testdata)/' "$work/files"; } + +list_items() { + awk -v only_check="${2:-0}" ' + function flush() { if (text != "" && (only_check != 1 || check)) print start "\t" text; text = ""; check = 0 } + function add(s) { sub(/^[[:space:]]+/, "", s); sub(/[[:space:]]+$/, "", s); text = text == "" ? s : text " " s } + NR == 1 && /^---[[:space:]]*$/ { front = 1; next } + front { if (/^---[[:space:]]*$/) front = 0; next } + /^[[:space:]]*(```|~~~)/ { flush(); fence = !fence; next } + fence { next } + comment { if (/-->/) comment = 0; next } + /^[[:space:]]*/) comment = 1; next } + /^[[:space:]]*$/ || /^[[:space:]]*#/ || /^[[:space:]]*\|/ || /^@[^[:space:]]+[[:space:]]*$/ { flush(); next } + /^[[:space:]]*([-*+]|[0-9]+[.)])[[:space:]]+[^[:space:]]/ { + flush(); start = NR; t = $0 + if (t ~ /^[[:space:]]*[-*+][[:space:]]+\[[ xX]\][[:space:]]+/) { check = 1; sub(/^[[:space:]]*[-*+][[:space:]]+\[[ xX]\][[:space:]]+/, "", t) } + else sub(/^[[:space:]]*([-*+]|[0-9]+[.)])[[:space:]]+/, "", t) + add(t); next + } + { if (text == "") start = NR; add($0) } + END { flush() } + ' "$1" +} +emit_items() { + local line text + while IFS=$'\t' read -r line text; do + emit "$text" "$1:$line" "$2" + done < <(list_items "$1" "${3:-0}") +} + +read_status() { + if [[ ! -e "$1" ]]; then echo absent + elif [[ ! -r "$1" ]]; then echo unreadable + else echo ok; fi +} + +home="docs/conventions" +if [[ -r "$resolve_home" ]]; then + h="$(bash "$resolve_home" --root "$root" 2>/dev/null)" && home="$h" +fi +decl="$home/harness-config.yaml" +case "$(read_status "$decl")" in +absent) note "declared: $decl absent, so no declared rows" ;; +unreadable) note "declared: $decl unreadable" ;; +*) + if ! command -v awk >/dev/null || [[ ! -r "$yaml_awk" ]]; then + note "declared: $decl not parsed (no YAML reader at $yaml_awk)" + elif ! flat="$(awk -f "$yaml_awk" "$decl")"; then + note "declared: $decl does not parse: ${flat#error }" + else + n=0 + while :; do + r="$(awk -F'\t' -v k="rule_inventory.declared.$n.rule" '$1 == k {print $2}' <<<"$flat")" + e="$(awk -F'\t' -v k="rule_inventory.declared.$n.enforcer" '$1 == k {print $2}' <<<"$flat")" + g="$(awk -F'\t' -v k="rule_inventory.declared.$n.rung" '$1 == k {print $2}' <<<"$flat")" + [[ -n "$r$e$g" ]] || break + if [[ "$g" == semgrep-rule ]]; then + note "declared: item $n names semgrep-rule, read as syntax-pattern-rule (renamed in ladder 3.0.0)" + g=syntax-pattern-rule + fi + if [[ -z "$r" || -z "$g" ]]; then + note "declared: item $n in $decl lacks a rule or a rung; skipped" + else + emit "$r" "${e:-declared in $decl}" "$g" + fi + n=$((n + 1)) + done + note "declared: $decl read, $n item(s)" + fi + ;; +esac + +settings=".claude/settings.json" +case "$(read_status "$settings")" in +absent) note "settings: $settings absent" ;; +unreadable) note "settings: $settings unreadable" ;; +*) + if ! command -v jq >/dev/null; then + note "settings: $settings not read (jq is not on PATH)" + elif ! jq -e 'type == "object"' "$settings" >/dev/null 2>&1; then + note "settings: $settings is not a JSON object" + else + while IFS= read -r d; do emit "$d" "$settings permissions.deny" harness-setting; done \ + < <(jq -r '(.permissions.deny // [])[] | strings' "$settings") + while IFS= read -r h; do emit "$h" "$settings hooks" action-guard; done \ + < <(jq -r '(.hooks // {}) | to_entries[] | select(.key == "PreToolUse" or .key == "PermissionRequest") + | .key as $ev | (.value // [])[] | (.matcher // "*") as $m | (.hooks // [])[] + | "\($ev) \($m)" + (if .if then " if \(.if)" else "" end)' "$settings") + note "settings: $settings read" + fi + ;; +esac + +eco_n=0 +for f in .claude/ecosystems/*.yaml; do + [[ -e "$f" ]] || continue + if [[ ! -r "$f" ]]; then note "ecosystems: $f unreadable"; continue; fi + eco_n=$((eco_n + 1)) + name="${f##*/}"; name="${name%.yaml}" + while IFS=$'\t' read -r key cmd; do + case "$key" in + check-cmd) rung=analyzer-pack-rule ;; + test-cmd) rung=behavior-test ;; + build-cmd) rung=compiler-setting ;; + *) continue ;; + esac + emit "$name $key: $cmd" "$f" "$rung" + done < <(awk ' + function val(v) { sub(/^[[:space:]]+/, "", v); sub(/[[:space:]]+$/, "", v); return v } + block && /^[^[:space:]#]/ { block = "" } + block { l = val($0); if (l != "" && l !~ /^#/) print block "\t" l; next } + /^(check|test|build)-cmd:/ { + k = $0; sub(/:.*/, "", k); v = $0; sub(/^[^:]*:/, "", v); v = val(v) + if (v == "|" || v == ">" || v == "|-" || v == ">-") { block = k; next } + if (v == "" || v == "null" || v == "~") next + if (v ~ /^".*"$/ || v ~ /^\047.*\047$/) v = substr(v, 2, length(v) - 2) + print k "\t" v + } + ' "$f") +done +if [[ "$eco_n" -eq 0 ]]; then + note "ecosystems: no .claude/ecosystems/*.yaml record, so no recorded build, test or check command" +else + note "ecosystems: $eco_n record(s) read" +fi + +tests="$(own_files | grep -E '(^|/)([^/]+\.test\.[^/]+|[^/]+\.spec\.[^/]+|[^/]+_test\.[^/]+|test_[^/]+\.[^/]+|[^/]*[a-z]Tests\.[^/]+)$' || true)" +if [[ -n "$tests" ]]; then + while IFS= read -r t; do emit "tests in $t" "$t" behavior-test; done <<<"$tests" +fi +note "test files: one behavior-test row per test file found by name; tests in files named otherwise are not counted" + +for s in .claude/skills/*/SKILL.md; do + [[ -e "$s" ]] || continue + d="${s%/SKILL.md}"; d="${d##*/}" + case "$d" in + control-*) emit "runtime verification through the $d skill" "$s" runtime-verification ;; + *) emit "the $d skill's procedure" "$s" skill ;; + esac +done + +review_seen=0 +if [[ -r REVIEW.md ]]; then emit_items REVIEW.md automated-review; review_seen=1; fi +[[ "$review_seen" = 1 ]] || note "review: REVIEW.md absent" +inst="$(own_files | grep -E '(^|/)(CLAUDE|AGENTS)\.md$' || true)" +rules="$(own_files | grep -E '^\.claude/rules/[^/]+\.md$' || true)" +inst_n=0 +while IFS= read -r f; do + [[ -n "$f" ]] || continue + if [[ -r "$f" ]]; then emit_items "$f" instruction; inst_n=$((inst_n + 1)); else note "instruction: $f unreadable"; fi +done <<<"$inst +$rules" +note "instruction: $inst_n instruction file(s) read" + +human_n=0 +for f in CONTRIBUTING.md docs/CONTRIBUTING.md .github/CONTRIBUTING.md; do + [[ -r "$f" ]] && { emit_items "$f" human-review; human_n=$((human_n + 1)); } +done +for f in .github/pull_request_template.md .github/PULL_REQUEST_TEMPLATE.md PULL_REQUEST_TEMPLATE.md docs/pull_request_template.md; do + [[ -r "$f" ]] && { emit_items "$f" human-review 1; human_n=$((human_n + 1)); } +done +[[ "$human_n" -gt 0 ]] || note "human: no CONTRIBUTING.md and no pull request template in this repository" + +if [[ "$rows_only" = 1 ]]; then + cat "$rows" + exit 0 +fi + +awk -F'\t' '$4 <= 2 { print $1; print $2 }' "$rows" >"$work/strong.txt" +lower="$work/lower.txt" +: >"$lower" +# shellcheck disable=SC2016 # the backticks are Markdown code-span delimiters +forbid_re='(^|[^[:alpha:]])([Nn]ever|NEVER|[Dd]o not|[Dd]on'"'"'t|[Mm]ust not)(([[:space:]]+[^[:space:]`.,;:]+){0,6})[[:space:]]+`([^`]+)`' +while IFS=$'\t' read -r rule enforcer rung tier; do + [[ "$tier" -ge 4 ]] || continue + [[ "$rule" != *"Enforced by:"* ]] || continue + rest="$rule" + open="" + while [[ "$rest" =~ $forbid_re ]]; do + lit="${BASH_REMATCH[5]}" + rest="${rest#*"${BASH_REMATCH[0]}"}" + grep -Fq -- "$lit" "$work/strong.txt" || open="${open:+$open, }\`$lit\`" + done + [[ -z "$open" ]] || printf '%s: forbids %s (%s)\n' "$enforcer" "$open" "$rung" >>"$lower" +done <"$rows" + +total="$(wc -l <"$rows" | tr -d ' ')" +echo "Rule-to-enforcer inventory: $total rule(s) in $root" +echo "Ladder: enforcement-ladder ${ladder_version:-unknown}. Counts only; the ladder sets no share or count to reach." +echo +echo "| Tier | Rules | By rung |" +echo "|---|---|---|" +for t in $(cut -f2 <<<"$rungs" | uniq); do + tname="$(awk -F'\t' -v t="$t" '$2 == t {print $3; exit}' <<<"$rungs")" + n="$(awk -F'\t' -v t="$t" '$4 == t' "$rows" | wc -l | tr -d ' ')" + by="$(awk -F'\t' -v t="$t" '$4 == t {c[$3]++} END {for (r in c) print r " " c[r]}' "$rows" | sort | paste -sd, - | sed 's/,/, /g')" + if [[ "$n" -eq 0 ]]; then + case "$t" in + 1) by="none declared; no file shows this tier, so only declared rows count" ;; + *) by="none found in the sources below" ;; + esac + fi + echo "| $t $tname | $n | $by |" +done +echo +echo "A row is one deny entry, hook handler, recorded command, test file, skill or guidance statement, whatever it holds." +echo +nl="$(wc -l <"$lower" | tr -d ' ')" +echo "Sits lower than it could: $nl guidance rule(s) forbid a literal that a pattern check could match (candidates to judge, not findings)" +head -n "$limit" "$lower" | sed 's/^/- /' +[[ "$nl" -le "$limit" ]] || echo "- ... and $((nl - limit)) more (--limit to list more)" +echo +echo "Sources:" +sed 's/^/- /' "$notes" +exit 0 diff --git a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh new file mode 100755 index 00000000000..e94aee59ac4 --- /dev/null +++ b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +# test-scope: plugins/harness-config/skills/audit-automation-gaps/context/enforcement-ladder.md plugins/harness-config/lib/yaml-subset.awk plugins/harness-config/lib/resolve-convention-home.sh +# shellcheck disable=SC2016 +set -uo pipefail +unset GIT_DIR GIT_WORK_TREE GIT_CONFIG GIT_INDEX_FILE + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +GEN="$SCRIPT_DIR/rule-inventory.sh" +LADDER="$SCRIPT_DIR/../context/enforcement-ladder.md" + +FAILED=0 +CASES=0 +pass() { CASES=$((CASES + 1)); printf 'PASS: %s\n' "$1"; } +fail() { CASES=$((CASES + 1)); FAILED=$((FAILED + 1)); printf 'FAIL: %s\n detail: %s\n' "$1" "$2" >&2; } +assert_eq() { if [[ "$2" == "$3" ]]; then pass "$1"; else fail "$1" "expected [$2], got [$3]"; fi; } +assert_line() { if grep -Fxq -- "$3" <<<"$2"; then pass "$1"; else fail "$1" "no line [$3] in: +$2"; fi; } +assert_no() { if grep -Fq -- "$3" <<<"$2"; then fail "$1" "unexpected [$3] in: +$2"; else pass "$1"; fi; } + +TMP="$(mktemp -d)" +trap 'rm -rf -- "$TMP"' EXIT + +new_repo() { + mkdir -p "$1" + git -C "$1" init -q +} +commit_all() { + git -C "$1" add -A + git -C "$1" -c user.name=t -c user.email=t@t -c commit.gpgsign=false commit -qm fixture +} + +A="$TMP/one-per-tier" +new_repo "$A" +mkdir -p "$A/docs/conventions" "$A/.claude/skills/control-app" +cat >"$A/docs/conventions/harness-config.yaml" <<'EOF' +rule_inventory: + declared: + - rule: "an order is draft or submitted, one status field" + enforcer: "src/order.ts Order type" + rung: make-impossible +EOF +cat >"$A/.claude/settings.json" <<'EOF' +{"permissions": {"deny": ["Read(./.env)"]}} +EOF +printf -- '---\nname: control-app\n---\n\nDrive the app.\n' >"$A/.claude/skills/control-app/SKILL.md" +printf -- '- Never call `eval` in shell scripts.\n' >"$A/AGENTS.md" +printf -- '- Ask a maintainer before changing the release process.\n' >"$A/CONTRIBUTING.md" +commit_all "$A" + +rows="$(bash "$GEN" --root "$A" --rows)" +assert_eq "A: one row per rule" 5 "$(wc -l <<<"$rows" | tr -d ' ')" +assert_line "A: declared codebase row" "$rows" "$(printf 'an order is draft or submitted, one status field\tsrc/order.ts Order type\tmake-impossible\t1')" +assert_line "A: deny row" "$rows" "$(printf 'Read(./.env)\t.claude/settings.json permissions.deny\tharness-setting\t2')" +assert_line "A: control skill row" "$rows" "$(printf 'runtime verification through the control-app skill\t.claude/skills/control-app/SKILL.md\truntime-verification\t3')" +assert_line "A: instruction row" "$rows" "$(printf 'Never call `eval` in shell scripts.\tAGENTS.md:1\tinstruction\t4')" +assert_line "A: human row" "$rows" "$(printf 'Ask a maintainer before changing the release process.\tCONTRIBUTING.md:1\thuman-review\t5')" + +out="$(bash "$GEN" --root "$A")" +assert_line "A: tier 1 count" "$out" "| 1 Codebase | 1 | make-impossible 1 |" +assert_line "A: tier 2 count" "$out" "| 2 Static analysis | 1 | harness-setting 1 |" +assert_line "A: tier 3 count" "$out" "| 3 Verification | 1 | runtime-verification 1 |" +assert_line "A: tier 4 count" "$out" "| 4 Guidance | 1 | instruction 1 |" +assert_line "A: tier 5 count" "$out" "| 5 Human only | 1 | human-review 1 |" +assert_line "A: lower-than total" "$out" "Sits lower than it could: 1 guidance rule(s) forbid a literal that a pattern check could match (candidates to judge, not findings)" +assert_line "A: lower-than entry" "$out" '- AGENTS.md:1: forbids `eval` (instruction)' +assert_no "A: no target wording" "$out" "%" + +B="$TMP/negatives" +new_repo "$B" +mkdir -p "$B/.claude" +printf '{"permissions": {"deny": ["Read(./.env)"]}}\n' >"$B/.claude/settings.json" +cat >"$B/AGENTS.md" <<'EOF' +- Prefer small pull requests that a reviewer can read in one sitting. +- Never read `./.env` directly. +- Never call `sudo` in a script. Enforced by: `bash scripts/check.sh`. +- If a pull request must not merge, apply the `hold` label. + +Keep `README.md` short enough to read in a minute. +EOF +commit_all "$B" +out="$(bash "$GEN" --root "$B")" +assert_line "B: no candidate" "$out" "Sits lower than it could: 0 guidance rule(s) forbid a literal that a pattern check could match (candidates to judge, not findings)" +assert_line "B: every statement counted" "$out" "| 4 Guidance | 5 | instruction 5 |" + +C="$TMP/empty" +new_repo "$C" +out="$(bash "$GEN" --root "$C")" +assert_line "C: tier 1 says why it is empty" "$out" "| 1 Codebase | 0 | none declared; no file shows this tier, so only declared rows count |" +for t in "2 Static analysis" "3 Verification" "4 Guidance" "5 Human only"; do + assert_line "C: $t printed" "$out" "| $t | 0 | none found in the sources below |" +done +assert_line "C: settings absent named" "$out" "- settings: .claude/settings.json absent" +assert_line "C: declared absent named" "$out" "- declared: docs/conventions/harness-config.yaml absent, so no declared rows" +assert_line "C: no ecosystem record named" "$out" "- ecosystems: no .claude/ecosystems/*.yaml record, so no recorded build, test or check command" +assert_line "C: review absent named" "$out" "- review: REVIEW.md absent" + +D="$TMP/records" +new_repo "$D" +mkdir -p "$D/.claude/ecosystems" "$D/docs/conventions" "$D/src" "$D/tests/fixtures" +cat >"$D/.claude/ecosystems/shell.yaml" <<'EOF' +globs: ["*.sh"] +build-cmd: null +test-cmd: "bash run-tests.sh" +check-cmd: | + shellcheck + # a comment line is not a command + shfmt -d +fix-cmd: "shfmt -w " +EOF +cat >"$D/docs/conventions/harness-config.yaml" <<'EOF' +rule_inventory: + declared: + - rule: "no eval of request text" + rung: semgrep-rule + - rule: "a made-up rung" + rung: bogus-rung +EOF +cat >"$D/CLAUDE.md" <<'EOF' +--- +description: frontmatter is not a rule +--- + +# Heading is not a rule + +| a table | is not a rule | +|---|---| + +```bash +- a fenced line is not a rule +``` + + + +@AGENTS.md + +One paragraph that wraps +onto a second line is one rule. +EOF +printf 'x\n' >"$D/src/a.test.js" +printf 'x\n' >"$D/src/b_test.go" +printf -- '- a fixture instruction is not ours\n' >"$D/tests/fixtures/AGENTS.md" +printf 'x\n' >"$D/tests/fixtures/c.test.js" +commit_all "$D" +rows="$(bash "$GEN" --root "$D" --rows)" +assert_line "D: check-cmd line 1" "$rows" "$(printf 'shell check-cmd: shellcheck \t.claude/ecosystems/shell.yaml\tanalyzer-pack-rule\t2')" +assert_line "D: check-cmd line 2" "$rows" "$(printf 'shell check-cmd: shfmt -d \t.claude/ecosystems/shell.yaml\tanalyzer-pack-rule\t2')" +assert_line "D: test-cmd" "$rows" "$(printf 'shell test-cmd: bash run-tests.sh\t.claude/ecosystems/shell.yaml\tbehavior-test\t2')" +assert_no "D: fix-cmd is not a check" "$rows" "fix-cmd" +assert_line "D: alias read as new id" "$rows" "$(printf 'no eval of request text\tdeclared in docs/conventions/harness-config.yaml\tsyntax-pattern-rule\t2')" +assert_no "D: unknown rung not emitted" "$rows" "bogus-rung" +assert_line "D: paragraph joined" "$rows" "$(printf 'One paragraph that wraps onto a second line is one rule.\tCLAUDE.md:19\tinstruction\t4')" +assert_eq "D: one instruction statement only" 1 "$(grep -c " instruction " <<<"$rows")" +assert_eq "D: two test files, fixtures excluded" 2 "$(grep -c "^tests in " <<<"$rows")" +out="$(bash "$GEN" --root "$D")" +assert_line "D: alias notice" "$out" "- declared: item 0 names semgrep-rule, read as syntax-pattern-rule (renamed in ladder 3.0.0)" +assert_line "D: unknown rung named" "$out" '- skipped: rung `bogus-rung` from declared in docs/conventions/harness-config.yaml is not a rung of ladder '"$(awk -F': *' '/^enforcement-ladder:/ {print $2; exit}' "$LADDER")" + +E="$TMP/parity" +new_repo "$E" +mkdir -p "$E/docs/conventions" +expected="make-impossible:1 shared-helper:1 compiler-setting:2 editorconfig-severity:2 analyzer-pack-rule:2 custom-analyzer:2 syntax-pattern-rule:2 architecture-test:2 behavior-test:2 harness-setting:2 action-guard:2 scaffold:2 runtime-verification:3 formal-verification:3 automated-review:4 instruction:4 skill:4 human-review:5" +{ + echo "rule_inventory:" + echo " declared:" + for p in $expected; do + printf ' - rule: "r %s"\n rung: %s\n' "${p%%:*}" "${p%%:*}" + done +} >"$E/docs/conventions/harness-config.yaml" +commit_all "$E" +got="$(bash "$GEN" --root "$E" --rows | awk -F'\t' '{printf "%s%s:%s", sep, $3, $4; sep = " "}')" +assert_eq "parity: every ladder rung maps to its tier" "$expected" "$got" +n_ids="$(grep -Eo '^(\| `[a-z-]+` \||[0-9]+\. \*\*`[a-z-]+`\*\*)' "$LADDER" | wc -l | tr -d ' ')" +assert_eq "parity: the ladder copy holds no other rung id" 18 "$n_ids" + +bash "$GEN" --bogus >/dev/null 2>&1 +assert_eq "unknown argument exits 2" 2 "$?" +bash "$GEN" --root "$TMP/missing" >/dev/null 2>&1 +assert_eq "missing root exits 2" 2 "$?" +RULE_INVENTORY_LADDER="$TMP/no-ladder.md" bash "$GEN" --root "$A" >/dev/null 2>&1 +assert_eq "unreadable ladder exits 2" 2 "$?" + +printf '\n%d case(s), %d failed\n' "$CASES" "$FAILED" +[[ "$FAILED" -eq 0 ]] From bc904dfa8d03d697b32643baa08d31afaf23a05b Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 16 Nov 2026 00:44:25 -0500 Subject: [PATCH 2/8] feat(harness-config): print the ladder spread in the automation-gaps report Present Results now runs rule-inventory.sh and copies its five tier counts and its "sits lower than it could" candidates into the report, read-only and with no target. Candidates go to /corrections:codify, never into this audit's own candidate list. Refs: #15711 Co-Authored-By: Claude Fable 5.2 --- ...711-rule-to-enforcer-inventory-e006297b.md | 16 +++++++++++++ .../skills/audit-automation-gaps/SKILL.md | 24 +++++++++++++++++++ .../templates/checklist.md | 2 +- 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 .changes/harness-config/feat-15711-rule-to-enforcer-inventory-e006297b.md diff --git a/.changes/harness-config/feat-15711-rule-to-enforcer-inventory-e006297b.md b/.changes/harness-config/feat-15711-rule-to-enforcer-inventory-e006297b.md new file mode 100644 index 00000000000..16e0a73052b --- /dev/null +++ b/.changes/harness-config/feat-15711-rule-to-enforcer-inventory-e006297b.md @@ -0,0 +1,16 @@ +--- +bump: minor +--- + +### Added + +- **Ladder spread in `/harness-config:audit-automation-gaps`.** Present Results prints, read-only + and with no target, how many of the repository's rules sit on each of the enforcement ladder's + five tiers, and the guidance rules that forbid a literal a pattern check could match ("sits lower + than it could", candidates for a person to judge). Both are rollups of one rule-to-enforcer + inventory that `scripts/rule-inventory.sh` generates, one row per rule (rule, enforcer, rung, + tier), from the repository's settings, ecosystem records, test files, control skills and + instruction files. Rung ids and tiers come from the ladder copy the skill carries. A source the + script cannot find or read is named, never counted as zero. +- **`rule_inventory.declared` in `harness-config.yaml`.** Rows for rules no file shows, such as a + type that makes the mistake impossible; the schema describes the key. diff --git a/plugins/harness-config/skills/audit-automation-gaps/SKILL.md b/plugins/harness-config/skills/audit-automation-gaps/SKILL.md index 19ec99ffcac..f7fbb39c751 100644 --- a/plugins/harness-config/skills/audit-automation-gaps/SKILL.md +++ b/plugins/harness-config/skills/audit-automation-gaps/SKILL.md @@ -344,6 +344,30 @@ Either way, a time budget the user states is quoted in the user's words. **Order by value/impact**, highest value first. Value = frequency × severity × ease of implementation. +### Ladder spread + +Print how the repository's enforcement spreads across the ladder's tiers, on every run and whatever +the category filter: + +```bash +bash "${CLAUDE_PLUGIN_ROOT}/skills/audit-automation-gaps/scripts/rule-inventory.sh" +``` + +Copy its tier table and its "Sits lower than it could" list into the report as printed, with the +script's line naming what a row counts and every source line that names an absent or unreadable +file. The table holds all five tiers; a tier the script found empty stays in it with its reason. +Both are rollups of one inventory, one row per rule (rule, enforcer, rung, tier), which `--rows` +prints when a reader needs the rows behind a count. + +This section is read-only and sets no target. State no share, ratio or count a tier should reach, +and do not call a tier too high or too low: the ladder's proportion is a direction with no number. +The "Sits lower than it could" entries are candidates for a person to judge, not verdicts. One that +looks worth moving up goes to `/corrections:codify` when the `corrections` plugin is enabled, and +is otherwise named for the human; this audit adds none of them to its candidate list. A rule held +by a mechanism no file shows, such as a type that makes the mistake impossible, is counted only +when the repository declares it under `rule_inventory.declared` in its `harness-config.yaml` +convention file; say so when the codebase tier reads zero. + ### Maturity Assessment End with a one-paragraph assessment of the repo's automation maturity: what's well-covered, the diff --git a/plugins/harness-config/skills/audit-automation-gaps/templates/checklist.md b/plugins/harness-config/skills/audit-automation-gaps/templates/checklist.md index db0dab8d363..449433d66d2 100644 --- a/plugins/harness-config/skills/audit-automation-gaps/templates/checklist.md +++ b/plugins/harness-config/skills/audit-automation-gaps/templates/checklist.md @@ -6,7 +6,7 @@ Copy into your working task notes. Tick as each phase completes. - [ ] Phase 1: Discover candidates (self-generated). Enumerate automation gaps across hooks + MCP + skills + agents + scheduled routines - [ ] Phase 2: Deep-dive each candidate. Evaluate against the rungs in `context/enforcement-ladder.md`; cost-benefit analysis -- [ ] Phase 3: Present results. Categorized recommendations with rationale + anti-noise doctrine compliance +- [ ] Phase 3: Present results. Categorized recommendations with rationale + anti-noise doctrine compliance + the ladder spread (five tier counts, sits lower than it could) - [ ] Phase 4: Implement (if `--implement` or user requests). Apply approved candidates with plan + incremental validation ## Anti-noise doctrine (applied throughout Phase 2) From 711afff6c42d1c0ef9979f98670e98900501ba2c Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 16 Nov 2026 00:47:12 -0500 Subject: [PATCH 3/8] fix(harness-config): count test files that live under a tests directory The instruction filter that skips fixtures, evals and tests directories also dropped every test file kept in tests/. Test files now skip only fixtures and testdata directories. Refs: #15711 Co-Authored-By: Claude Fable 5.2 --- .../skills/audit-automation-gaps/scripts/rule-inventory.sh | 6 +++--- .../audit-automation-gaps/scripts/rule-inventory.test.sh | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh index d55ccf4f77f..e86bbc42d7c 100755 --- a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh +++ b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh @@ -26,8 +26,8 @@ Sources, one adapter each: analyzer-pack-rule, of test-cmd a behavior-test, of build-cmd a compiler-setting test files tracked files named *.test.*, *.spec.*, *_test.*, test_*.* or - *Tests.*, one behavior-test row per file (a unit of files, not - of test cases) + *Tests.*, outside fixtures and testdata directories, one + behavior-test row per file (a unit of files, not of test cases) control .claude/skills/control-*/SKILL.md, a runtime-verification each review statements of REVIEW.md, an automated-review each instruction statements of every tracked CLAUDE.md and AGENTS.md and of @@ -249,7 +249,7 @@ else note "ecosystems: $eco_n record(s) read" fi -tests="$(own_files | grep -E '(^|/)([^/]+\.test\.[^/]+|[^/]+\.spec\.[^/]+|[^/]+_test\.[^/]+|test_[^/]+\.[^/]+|[^/]*[a-z]Tests\.[^/]+)$' || true)" +tests="$(grep -Ev '(^|/)(fixtures|testdata)/' "$work/files" | grep -E '(^|/)([^/]+\.test\.[^/]+|[^/]+\.spec\.[^/]+|[^/]+_test\.[^/]+|test_[^/]+\.[^/]+|[^/]*[a-z]Tests\.[^/]+)$' || true)" if [[ -n "$tests" ]]; then while IFS= read -r t; do emit "tests in $t" "$t" behavior-test; done <<<"$tests" fi diff --git a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh index e94aee59ac4..84dc1c2ab4a 100755 --- a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh +++ b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh @@ -142,6 +142,7 @@ printf 'x\n' >"$D/src/a.test.js" printf 'x\n' >"$D/src/b_test.go" printf -- '- a fixture instruction is not ours\n' >"$D/tests/fixtures/AGENTS.md" printf 'x\n' >"$D/tests/fixtures/c.test.js" +printf 'x\n' >"$D/tests/test_d.py" commit_all "$D" rows="$(bash "$GEN" --root "$D" --rows)" assert_line "D: check-cmd line 1" "$rows" "$(printf 'shell check-cmd: shellcheck \t.claude/ecosystems/shell.yaml\tanalyzer-pack-rule\t2')" @@ -152,7 +153,7 @@ assert_line "D: alias read as new id" "$rows" "$(printf 'no eval of request text assert_no "D: unknown rung not emitted" "$rows" "bogus-rung" assert_line "D: paragraph joined" "$rows" "$(printf 'One paragraph that wraps onto a second line is one rule.\tCLAUDE.md:19\tinstruction\t4')" assert_eq "D: one instruction statement only" 1 "$(grep -c " instruction " <<<"$rows")" -assert_eq "D: two test files, fixtures excluded" 2 "$(grep -c "^tests in " <<<"$rows")" +assert_eq "D: three test files, tests/ kept, fixtures excluded" 3 "$(grep -c "^tests in " <<<"$rows")" out="$(bash "$GEN" --root "$D")" assert_line "D: alias notice" "$out" "- declared: item 0 names semgrep-rule, read as syntax-pattern-rule (renamed in ladder 3.0.0)" assert_line "D: unknown rung named" "$out" '- skipped: rung `bogus-rung` from declared in docs/conventions/harness-config.yaml is not a rung of ladder '"$(awk -F': *' '/^enforcement-ladder:/ {print $2; exit}' "$LADDER")" From 08f355d1c134ad422a95bb5374a79c800ee7c126 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 16 Nov 2026 00:48:55 -0500 Subject: [PATCH 4/8] test(harness-config): add the ag-ladder-spread eval case The quay fixture gains a ladder variant with one rule on each enforcement ladder tier and one guidance rule forbidding a literal no check covers. The case asks only about scheduling, so a report that carries the five tier counts (1, 2, 1, 4, 1) and names the eval rule as sitting lower than it could does so because Present Results says to; it also checks that no tier target is set and that no settings, hook or convention file is written. Refs: #15711 Co-Authored-By: Claude Fable 5.2 --- .../evals/ag-ladder-spread/case.yaml | 36 ++++++++++++ .../evals/ag-ladder-spread/prompt.md | 13 +++++ .../samples/five-tier-counts.json | 26 +++++++++ .../samples/inventory-ran.json | 34 +++++++++++ .../samples/lower-rule-named.json | 22 +++++++ .../samples/no-automation-edit.json | 49 ++++++++++++++++ .../samples/no-automation-write.json | 58 +++++++++++++++++++ .../samples/no-tier-target.json | 18 ++++++ .../evals/ag-ladder-spread/scaffold.sh | 4 ++ .../evals/fixtures/quay-repo.sh | 30 ++++++++++ 10 files changed, 290 insertions(+) create mode 100644 plugins/harness-config/evals/ag-ladder-spread/case.yaml create mode 100644 plugins/harness-config/evals/ag-ladder-spread/prompt.md create mode 100644 plugins/harness-config/evals/ag-ladder-spread/samples/five-tier-counts.json create mode 100644 plugins/harness-config/evals/ag-ladder-spread/samples/inventory-ran.json create mode 100644 plugins/harness-config/evals/ag-ladder-spread/samples/lower-rule-named.json create mode 100644 plugins/harness-config/evals/ag-ladder-spread/samples/no-automation-edit.json create mode 100644 plugins/harness-config/evals/ag-ladder-spread/samples/no-automation-write.json create mode 100644 plugins/harness-config/evals/ag-ladder-spread/samples/no-tier-target.json create mode 100755 plugins/harness-config/evals/ag-ladder-spread/scaffold.sh diff --git a/plugins/harness-config/evals/ag-ladder-spread/case.yaml b/plugins/harness-config/evals/ag-ladder-spread/case.yaml new file mode 100644 index 00000000000..eb7ebd54531 --- /dev/null +++ b/plugins/harness-config/evals/ag-ladder-spread/case.yaml @@ -0,0 +1,36 @@ +schema_version: "1.1" +name: ag-ladder-spread +context: + scaffold_script: scaffold.sh +graders: + - name: 'inventory-ran' + type: 'tool_used' + tool: 'Bash' + input_match: 'rule-inventory\.sh' + min: 1 + - name: 'five-tier-counts' + type: 'regex' + pattern: 'Codebase\D{0,40}\b1\b[\s\S]{0,400}?Static analysis\D{0,40}\b2\b[\s\S]{0,400}?Verification\D{0,40}\b1\b[\s\S]{0,400}?Guidance\D{0,40}\b4\b[\s\S]{0,400}?Human\D{0,40}\b1\b' + target: 'last_message' + - name: 'lower-rule-named' + type: 'regex' + pattern: 'lower than (?:it|they) could[\s\S]{0,500}?\beval\b' + target: 'last_message' + flags: 'i' + - name: 'no-tier-target' + type: 'llm' + criteria: 'PASS only if the reply reports how many rules sit on each enforcement-ladder tier and sets no target for them: it states no share, ratio or count a tier should reach and does not call any tier too high, too low or out of balance. Naming the guidance rule that forbids `eval` as a candidate to move up is allowed. FAIL if the reply gives no per-tier counts, or if it sets or implies a target for a tier, for example "most rules should be static analysis here" or "guidance holds too many rules".' + - name: 'no-automation-write' + type: 'tool_used' + tool: 'Write' + input_match: '"file_path":"(?:[^"\\]|\\.)*?(?:settings(?:\.local)?\.json"|\.mcp\.json"|\.claude/hooks/|\.pre-commit-config\.yaml"|harness-config\.yaml")' + min: 0 + max: 0 + arm: 'both' + - name: 'no-automation-edit' + type: 'tool_used' + tool: 'Edit' + input_match: '"file_path":"(?:[^"\\]|\\.)*?(?:settings(?:\.local)?\.json"|\.mcp\.json"|\.claude/hooks/|\.pre-commit-config\.yaml"|harness-config\.yaml")' + min: 0 + max: 0 + arm: 'both' diff --git a/plugins/harness-config/evals/ag-ladder-spread/prompt.md b/plugins/harness-config/evals/ag-ladder-spread/prompt.md new file mode 100644 index 00000000000..9ee38fccc80 --- /dev/null +++ b/plugins/harness-config/evals/ag-ladder-spread/prompt.md @@ -0,0 +1,13 @@ +--- +description: "Routine. Present Results, Ladder spread: on every run, whatever the category filter, the report carries the rule-inventory.sh tier table (all five tiers) and its 'Sits lower than it could' list, read-only and with no target. The quay ladder variant holds one declared make-impossible rule, a deny rule and a test file, a control skill, four CLAUDE.md statements and one CONTRIBUTING.md rule, so the counts are 1, 2, 1, 4 and 1, and the CLAUDE.md rule forbidding `eval` is the one candidate. The prompt asks only about scheduling, so a report that carries the spread does so because the skill says to. At risk in the sandbox: the skill persists verdicts under ${CLAUDE_PLUGIN_DATA} (Persist the findings), which the sandbox denied on slice A; every grader reads the report or the tool calls before that write. Needs --scaffold --allow-tools Bash Write Edit WebFetch Agent." +tags: [routine, audit-automation-gaps, outcome, ladder-spread, scaffold, plugin-data-risk] +runs: 3 +max_turns: 30 +timeout_seconds: 1800 +allowed_tools: [Skill, Read, Glob, Grep, Bash, Write, Edit, WebFetch, Agent] +expected_outcome: "The report carries the five tier counts 1, 2, 1, 4 and 1 and names the CLAUDE.md rule forbidding `eval` as sitting lower than it could, sets no target for any tier, and writes no settings, hook or convention file." +--- + +/harness-config:audit-automation-gaps scheduled + +Is anything in this repo worth putting on a schedule? diff --git a/plugins/harness-config/evals/ag-ladder-spread/samples/five-tier-counts.json b/plugins/harness-config/evals/ag-ladder-spread/samples/five-tier-counts.json new file mode 100644 index 00000000000..7691f7e9f53 --- /dev/null +++ b/plugins/harness-config/evals/ag-ladder-spread/samples/five-tier-counts.json @@ -0,0 +1,26 @@ +{ + "pass": [ + { + "answer": "## Ladder spread\n\n| Tier | Rules | By rung |\n|---|---|---|\n| 1 Codebase | 1 | make-impossible 1 |\n| 2 Static analysis | 2 | behavior-test 1, harness-setting 1 |\n| 3 Verification | 1 | runtime-verification 1 |\n| 4 Guidance | 4 | instruction 4 |\n| 5 Human only | 1 | human-review 1 |\n\nSits lower than it could: 1 guidance rule(s) forbid a literal that a pattern check could match (candidates to judge, not findings)\n- CLAUDE.md:12: forbids `eval` (instruction)", + "why": "the script's table copied as printed" + }, + { + "answer": "Ladder spread: Codebase 1 (make-impossible), Static analysis 2 (a deny rule and a test file), Verification 1 (control-quay), Guidance 4, Human only 1.", + "why": "the counts restated in prose" + } + ], + "fail": [ + { + "answer": "No scheduled task is worth adding: s1 (nightly test run) is REJECT, CI already runs the tests on every pull request.", + "why": "no spread at all" + }, + { + "answer": "| 1 Codebase | 0 |\n| 2 Static analysis | 2 |\n| 3 Verification | 1 |\n| 4 Guidance | 4 |\n| 5 Human only | 1 |", + "why": "the declared codebase rule is missing" + }, + { + "answer": "| 2 Static analysis | 2 |\n| 3 Verification | 1 |\n| 4 Guidance | 4 |", + "why": "only three tiers" + } + ] +} diff --git a/plugins/harness-config/evals/ag-ladder-spread/samples/inventory-ran.json b/plugins/harness-config/evals/ag-ladder-spread/samples/inventory-ran.json new file mode 100644 index 00000000000..169a4965ab1 --- /dev/null +++ b/plugins/harness-config/evals/ag-ladder-spread/samples/inventory-ran.json @@ -0,0 +1,34 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "bash \"/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh\"", + "description": "Print the ladder spread" + } + } + ], + "why": "the Present Results step" + } + ], + "fail": [ + { + "answer": [], + "why": "no call" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "bash \"/plugins/harness-config/skills/audit-automation-gaps/scripts/inventory.sh\"", + "description": "Automation inventory" + } + } + ], + "why": "the automation inventory, not the rule inventory" + } + ] +} diff --git a/plugins/harness-config/evals/ag-ladder-spread/samples/lower-rule-named.json b/plugins/harness-config/evals/ag-ladder-spread/samples/lower-rule-named.json new file mode 100644 index 00000000000..482358e40f4 --- /dev/null +++ b/plugins/harness-config/evals/ag-ladder-spread/samples/lower-rule-named.json @@ -0,0 +1,22 @@ +{ + "pass": [ + { + "answer": "Sits lower than it could: 1 guidance rule(s) forbid a literal that a pattern check could match (candidates to judge, not findings)\n- CLAUDE.md:12: forbids `eval` (instruction)", + "why": "the script's list copied" + }, + { + "answer": "One rule sits lower than it could: CLAUDE.md says never to call `eval` on an order payload, and a pattern check could match that. It is a candidate for /corrections:codify, not a verdict here.", + "why": "named in prose" + } + ], + "fail": [ + { + "answer": "The tiers hold 1, 2, 1, 4 and 1 rules.", + "why": "no lower-than list" + }, + { + "answer": "Sits lower than it could: none.", + "why": "the eval rule is not named" + } + ] +} diff --git a/plugins/harness-config/evals/ag-ladder-spread/samples/no-automation-edit.json b/plugins/harness-config/evals/ag-ladder-spread/samples/no-automation-edit.json new file mode 100644 index 00000000000..b71dae8efcb --- /dev/null +++ b/plugins/harness-config/evals/ag-ladder-spread/samples/no-automation-edit.json @@ -0,0 +1,49 @@ +{ + "pass": [ + { + "answer": [], + "why": "no call" + }, + { + "answer": [ + { + "tool": "Edit", + "input": { + "file_path": "/ws/.work/checklist.md", + "old_string": "a", + "new_string": "b" + } + } + ], + "why": "the phase checklist" + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Edit", + "input": { + "file_path": "/ws/.claude/settings.json", + "old_string": "a", + "new_string": "b" + } + } + ], + "why": "edits settings" + }, + { + "answer": [ + { + "tool": "Edit", + "input": { + "file_path": "/ws/docs/conventions/harness-config.yaml", + "old_string": "a", + "new_string": "b" + } + } + ], + "why": "edits the convention file" + } + ] +} diff --git a/plugins/harness-config/evals/ag-ladder-spread/samples/no-automation-write.json b/plugins/harness-config/evals/ag-ladder-spread/samples/no-automation-write.json new file mode 100644 index 00000000000..f86b437afa8 --- /dev/null +++ b/plugins/harness-config/evals/ag-ladder-spread/samples/no-automation-write.json @@ -0,0 +1,58 @@ +{ + "pass": [ + { + "answer": [], + "why": "no call" + }, + { + "answer": [ + { + "tool": "Write", + "input": { + "file_path": "/opt/claude/plugins/data/harness-config-melodic-software/findings.json", + "content": "x" + } + } + ], + "why": "the persisted verdicts" + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Write", + "input": { + "file_path": "/ws/.claude/settings.json", + "content": "{}" + } + } + ], + "why": "writes settings" + }, + { + "answer": [ + { + "tool": "Write", + "input": { + "file_path": "/ws/docs/conventions/harness-config.yaml", + "content": "x" + } + } + ], + "why": "declares a rule the audit should only report" + }, + { + "answer": [ + { + "tool": "Write", + "input": { + "file_path": "/ws/.claude/hooks/no-eval.sh", + "content": "x" + } + } + ], + "why": "writes a hook script" + } + ] +} diff --git a/plugins/harness-config/evals/ag-ladder-spread/samples/no-tier-target.json b/plugins/harness-config/evals/ag-ladder-spread/samples/no-tier-target.json new file mode 100644 index 00000000000..2b87d12e2d1 --- /dev/null +++ b/plugins/harness-config/evals/ag-ladder-spread/samples/no-tier-target.json @@ -0,0 +1,18 @@ +{ + "pass": [ + { + "answer": "## Ladder spread\n\n| Tier | Rules | By rung |\n|---|---|---|\n| 1 Codebase | 1 | make-impossible 1 |\n| 2 Static analysis | 2 | behavior-test 1, harness-setting 1 |\n| 3 Verification | 1 | runtime-verification 1 |\n| 4 Guidance | 4 | instruction 4 |\n| 5 Human only | 1 | human-review 1 |\n\nSits lower than it could (candidates to judge, not findings):\n- CLAUDE.md:12: forbids `eval` (instruction). If you want it enforced, /corrections:codify can pick the check.", + "why": "counts and a candidate, no target" + } + ], + "fail": [ + { + "answer": "| 4 Guidance | 4 |\n| 2 Static analysis | 2 |\n\nGuidance holds twice as many rules as static analysis; aim to get static analysis to at least half of all rules.", + "why": "sets a target share" + }, + { + "answer": "No scheduled task is worth adding here.", + "why": "no per-tier counts" + } + ] +} diff --git a/plugins/harness-config/evals/ag-ladder-spread/scaffold.sh b/plugins/harness-config/evals/ag-ladder-spread/scaffold.sh new file mode 100755 index 00000000000..273f2dde162 --- /dev/null +++ b/plugins/harness-config/evals/ag-ladder-spread/scaffold.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# The quay repo with one rule on each enforcement-ladder tier. +set -euo pipefail +bash "$(dirname "${BASH_SOURCE[0]}")/../fixtures/quay-repo.sh" ladder diff --git a/plugins/harness-config/evals/fixtures/quay-repo.sh b/plugins/harness-config/evals/fixtures/quay-repo.sh index 911e0ec0a56..339caf42da5 100755 --- a/plugins/harness-config/evals/fixtures/quay-repo.sh +++ b/plugins/harness-config/evals/fixtures/quay-repo.sh @@ -13,6 +13,8 @@ # postures three project skills: one deletes with no gate, one delegates deletion to a # script that asks first, and one reports and stops # unhobble instruction surfaces of mixed kinds and two hooks, one a policy gate +# ladder a rule on each enforcement-ladder tier, one of them a guidance rule that forbids +# a literal no check covers set -euo pipefail variant="${1:?usage: quay-repo.sh }" @@ -286,6 +288,34 @@ SH } JSON ;; +ladder) + mkdir -p docs/conventions .claude/skills/control-quay + cat >docs/conventions/harness-config.yaml <<'YAML' +rule_inventory: + declared: + - rule: "an order status is one enum field, never two booleans" + enforcer: "src/quay/app.py OrderStatus" + rung: make-impossible +YAML + printf '{\n "permissions": {\n "allow": ["Bash(pytest)"],\n "deny": ["Read(./.env)"]\n }\n}\n' >.claude/settings.json + cat >.claude/skills/control-quay/SKILL.md <<'MD' +--- +name: control-quay +description: Start quay and post a sample order to prove the queue works. Use to verify a change in the running service. +--- + +# Control quay + +Run `python -m quay` and post one order; report the queued status it returns. +MD + cat >>CLAUDE.md <<'MD' + +## Rules + +- Never call `eval` on an order payload. +MD + printf '# Contributing\n\n- Ask the warehouse team before changing the order status names.\n' >CONTRIBUTING.md + ;; *) echo "unknown variant: $variant" >&2 exit 2 From 30b75dc7062abdd342053bd42260ac6a763ad655 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 16 Nov 2026 00:51:11 -0500 Subject: [PATCH 5/8] perf(harness-config): look up each row's tier without a subprocess rule-inventory.sh ran two processes per row to find its tier and trim its text, which took 9.4 s on this repository's 1826 rows. A lookup string and parameter expansion do the same in 0.43 s, with identical output. Refs: #15711 Co-Authored-By: Claude Fable 5.2 --- .../audit-automation-gaps/scripts/rule-inventory.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh index e86bbc42d7c..de3f237096e 100755 --- a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh +++ b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh @@ -97,7 +97,7 @@ rungs="$(awk ' /^\| `[a-z-]+` \|/ { id = $0; sub(/^\| `/, "", id); sub(/`.*/, "", id); print id "\t" tier "\t" name } ' "$ladder")" [[ -n "$rungs" ]] || { echo "rule-inventory.sh: no rung ids found in $ladder" >&2; exit 2; } -tier_of() { awk -F'\t' -v r="$1" '$1 == r {print $2; exit}' <<<"$rungs"; } +rung_map=" $(awk -F'\t' '{printf "%s:%s ", $1, $2}' <<<"$rungs")" work="$(mktemp -d)" || exit 2 trap 'rm -rf -- "$work"' EXIT @@ -107,15 +107,16 @@ notes="$work/notes.txt" : >"$notes" note() { printf '%s\n' "$*" >>"$notes"; } -clean() { tr '\t\r' ' ' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'; } emit() { - local t - t="$(tier_of "$3")" - if [[ -z "$t" ]]; then + local t="${rung_map#* "$3":}" + local text="${1//[$'\t\r']/ }" + if [[ "$t" == "$rung_map" ]]; then note "skipped: rung \`$3\` from $2 is not a rung of ladder $ladder_version" return fi - printf '%s\t%s\t%s\t%s\n' "$(printf '%s' "$1" | clean)" "$2" "$3" "$t" >>"$rows" + t="${t%% *}" + [[ "$text" =~ ^[[:space:]]*(.*[^[:space:]])[[:space:]]*$ ]] && text="${BASH_REMATCH[1]}" + printf '%s\t%s\t%s\t%s\n' "$text" "$2" "$3" "$t" >>"$rows" } if git rev-parse --is-inside-work-tree >/dev/null 2>&1; then From 6dda3b8f41671e3a89aba33bf3485d39432d2cec Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 16 Nov 2026 01:01:50 -0500 Subject: [PATCH 6/8] feat(harness-config): name the sources the rule inventory does not count The inventory listed only what it read or found absent, so a repository whose checks run only in CI or in git hooks read as if its static-analysis count were complete. It now names each source that is present but never counted: CI workflows, lefthook, pre-commit and husky, local settings, and the hooks of plugins the project settings enable. The skill copies those lines into the report, and the README names the rule_inventory key. Refs: #15711 Co-Authored-By: Claude Fable 5.2 --- ...-15711-rule-to-enforcer-inventory-e006297b.md | 3 ++- plugins/harness-config/README.md | 4 +++- .../evals/ag-ladder-spread/scaffold.sh | 1 - .../skills/audit-automation-gaps/SKILL.md | 2 +- .../scripts/rule-inventory.sh | 16 ++++++++++++++++ .../scripts/rule-inventory.test.sh | 15 +++++++++++++++ 6 files changed, 37 insertions(+), 4 deletions(-) diff --git a/.changes/harness-config/feat-15711-rule-to-enforcer-inventory-e006297b.md b/.changes/harness-config/feat-15711-rule-to-enforcer-inventory-e006297b.md index 16e0a73052b..fc357f02ca2 100644 --- a/.changes/harness-config/feat-15711-rule-to-enforcer-inventory-e006297b.md +++ b/.changes/harness-config/feat-15711-rule-to-enforcer-inventory-e006297b.md @@ -11,6 +11,7 @@ bump: minor inventory that `scripts/rule-inventory.sh` generates, one row per rule (rule, enforcer, rung, tier), from the repository's settings, ecosystem records, test files, control skills and instruction files. Rung ids and tiers come from the ladder copy the skill carries. A source the - script cannot find or read is named, never counted as zero. + script cannot find or read is named, never counted as zero, and so is each source present but not + counted: CI workflows, git hook managers, local settings and enabled plugins' hooks. - **`rule_inventory.declared` in `harness-config.yaml`.** Rows for rules no file shows, such as a type that makes the mistake impossible; the schema describes the key. diff --git a/plugins/harness-config/README.md b/plugins/harness-config/README.md index c3a629b865c..60901626c09 100644 --- a/plugins/harness-config/README.md +++ b/plugins/harness-config/README.md @@ -252,7 +252,9 @@ the `verification` level (`off | light | standard | strict`, default `light`, wi `verification_skills`), read from the default branch only by `lib/read-verification.sh`, which decides whether `audit-automation-gaps`, `audit-instructions`, `audit-pass` and `audit-prompting-postures` dispatch their fresh-context verifiers -([levels](../../docs/contracts/verification-level/README.md)). +([levels](../../docs/contracts/verification-level/README.md)). Its `rule_inventory.declared` +list adds rows to the rule-to-enforcer inventory `audit-automation-gaps` prints, for rules no +file shows, such as a type that makes the mistake impossible. Persistent plugin state: `audit-pass` writes its run reports and manifests under `${CLAUDE_PLUGIN_DATA}`, which resolves under `~`, outside a target below the home directory and inside one at or above it. `audit-instructions` keeps its run files under diff --git a/plugins/harness-config/evals/ag-ladder-spread/scaffold.sh b/plugins/harness-config/evals/ag-ladder-spread/scaffold.sh index 273f2dde162..049b6614e6e 100755 --- a/plugins/harness-config/evals/ag-ladder-spread/scaffold.sh +++ b/plugins/harness-config/evals/ag-ladder-spread/scaffold.sh @@ -1,4 +1,3 @@ #!/usr/bin/env bash -# The quay repo with one rule on each enforcement-ladder tier. set -euo pipefail bash "$(dirname "${BASH_SOURCE[0]}")/../fixtures/quay-repo.sh" ladder diff --git a/plugins/harness-config/skills/audit-automation-gaps/SKILL.md b/plugins/harness-config/skills/audit-automation-gaps/SKILL.md index f7fbb39c751..d94af7a4843 100644 --- a/plugins/harness-config/skills/audit-automation-gaps/SKILL.md +++ b/plugins/harness-config/skills/audit-automation-gaps/SKILL.md @@ -355,7 +355,7 @@ bash "${CLAUDE_PLUGIN_ROOT}/skills/audit-automation-gaps/scripts/rule-inventory. Copy its tier table and its "Sits lower than it could" list into the report as printed, with the script's line naming what a row counts and every source line that names an absent or unreadable -file. The table holds all five tiers; a tier the script found empty stays in it with its reason. +file or a source it does not count, such as CI workflows or git hooks. The table holds all five tiers; a tier the script found empty stays in it with its reason. Both are rollups of one inventory, one row per rule (rule, enforcer, rung, tier), which `--rows` prints when a reader needs the rows behind a count. diff --git a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh index de3f237096e..2f88ec04a72 100755 --- a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh +++ b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh @@ -40,6 +40,11 @@ headings, tables, fences, comments and @-imports are not statements. Files under a fixtures, evals, tests or testdata directory are not read as the repository's own instructions. +Never counted, and named as "present, not counted" when present: CI workflows, +git hook managers (lefthook, pre-commit, husky), .claude/settings.local.json, +and the hooks and settings of the plugins the project settings enable. A check +that runs only in one of those is in no row. + Sits lower than it could: a guidance-tier statement in which never, do not, don't or must not is followed, within six words of the same clause, by a backticked literal forbids that literal, which a pattern check could match. @@ -287,6 +292,17 @@ for f in .github/pull_request_template.md .github/PULL_REQUEST_TEMPLATE.md PULL_ done [[ "$human_n" -gt 0 ]] || note "human: no CONTRIBUTING.md and no pull request template in this repository" +wf="$(grep -Ec '^\.github/workflows/[^/]+\.ya?ml$' "$work/files")" +[[ "$wf" -eq 0 ]] || note "present, not counted: $wf CI workflow file(s) under .github/workflows; a check that runs only there is in no row" +for f in lefthook.yml lefthook.yaml .lefthook.yml .pre-commit-config.yaml .husky; do + [[ -e "$f" ]] && note "present, not counted: $f (git hooks); a check that runs only there is in no row" +done +[[ -e .claude/settings.local.json ]] && note "present, not counted: .claude/settings.local.json (one person's settings)" +if [[ -r "$settings" ]] && command -v jq >/dev/null && + jq -e '(.enabledPlugins // {}) | to_entries | any(.value == true)' "$settings" >/dev/null 2>&1; then + note "present, not counted: hooks and settings of the plugins $settings enables" +fi + if [[ "$rows_only" = 1 ]]; then cat "$rows" exit 0 diff --git a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh index 84dc1c2ab4a..dcc77071836 100755 --- a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh +++ b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh @@ -158,6 +158,21 @@ out="$(bash "$GEN" --root "$D")" assert_line "D: alias notice" "$out" "- declared: item 0 names semgrep-rule, read as syntax-pattern-rule (renamed in ladder 3.0.0)" assert_line "D: unknown rung named" "$out" '- skipped: rung `bogus-rung` from declared in docs/conventions/harness-config.yaml is not a rung of ladder '"$(awk -F': *' '/^enforcement-ladder:/ {print $2; exit}' "$LADDER")" +F="$TMP/uncounted" +new_repo "$F" +mkdir -p "$F/.github/workflows" "$F/.claude" +printf 'on: [pull_request]\n' >"$F/.github/workflows/ci.yml" +printf 'pre-commit:\n commands: {}\n' >"$F/lefthook.yml" +printf '{"permissions": {"deny": ["Read(./.env)"]}, "enabledPlugins": {"guard@market": true}}\n' >"$F/.claude/settings.json" +commit_all "$F" +out="$(bash "$GEN" --root "$F")" +assert_line "F: counts unchanged by uncounted sources" "$out" "| 2 Static analysis | 1 | harness-setting 1 |" +assert_line "F: workflows named" "$out" "- present, not counted: 1 CI workflow file(s) under .github/workflows; a check that runs only there is in no row" +assert_line "F: git hooks named" "$out" "- present, not counted: lefthook.yml (git hooks); a check that runs only there is in no row" +assert_line "F: plugin hooks named" "$out" "- present, not counted: hooks and settings of the plugins .claude/settings.json enables" +out="$(bash "$GEN" --root "$C")" +assert_no "C: nothing named as uncounted" "$out" "present, not counted" + E="$TMP/parity" new_repo "$E" mkdir -p "$E/docs/conventions" From 708baaade643774a8fc4cfc7e30aaf18ebd3b82d Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:57:58 -0500 Subject: [PATCH 7/8] fix(harness-config): read no declared rows when the convention home is declined or unresolved rule-inventory.sh read docs/conventions/harness-config.yaml whenever the convention-home resolver exited non-zero, so a declined home or a pointer to a missing directory still counted a leftover file's rows. Exit 1 (no pointer region) keeps the default home; exits 4 and 5 and any failure now read no declared rows and name why, and tier 1 says the source was not read. Co-Authored-By: Claude Opus 5.5 --- ...711-rule-to-enforcer-inventory-e006297b.md | 4 +- .../skills/audit-automation-gaps/SKILL.md | 4 +- .../scripts/rule-inventory.sh | 29 ++++++++++--- .../scripts/rule-inventory.test.sh | 43 +++++++++++++++++++ 4 files changed, 73 insertions(+), 7 deletions(-) diff --git a/.changes/harness-config/feat-15711-rule-to-enforcer-inventory-e006297b.md b/.changes/harness-config/feat-15711-rule-to-enforcer-inventory-e006297b.md index fc357f02ca2..d12306557bf 100644 --- a/.changes/harness-config/feat-15711-rule-to-enforcer-inventory-e006297b.md +++ b/.changes/harness-config/feat-15711-rule-to-enforcer-inventory-e006297b.md @@ -14,4 +14,6 @@ bump: minor script cannot find or read is named, never counted as zero, and so is each source present but not counted: CI workflows, git hook managers, local settings and enabled plugins' hooks. - **`rule_inventory.declared` in `harness-config.yaml`.** Rows for rules no file shows, such as a - type that makes the mistake impossible; the schema describes the key. + type that makes the mistake impossible; the schema describes the key. The file is read from the + convention home, or `docs/conventions` when the repository has no pointer region; a declined home + or one that does not resolve gives no declared rows and is named with the reason. diff --git a/plugins/harness-config/skills/audit-automation-gaps/SKILL.md b/plugins/harness-config/skills/audit-automation-gaps/SKILL.md index d94af7a4843..ea1f2f3d1f5 100644 --- a/plugins/harness-config/skills/audit-automation-gaps/SKILL.md +++ b/plugins/harness-config/skills/audit-automation-gaps/SKILL.md @@ -366,7 +366,9 @@ looks worth moving up goes to `/corrections:codify` when the `corrections` plugi is otherwise named for the human; this audit adds none of them to its candidate list. A rule held by a mechanism no file shows, such as a type that makes the mistake impossible, is counted only when the repository declares it under `rule_inventory.declared` in its `harness-config.yaml` -convention file; say so when the codebase tier reads zero. +convention file; say so when the codebase tier reads zero. A repository that declined a convention +home, or whose home does not resolve, has no declared rows, and the script's declared line says +why: copy it. ### Maturity Assessment diff --git a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh index 2f88ec04a72..e6cb8297988 100755 --- a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh +++ b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh @@ -18,7 +18,9 @@ Usage: Sources, one adapter each: declared rule_inventory.declared in /harness-config.yaml, each item {rule, enforcer, rung}; the only source for rungs no - file shows + file shows. With no pointer region the home is docs/conventions; + a declined home, one that cannot apply, or one that does not + resolve gives no declared rows, named with the reason settings .claude/settings.json: each permissions.deny entry is a harness-setting; each PreToolUse or PermissionRequest handler is an action-guard @@ -166,11 +168,27 @@ read_status() { } home="docs/conventions" -if [[ -r "$resolve_home" ]]; then - h="$(bash "$resolve_home" --root "$root" 2>/dev/null)" && home="$h" +skip="" +if [[ ! -r "$resolve_home" ]]; then + note "declared: no convention-home resolver at $resolve_home, so the default home $home is read" +else + h="$(bash "$resolve_home" --root "$root" 2>"$work/home.err")" + case $? in + 0) home="$h" ;; + 1) ;; + 4) skip="the repository declined a convention home" ;; + 5) skip="no convention home applies to this root" ;; + *) skip="the convention home did not resolve ($(head -n 1 "$work/home.err"))" ;; + esac fi decl="$home/harness-config.yaml" -case "$(read_status "$decl")" in +status="$(read_status "$decl")" +if [[ -n "$skip" ]]; then + [[ "$status" == absent ]] || skip="$skip ($decl is present and not read)" + status=skipped +fi +case "$status" in +skipped) note "declared: not read, so no declared rows: $skip" ;; absent) note "declared: $decl absent, so no declared rows" ;; unreadable) note "declared: $decl unreadable" ;; *) @@ -338,7 +356,8 @@ for t in $(cut -f2 <<<"$rungs" | uniq); do by="$(awk -F'\t' -v t="$t" '$4 == t {c[$3]++} END {for (r in c) print r " " c[r]}' "$rows" | sort | paste -sd, - | sed 's/,/, /g')" if [[ "$n" -eq 0 ]]; then case "$t" in - 1) by="none declared; no file shows this tier, so only declared rows count" ;; + 1) if [[ -n "$skip" ]]; then by="declared rows not read (see the declared source line); no file shows this tier" + else by="none declared; no file shows this tier, so only declared rows count"; fi ;; *) by="none found in the sources below" ;; esac fi diff --git a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh index dcc77071836..5c4b01d657a 100755 --- a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh +++ b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh @@ -17,6 +17,8 @@ assert_line() { if grep -Fxq -- "$3" <<<"$2"; then pass "$1"; else fail "$1" "no $2"; fi; } assert_no() { if grep -Fq -- "$3" <<<"$2"; then fail "$1" "unexpected [$3] in: $2"; else pass "$1"; fi; } +assert_has() { if grep -Fq -- "$3" <<<"$2"; then pass "$1"; else fail "$1" "no [$3] in: +$2"; fi; } TMP="$(mktemp -d)" trap 'rm -rf -- "$TMP"' EXIT @@ -190,6 +192,47 @@ assert_eq "parity: every ladder rung maps to its tier" "$expected" "$got" n_ids="$(grep -Eo '^(\| `[a-z-]+` \||[0-9]+\. \*\*`[a-z-]+`\*\*)' "$LADDER" | wc -l | tr -d ' ')" assert_eq "parity: the ladder copy holds no other rung id" 18 "$n_ids" +home_repo() { + new_repo "$1" + mkdir -p "$1/docs/conventions" + printf 'rule_inventory:\n declared:\n - rule: "stale default-home rule"\n rung: make-impossible\n' >"$1/docs/conventions/harness-config.yaml" + printf '\n%s\n\n' "$2" >"$1/AGENTS.md" +} +TIER1_SKIPPED="| 1 Codebase | 0 | declared rows not read (see the declared source line); no file shows this tier |" + +H="$TMP/home-resolved" +home_repo "$H" 'Team conventions live in `team/conv`.' +mkdir -p "$H/team/conv" +printf 'rule_inventory:\n declared:\n - rule: "bound-home rule"\n rung: make-impossible\n' >"$H/team/conv/harness-config.yaml" +commit_all "$H" +rows="$(bash "$GEN" --root "$H" --rows)" +assert_line "home resolved: its row counted" "$rows" "$(printf 'bound-home rule\tdeclared in team/conv/harness-config.yaml\tmake-impossible\t1')" +assert_no "home resolved: default-home file not read" "$rows" "stale default-home rule" +assert_line "home resolved: source named" "$(bash "$GEN" --root "$H")" "- declared: team/conv/harness-config.yaml read, 1 item(s)" + +H="$TMP/home-declined" +home_repo "$H" 'declined' +commit_all "$H" +out="$(bash "$GEN" --root "$H")" +assert_no "home declined: leftover file not read" "$(bash "$GEN" --root "$H" --rows)" "stale default-home rule" +assert_line "home declined: tier 1 not a plain zero" "$out" "$TIER1_SKIPPED" +assert_line "home declined: named" "$out" "- declared: not read, so no declared rows: the repository declined a convention home (docs/conventions/harness-config.yaml is present and not read)" + +H="$TMP/home-unresolved" +home_repo "$H" 'Team conventions live in `team/missing`.' +commit_all "$H" +out="$(bash "$GEN" --root "$H")" +assert_no "home unresolved: not rebound to docs/conventions" "$(bash "$GEN" --root "$H" --rows)" "stale default-home rule" +assert_line "home unresolved: tier 1 not a plain zero" "$out" "$TIER1_SKIPPED" +assert_has "home unresolved: failure named" "$out" "- declared: not read, so no declared rows: the convention home did not resolve (FAIL: pointer target directory missing: team/missing" + +H="$TMP/home-not-applicable" +mkdir -p "$H/docs/conventions" +printf 'rule_inventory:\n declared:\n - rule: "stale default-home rule"\n rung: make-impossible\n' >"$H/docs/conventions/harness-config.yaml" +out="$(bash "$GEN" --root "$H")" +assert_no "home not applicable: file not read" "$(bash "$GEN" --root "$H" --rows)" "stale default-home rule" +assert_line "home not applicable: named" "$out" "- declared: not read, so no declared rows: no convention home applies to this root (docs/conventions/harness-config.yaml is present and not read)" + bash "$GEN" --bogus >/dev/null 2>&1 assert_eq "unknown argument exits 2" 2 "$?" bash "$GEN" --root "$TMP/missing" >/dev/null 2>&1 From 3d6377b02dc7e6c47dae5ad785f4538d9c9f2b91 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:21:05 -0500 Subject: [PATCH 8/8] fix(harness-config): match a forbidden literal only as a whole token in pattern text The "sits lower than it could" check counted a literal as covered when it appeared anywhere, as a substring, in the rule or enforcer column of a tier-1 or tier-2 row. A forbidden `--force` was hidden by `--force-exclude` in a check command, `settings.json` by a deny row's enforcer label, and `eval` by a test path under `evals/`. The coverage set now holds only text that can carry a pattern: deny entries, a hook's matcher and if value, recorded command lines and declared rule text. A literal counts as covered only when no letter, digit, _, - or . sits directly before or after it. Test file names and enforcer labels are not searched. Co-Authored-By: Claude Opus 5.5 --- .../scripts/rule-inventory.sh | 42 ++++++++++++++----- .../scripts/rule-inventory.test.sh | 28 +++++++++++++ 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh index e6cb8297988..f947b5df506 100755 --- a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh +++ b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.sh @@ -51,8 +51,12 @@ Sits lower than it could: a guidance-tier statement in which never, do not, don't or must not is followed, within six words of the same clause, by a backticked literal forbids that literal, which a pattern check could match. It is listed unless the statement carries "Enforced by:" or the literal -appears in a static-analysis or codebase row. These are candidates for a -person to judge, not findings. +appears as a whole token in the pattern text of a static-analysis or +codebase row: a deny entry, a hook's matcher and if value, a recorded +command line, or a declared rule. A whole token is not preceded or followed +by a letter, digit, _, - or ., so --force is not inside --force-exclude. +Test file names and enforcer labels name a file, not a pattern, and are not +searched. These are candidates for a person to judge, not findings. Exit: 0 once the inventory is produced (a source it could not read is named in the output); 2 for a usage error, a root it cannot enter, or a ladder copy @@ -113,6 +117,8 @@ rows="$work/rows.tsv" notes="$work/notes.txt" : >"$notes" note() { printf '%s\n' "$*" >>"$notes"; } +cover="$work/cover.txt" +: >"$cover" emit() { local t="${rung_map#* "$3":}" @@ -124,6 +130,7 @@ emit() { t="${t%% *}" [[ "$text" =~ ^[[:space:]]*(.*[^[:space:]])[[:space:]]*$ ]] && text="${BASH_REMATCH[1]}" printf '%s\t%s\t%s\t%s\n' "$text" "$2" "$3" "$t" >>"$rows" + [[ -z "${4:-}" || "$t" -gt 2 ]] || printf '%s\n' "${4//[$'\t\r']/ }" >>"$cover" } if git rev-parse --is-inside-work-tree >/dev/null 2>&1; then @@ -210,7 +217,7 @@ unreadable) note "declared: $decl unreadable" ;; if [[ -z "$r" || -z "$g" ]]; then note "declared: item $n in $decl lacks a rule or a rung; skipped" else - emit "$r" "${e:-declared in $decl}" "$g" + emit "$r" "${e:-declared in $decl}" "$g" "$r" fi n=$((n + 1)) done @@ -229,12 +236,13 @@ unreadable) note "settings: $settings unreadable" ;; elif ! jq -e 'type == "object"' "$settings" >/dev/null 2>&1; then note "settings: $settings is not a JSON object" else - while IFS= read -r d; do emit "$d" "$settings permissions.deny" harness-setting; done \ + while IFS= read -r d; do emit "$d" "$settings permissions.deny" harness-setting "$d"; done \ < <(jq -r '(.permissions.deny // [])[] | strings' "$settings") - while IFS= read -r h; do emit "$h" "$settings hooks" action-guard; done \ - < <(jq -r '(.hooks // {}) | to_entries[] | select(.key == "PreToolUse" or .key == "PermissionRequest") + while IFS=$'\t' read -r ev m cond; do + emit "$ev $m${cond:+ if $cond}" "$settings hooks" action-guard "$m${cond:+ $cond}" + done < <(jq -r '(.hooks // {}) | to_entries[] | select(.key == "PreToolUse" or .key == "PermissionRequest") | .key as $ev | (.value // [])[] | (.matcher // "*") as $m | (.hooks // [])[] - | "\($ev) \($m)" + (if .if then " if \(.if)" else "" end)' "$settings") + | [$ev, $m, (.if // "")] | map(tostring | gsub("[\t\r\n]"; " ")) | join("\t")' "$settings") note "settings: $settings read" fi ;; @@ -253,7 +261,7 @@ for f in .claude/ecosystems/*.yaml; do build-cmd) rung=compiler-setting ;; *) continue ;; esac - emit "$name $key: $cmd" "$f" "$rung" + emit "$name $key: $cmd" "$f" "$rung" "$cmd" done < <(awk ' function val(v) { sub(/^[[:space:]]+/, "", v); sub(/[[:space:]]+$/, "", v); return v } block && /^[^[:space:]#]/ { block = "" } @@ -326,7 +334,21 @@ if [[ "$rows_only" = 1 ]]; then exit 0 fi -awk -F'\t' '$4 <= 2 { print $1; print $2 }' "$rows" >"$work/strong.txt" +covered() { + awk ' + function tok(c) { return c ~ /[[:alnum:]_.-]/ } + BEGIN { lit = ARGV[1]; ARGV[1] = ""; n = length(lit) } + { + p = 0 + while ((i = index(substr($0, p + 1), lit)) > 0) { + s = p + i + if (!tok(substr($0, s - 1, s > 1)) && !tok(substr($0, s + n, 1))) { found = 1; exit } + p = s + } + } + END { exit !found } + ' "$1" "$cover" +} lower="$work/lower.txt" : >"$lower" # shellcheck disable=SC2016 # the backticks are Markdown code-span delimiters @@ -339,7 +361,7 @@ while IFS=$'\t' read -r rule enforcer rung tier; do while [[ "$rest" =~ $forbid_re ]]; do lit="${BASH_REMATCH[5]}" rest="${rest#*"${BASH_REMATCH[0]}"}" - grep -Fq -- "$lit" "$work/strong.txt" || open="${open:+$open, }\`$lit\`" + covered "$lit" || open="${open:+$open, }\`$lit\`" done [[ -z "$open" ]] || printf '%s: forbids %s (%s)\n' "$enforcer" "$open" "$rung" >>"$lower" done <"$rows" diff --git a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh index 5c4b01d657a..78d080329d9 100755 --- a/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh +++ b/plugins/harness-config/skills/audit-automation-gaps/scripts/rule-inventory.test.sh @@ -85,6 +85,34 @@ out="$(bash "$GEN" --root "$B")" assert_line "B: no candidate" "$out" "Sits lower than it could: 0 guidance rule(s) forbid a literal that a pattern check could match (candidates to judge, not findings)" assert_line "B: every statement counted" "$out" "| 4 Guidance | 5 | instruction 5 |" +G="$TMP/whole-token" +new_repo "$G" +mkdir -p "$G/.claude/ecosystems" "$G/evals" +printf 'check-cmd: "ruff check --force-exclude "\n' >"$G/.claude/ecosystems/python.yaml" +cat >"$G/.claude/settings.json" <<'EOF' +{"permissions": {"deny": ["Read(./.env)", "Edit(.github/workflows/*)"]}, + "hooks": {"PreToolUse": [{"matcher": "Bash", "hooks": [{"type": "command", "command": "guard.sh", "if": "Bash(git push --no-verify*)"}]}]}} +EOF +printf 'x\n' >"$G/evals/case.test.sh" +cat >"$G/AGENTS.md" <<'EOF' +- Never push with `--force` to a shared branch. +- Never call `eval` in shell scripts. +- Never edit `settings.json` without a review. +- Never edit `.github/workflows` from a session. +- Never pass `--no-verify` to git. +- Do not use `PreToolUse` hooks for formatting. +EOF +commit_all "$G" +assert_line "G: hook row" "$(bash "$GEN" --root "$G" --rows)" "$(printf 'PreToolUse Bash if Bash(git push --no-verify*)\t.claude/settings.json hooks\taction-guard\t2')" +out="$(bash "$GEN" --root "$G")" +assert_line "G: --force not hidden by --force-exclude" "$out" '- AGENTS.md:1: forbids `--force` (instruction)' +assert_line "G: eval not hidden by a test path under evals/" "$out" '- AGENTS.md:2: forbids `eval` (instruction)' +assert_line "G: settings.json not hidden by a deny row's enforcer label" "$out" '- AGENTS.md:3: forbids `settings.json` (instruction)' +assert_no "G: a literal a deny entry holds stays out" "$out" 'forbids `.github/workflows`' +assert_no "G: a literal a hook if value holds stays out" "$out" 'forbids `--no-verify`' +assert_line "G: a hook event name covers nothing" "$out" '- AGENTS.md:6: forbids `PreToolUse` (instruction)' +assert_line "G: four candidates" "$out" "Sits lower than it could: 4 guidance rule(s) forbid a literal that a pattern check could match (candidates to judge, not findings)" + C="$TMP/empty" new_repo "$C" out="$(bash "$GEN" --root "$C")"