From 150b995e7b18aafe5dd96b0e64b48c9ae8ae4815 Mon Sep 17 00:00:00 2001 From: Remi Antczak <140408771+remigiusz-antczak@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:47:57 +0100 Subject: [PATCH 1/4] CI: real-tool smoke for the review_checks.sh hub scanners Add a real-hub job that installs the real tfsec 1.28.11 (sha256-checked), ansible-lint 24.12.2, cfn-lint 1.20.0 and rubocop 1.69.2 and runs scripts/test-review-checks-real-hub.sh: one fictional known-bad canary per tool through review_checks.sh, asserting each is reported as RAN with its expected rule id. bundler-audit, kubescape and retire need a downloaded DB or rule set and stay stub-tested. The job joins the gates aggregate. Tests: 0/0 locally (the 4 real-tool canaries run only in the CI real-hub job; local gates: actionlint, bash -n, size, size-ratchet, fragments) Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 36 ++++++++++++++++++++- changelog.d/hub-real-smoke.md | 2 ++ scripts/test-review-checks-real-hub.sh | 44 ++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 changelog.d/hub-real-smoke.md create mode 100755 scripts/test-review-checks-real-hub.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd845706e..831d36a88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -515,9 +515,43 @@ jobs: - name: Allowlist guard run: REQUIRE_GITLEAKS=1 bash scripts/test_gitleaks_allowlist.sh + # The review_checks.sh stub tests never run a real scanner. This job installs the real tfsec, ansible-lint, rubocop and cfn-lint + # (pinned; no advisory-DB download) and checks each is RAN with its rule id on one canary. bundler-audit, kubescape and retire need a downloaded DB, so they stay stub-only. + real-hub: + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install tfsec 1.28.11 (pinned sha256) + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/bin" + curl -fsSL -o "$RUNNER_TEMP/bin/tfsec" https://github.com/aquasecurity/tfsec/releases/download/v1.28.11/tfsec-linux-amd64 + echo "18e2f947a4f0ec74bf6db07245fc81f83930835ef1b5d51d2d5152a6548b5e45 $RUNNER_TEMP/bin/tfsec" | sha256sum -c - + chmod +x "$RUNNER_TEMP/bin/tfsec" + echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + - name: Install ansible-lint 24.12.2 and cfn-lint 1.20.0 (pinned, venv) + run: | + set -euo pipefail + python3 -m venv "$RUNNER_TEMP/venv" + "$RUNNER_TEMP/venv/bin/pip" install --quiet ansible-lint==24.12.2 cfn-lint==1.20.0 + echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH" + - name: Install rubocop 1.69.2 (pinned, private GEM_HOME) + run: | + set -euo pipefail + gem install --no-document --install-dir "$RUNNER_TEMP/gems" --bindir "$RUNNER_TEMP/gems/bin" rubocop -v 1.69.2 + echo "GEM_HOME=$RUNNER_TEMP/gems" >> "$GITHUB_ENV" + echo "$RUNNER_TEMP/gems/bin" >> "$GITHUB_PATH" + - name: Real-tool canaries + run: REQUIRE_REAL_HUB=1 bash scripts/test-review-checks-real-hub.sh + # Branch protection requires a check named exactly `gates`: it passes only when every shard job succeeded. gates: - needs: [selftests, enforce, perpr, evals, cheap, gitleaks] + needs: [selftests, enforce, perpr, evals, cheap, gitleaks, real-hub] if: always() runs-on: ubuntu-latest timeout-minutes: 5 diff --git a/changelog.d/hub-real-smoke.md b/changelog.d/hub-real-smoke.md new file mode 100644 index 000000000..c95355383 --- /dev/null +++ b/changelog.d/hub-real-smoke.md @@ -0,0 +1,2 @@ +### Added +- CI now runs a `real-hub` job: it installs the real tfsec 1.28.11 (sha256-checked), ansible-lint 24.12.2, cfn-lint 1.20.0 and rubocop 1.69.2, then scripts/test-review-checks-real-hub.sh runs review_checks.sh on one known-bad canary per tool and asserts each is reported as RAN with its expected rule id, so the real flags, output shapes and exit codes are checked and not only the stub behaviour. bundler-audit, kubescape and retire need a downloaded advisory DB or rule set and stay stub-tested. The script skips absent tools unless REQUIRE_REAL_HUB=1. diff --git a/scripts/test-review-checks-real-hub.sh b/scripts/test-review-checks-real-hub.sh new file mode 100755 index 000000000..38c202c1c --- /dev/null +++ b/scripts/test-review-checks-real-hub.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Real-tool smoke for the review_checks.sh hub: runs the REAL tfsec, ansible-lint, rubocop and cfn-lint (no stubs) on one tiny +# known-bad canary each, in a throwaway repo, and asserts each is reported as RAN (stderr "analyzers ran:", no "not run:" line) +# with its expected rule id in the JSON rows. The stub tests (scripts/test-review-checks-hub-*.sh) pin routing and failure +# handling; this pins the real flags, SARIF/JSON shapes and exit codes. Not covered: bundler-audit, kubescape and retire need a +# network-fetched advisory DB or rule set (they are NOT CHECKED without one), so a real run would need a download. +# A tool absent from PATH is skipped (exit 0) unless REQUIRE_REAL_HUB=1 (the ci.yml real-hub job sets it), then it FAILs. +# Read-only on the repo; writes only under a temp dir. Fixtures are fictional canaries. +set -uo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +RC=${REVIEW_CHECKS:-$ROOT/.claude/skills/deep-code-review/scripts/review_checks.sh} +fail=0; tools= +for t in tfsec ansible-lint rubocop cfn-lint; do + if command -v "$t" >/dev/null 2>&1; then tools+=" $t" + elif [ "${REQUIRE_REAL_HUB:-}" = 1 ]; then echo "FAIL - $t not installed (REQUIRE_REAL_HUB=1)"; fail=1 + else echo "skip - $t not installed"; fi +done +[ -n "$tools" ] || { [ "$fail" = 0 ] && echo "no real hub tool installed: SKIP test-review-checks-real-hub"; exit "$fail"; } +T=$(mktemp -d "${TMPDIR:-/tmp}/realhub.XXXXXX") && [ -d "$T" ] || { echo "FAIL - mktemp" >&2; exit 1; } +trap 'cd /; rm -r -- "$T"' EXIT +mkdir "$T/repo" && cd "$T/repo" || exit 1 +G() { git -c user.email=t@example.com -c user.name=t "$@"; } +G init -q . && printf 'x\n' >README.md && G add . && G commit -q -m base && G branch base +mkdir -p infra playbooks app +printf 'resource "aws_s3_bucket" "acme" {\n bucket = "acme-canary"\n}\n' >infra/main.tf +printf -- '---\n- hosts: all\n tasks:\n - ansible.builtin.shell: echo hello\n' >playbooks/site.yml +printf 'eval("1 + 1")\n' >app/canary.rb +printf 'AWSTemplateFormatVersion: "2010-09-09"\nResources:\n Acme:\n Type: AWS::S3::Bucket\n Properties:\n NoSuchProperty: 1\n' >infra/stack.yaml +G add . && G commit -q -m canaries +bash "$RC" --base base >"$T/out" 2>"$T/err"; rc=$? +echo "review_checks exit $rc; stderr tail:"; tail -n 15 "$T/err" +# tool -> rule id regex (rubocop and ansible-lint ids are exact; tfsec and cfn-lint match the rule family, unpinned ids) +want() { case $1 in tfsec) echo '^aws-s3-' ;; ansible-lint) echo '^no-changed-when' ;; rubocop) echo '^Security/Eval$' ;; cfn-lint) echo '^E3' ;; esac; } +for t in $tools; do + re=$(want "$t") + if grep -qE "analyzers ran:.*\b$t\b" "$T/err" && ! grep -q "not run: $t" "$T/err" && ! grep -q "NOT CHECKED: $t" "$T/err" \ + && jq -e --arg t "$t" --arg re "$re" 'any(.[]; .tool==$t and (.rule|test($re)))' "$T/out" >/dev/null; then + echo "ok - $t: RAN, rule matching $re reported" + else + echo "FAIL - $t: not RAN or no rule matching $re; its rows: $(jq -c --arg t "$t" '[.[]|select(.tool==$t)|{file,rule}]' "$T/out" 2>&1 | head -c 400)"; fail=1 + fi +done +[ "$fail" = 0 ] && echo "PASS: test-review-checks-real-hub" +exit "$fail" From ed36fe1438aa3704e2e8d60a9bf2f386eb1ad14f Mon Sep 17 00:00:00 2001 From: Remi Antczak <140408771+remigiusz-antczak@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:50:53 +0100 Subject: [PATCH 2/4] Real-hub smoke: pin canary file per row, strict mode, curl proto Match rows on the tool's canary path as well as rule id, run with set -euo pipefail while still capturing review_checks.sh's exit code, and restrict the tfsec download to https/TLS 1.2. Tests: 0/0 (canaries run in CI) Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- scripts/test-review-checks-real-hub.sh | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 831d36a88..b5b734e38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -530,7 +530,7 @@ jobs: run: | set -euo pipefail mkdir -p "$RUNNER_TEMP/bin" - curl -fsSL -o "$RUNNER_TEMP/bin/tfsec" https://github.com/aquasecurity/tfsec/releases/download/v1.28.11/tfsec-linux-amd64 + curl --proto '=https' --tlsv1.2 -fsSL -o "$RUNNER_TEMP/bin/tfsec" https://github.com/aquasecurity/tfsec/releases/download/v1.28.11/tfsec-linux-amd64 echo "18e2f947a4f0ec74bf6db07245fc81f83930835ef1b5d51d2d5152a6548b5e45 $RUNNER_TEMP/bin/tfsec" | sha256sum -c - chmod +x "$RUNNER_TEMP/bin/tfsec" echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" diff --git a/scripts/test-review-checks-real-hub.sh b/scripts/test-review-checks-real-hub.sh index 38c202c1c..32a08d4d6 100755 --- a/scripts/test-review-checks-real-hub.sh +++ b/scripts/test-review-checks-real-hub.sh @@ -6,7 +6,7 @@ # network-fetched advisory DB or rule set (they are NOT CHECKED without one), so a real run would need a download. # A tool absent from PATH is skipped (exit 0) unless REQUIRE_REAL_HUB=1 (the ci.yml real-hub job sets it), then it FAILs. # Read-only on the repo; writes only under a temp dir. Fixtures are fictional canaries. -set -uo pipefail +set -euo pipefail ROOT=$(cd "$(dirname "$0")/.." && pwd) RC=${REVIEW_CHECKS:-$ROOT/.claude/skills/deep-code-review/scripts/review_checks.sh} fail=0; tools= @@ -27,14 +27,16 @@ printf -- '---\n- hosts: all\n tasks:\n - ansible.builtin.shell: echo hello\ printf 'eval("1 + 1")\n' >app/canary.rb printf 'AWSTemplateFormatVersion: "2010-09-09"\nResources:\n Acme:\n Type: AWS::S3::Bucket\n Properties:\n NoSuchProperty: 1\n' >infra/stack.yaml G add . && G commit -q -m canaries -bash "$RC" --base base >"$T/out" 2>"$T/err"; rc=$? +rc=0; bash "$RC" --base base >"$T/out" 2>"$T/err" || rc=$? echo "review_checks exit $rc; stderr tail:"; tail -n 15 "$T/err" # tool -> rule id regex (rubocop and ansible-lint ids are exact; tfsec and cfn-lint match the rule family, unpinned ids) want() { case $1 in tfsec) echo '^aws-s3-' ;; ansible-lint) echo '^no-changed-when' ;; rubocop) echo '^Security/Eval$' ;; cfn-lint) echo '^E3' ;; esac; } +# tool -> its canary path (a row must be on this file, not just carry the rule id) +cany() { case $1 in tfsec) echo infra/main.tf ;; ansible-lint) echo playbooks/site.yml ;; rubocop) echo app/canary.rb ;; cfn-lint) echo infra/stack.yaml ;; esac; } for t in $tools; do - re=$(want "$t") + re=$(want "$t"); cf=$(cany "$t") if grep -qE "analyzers ran:.*\b$t\b" "$T/err" && ! grep -q "not run: $t" "$T/err" && ! grep -q "NOT CHECKED: $t" "$T/err" \ - && jq -e --arg t "$t" --arg re "$re" 'any(.[]; .tool==$t and (.rule|test($re)))' "$T/out" >/dev/null; then + && jq -e --arg t "$t" --arg re "$re" --arg cf "$cf" 'any(.[]; .tool==$t and (.rule|test($re)) and (.file|tostring|endswith($cf)))' "$T/out" >/dev/null; then echo "ok - $t: RAN, rule matching $re reported" else echo "FAIL - $t: not RAN or no rule matching $re; its rows: $(jq -c --arg t "$t" '[.[]|select(.tool==$t)|{file,rule}]' "$T/out" 2>&1 | head -c 400)"; fail=1 From 76245c2a927c8524ae927647e7ac629793651c25 Mon Sep 17 00:00:00 2001 From: Remi Antczak <140408771+remigiusz-antczak@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:57:42 +0100 Subject: [PATCH 3/4] Real-hub smoke: non-literal rubocop canary, dump raw scanner output on failure Security/Eval ignores a plain string literal argument (cop pattern $!str), so the canary now evals a non-literal. On a failing tool the smoke reruns the hub argv and prints the first 40 lines of raw output so the next CI failure is diagnosable. The ansible-lint hub invocation is unchanged: upstream v24.12.2 prints -f sarif to stdout, so the cause of the missing JSON needs that raw output. Tests: 2/2 stub suites (hub-tf-ansible, hub-rubocop); real-hub runs in CI Co-Authored-By: Claude Opus 5.5 --- scripts/test-review-checks-real-hub.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/test-review-checks-real-hub.sh b/scripts/test-review-checks-real-hub.sh index 32a08d4d6..bca5c292e 100755 --- a/scripts/test-review-checks-real-hub.sh +++ b/scripts/test-review-checks-real-hub.sh @@ -24,7 +24,7 @@ G init -q . && printf 'x\n' >README.md && G add . && G commit -q -m base && G br mkdir -p infra playbooks app printf 'resource "aws_s3_bucket" "acme" {\n bucket = "acme-canary"\n}\n' >infra/main.tf printf -- '---\n- hosts: all\n tasks:\n - ansible.builtin.shell: echo hello\n' >playbooks/site.yml -printf 'eval("1 + 1")\n' >app/canary.rb +printf 'eval(params)\n' >app/canary.rb # Security/Eval skips a plain string literal argument (cop source: pattern $!str), so the argument must not be one printf 'AWSTemplateFormatVersion: "2010-09-09"\nResources:\n Acme:\n Type: AWS::S3::Bucket\n Properties:\n NoSuchProperty: 1\n' >infra/stack.yaml G add . && G commit -q -m canaries rc=0; bash "$RC" --base base >"$T/out" 2>"$T/err" || rc=$? @@ -33,6 +33,9 @@ echo "review_checks exit $rc; stderr tail:"; tail -n 15 "$T/err" want() { case $1 in tfsec) echo '^aws-s3-' ;; ansible-lint) echo '^no-changed-when' ;; rubocop) echo '^Security/Eval$' ;; cfn-lint) echo '^E3' ;; esac; } # tool -> its canary path (a row must be on this file, not just carry the rule id) cany() { case $1 in tfsec) echo infra/main.tf ;; ansible-lint) echo playbooks/site.yml ;; rubocop) echo app/canary.rb ;; cfn-lint) echo infra/stack.yaml ;; esac; } +# the hub's own capture is private to its run: on a failure, rerun the tool's hub argv here and show its raw output head +raw() { case $1 in tfsec) (cd infra && tfsec . --format sarif --no-colour --soft-fail --no-module-downloads) ;; ansible-lint) ansible-lint --offline --nocolor -q -f sarif ./playbooks/site.yml ;; + rubocop) rubocop --force-default-config --fail-level fatal --format json --cache false -- ./app/canary.rb ;; cfn-lint) cfn-lint --format json -- ./infra/stack.yaml ;; esac; } for t in $tools; do re=$(want "$t"); cf=$(cany "$t") if grep -qE "analyzers ran:.*\b$t\b" "$T/err" && ! grep -q "not run: $t" "$T/err" && ! grep -q "NOT CHECKED: $t" "$T/err" \ @@ -40,6 +43,7 @@ for t in $tools; do echo "ok - $t: RAN, rule matching $re reported" else echo "FAIL - $t: not RAN or no rule matching $re; its rows: $(jq -c --arg t "$t" '[.[]|select(.tool==$t)|{file,rule}]' "$T/out" 2>&1 | head -c 400)"; fail=1 + echo "--- raw $t output (first 40 lines, exit via rerun) ---"; { raw "$t" 2>&1 || echo "[exit $?]"; } | head -n 40 || true; echo "--- end raw $t ---" fi done [ "$fail" = 0 ] && echo "PASS: test-review-checks-real-hub" From 852754976a0b69ce6fa0e9a161f6b2758a0f21ff Mon Sep 17 00:00:00 2001 From: Remi Antczak <140408771+remigiusz-antczak@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:02:42 +0100 Subject: [PATCH 4/4] ci: pin ansible-lint 26.11.0 with ansible-core 2.22.0 in real-hub ansible-lint 24.12.2 crashed on import under the runner's Python 3.14 with a floating, newer ansible-core. Pin a compatible pair that both list Python 3.14. Tests: 21/21 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 4 ++-- changelog.d/hub-real-smoke.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5b734e38..ff5b2e601 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -534,11 +534,11 @@ jobs: echo "18e2f947a4f0ec74bf6db07245fc81f83930835ef1b5d51d2d5152a6548b5e45 $RUNNER_TEMP/bin/tfsec" | sha256sum -c - chmod +x "$RUNNER_TEMP/bin/tfsec" echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" - - name: Install ansible-lint 24.12.2 and cfn-lint 1.20.0 (pinned, venv) + - name: Install ansible-lint 26.11.0, ansible-core 2.22.0 and cfn-lint 1.20.0 (pinned, venv) run: | set -euo pipefail python3 -m venv "$RUNNER_TEMP/venv" - "$RUNNER_TEMP/venv/bin/pip" install --quiet ansible-lint==24.12.2 cfn-lint==1.20.0 + "$RUNNER_TEMP/venv/bin/pip" install --quiet ansible-lint==26.11.0 ansible-core==2.22.0 cfn-lint==1.20.0 echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH" - name: Install rubocop 1.69.2 (pinned, private GEM_HOME) run: | diff --git a/changelog.d/hub-real-smoke.md b/changelog.d/hub-real-smoke.md index c95355383..93613ab8a 100644 --- a/changelog.d/hub-real-smoke.md +++ b/changelog.d/hub-real-smoke.md @@ -1,2 +1,2 @@ ### Added -- CI now runs a `real-hub` job: it installs the real tfsec 1.28.11 (sha256-checked), ansible-lint 24.12.2, cfn-lint 1.20.0 and rubocop 1.69.2, then scripts/test-review-checks-real-hub.sh runs review_checks.sh on one known-bad canary per tool and asserts each is reported as RAN with its expected rule id, so the real flags, output shapes and exit codes are checked and not only the stub behaviour. bundler-audit, kubescape and retire need a downloaded advisory DB or rule set and stay stub-tested. The script skips absent tools unless REQUIRE_REAL_HUB=1. +- CI now runs a `real-hub` job: it installs the real tfsec 1.28.11 (sha256-checked), ansible-lint 26.11.0 with ansible-core 2.22.0, cfn-lint 1.20.0 and rubocop 1.69.2, then scripts/test-review-checks-real-hub.sh runs review_checks.sh on one known-bad canary per tool and asserts each is reported as RAN with its expected rule id, so the real flags, output shapes and exit codes are checked and not only the stub behaviour. bundler-audit, kubescape and retire need a downloaded advisory DB or rule set and stay stub-tested. The script skips absent tools unless REQUIRE_REAL_HUB=1.