CI: real-tool smoke for review_checks.sh hub scanners

.github/workflows/ci.yml

@@ -515,9 +515,43 @@ jobs:
       - name: Allowlist guard
         run: REQUIRE_GITLEAKS=1 bash scripts/test_gitleaks_allowlist.sh
 
+  # The review_checks.sh stub tests never run a real scanner. This job installs the real tfsec, ansible-lint, rubocop and cfn-lint
+  # (pinned; no advisory-DB download) and checks each is RAN with its rule id on one canary. bundler-audit, kubescape and retire need a downloaded DB, so they stay stub-only.
+  real-hub:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Install tfsec 1.28.11 (pinned sha256)
+        run: |
+          set -euo pipefail
+          mkdir -p "$RUNNER_TEMP/bin"
+          curl --proto '=https' --tlsv1.2 -fsSL -o "$RUNNER_TEMP/bin/tfsec" https://github.com/aquasecurity/tfsec/releases/download/v1.28.11/tfsec-linux-amd64
+          echo "18e2f947a4f0ec74bf6db07245fc81f83930835ef1b5d51d2d5152a6548b5e45  $RUNNER_TEMP/bin/tfsec" | sha256sum -c -
+          chmod +x "$RUNNER_TEMP/bin/tfsec"
+          echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH"
+      - name: Install ansible-lint 26.11.0, ansible-core 2.22.0 and cfn-lint 1.20.0 (pinned, venv)
+        run: |
+          set -euo pipefail
+          python3 -m venv "$RUNNER_TEMP/venv"
+          "$RUNNER_TEMP/venv/bin/pip" install --quiet ansible-lint==26.11.0 ansible-core==2.22.0 cfn-lint==1.20.0
+          echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH"
+      - name: Install rubocop 1.69.2 (pinned, private GEM_HOME)
+        run: |
+          set -euo pipefail
+          gem install --no-document --install-dir "$RUNNER_TEMP/gems" --bindir "$RUNNER_TEMP/gems/bin" rubocop -v 1.69.2
+          echo "GEM_HOME=$RUNNER_TEMP/gems" >> "$GITHUB_ENV"
+          echo "$RUNNER_TEMP/gems/bin" >> "$GITHUB_PATH"
+      - name: Real-tool canaries
+        run: REQUIRE_REAL_HUB=1 bash scripts/test-review-checks-real-hub.sh
+
   # Branch protection requires a check named exactly `gates`: it passes only when every shard job succeeded.
   gates:
-    needs: [selftests, enforce, perpr, evals, cheap, gitleaks]
+    needs: [selftests, enforce, perpr, evals, cheap, gitleaks, real-hub]
     if: always()
     runs-on: ubuntu-latest
     timeout-minutes: 5

changelog.d/hub-real-smoke.md

@@ -0,0 +1,2 @@
+### Added
+- CI now runs a `real-hub` job: it installs the real tfsec 1.28.11 (sha256-checked), ansible-lint 26.11.0 with ansible-core 2.22.0, cfn-lint 1.20.0 and rubocop 1.69.2, then scripts/test-review-checks-real-hub.sh runs review_checks.sh on one known-bad canary per tool and asserts each is reported as RAN with its expected rule id, so the real flags, output shapes and exit codes are checked and not only the stub behaviour. bundler-audit, kubescape and retire need a downloaded advisory DB or rule set and stay stub-tested. The script skips absent tools unless REQUIRE_REAL_HUB=1.

scripts/test-review-checks-real-hub.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Real-tool smoke for the review_checks.sh hub: runs the REAL tfsec, ansible-lint, rubocop and cfn-lint (no stubs) on one tiny
+# known-bad canary each, in a throwaway repo, and asserts each is reported as RAN (stderr "analyzers ran:", no "not run:" line)
+# with its expected rule id in the JSON rows. The stub tests (scripts/test-review-checks-hub-*.sh) pin routing and failure
+# handling; this pins the real flags, SARIF/JSON shapes and exit codes. Not covered: bundler-audit, kubescape and retire need a
+# network-fetched advisory DB or rule set (they are NOT CHECKED without one), so a real run would need a download.
+# A tool absent from PATH is skipped (exit 0) unless REQUIRE_REAL_HUB=1 (the ci.yml real-hub job sets it), then it FAILs.
+# Read-only on the repo; writes only under a temp dir. Fixtures are fictional canaries.
+set -euo pipefail
+ROOT=$(cd "$(dirname "$0")/.." && pwd)
+RC=${REVIEW_CHECKS:-$ROOT/.claude/skills/deep-code-review/scripts/review_checks.sh}
+fail=0; tools=
+for t in tfsec ansible-lint rubocop cfn-lint; do
+  if command -v "$t" >/dev/null 2>&1; then tools+=" $t"
+  elif [ "${REQUIRE_REAL_HUB:-}" = 1 ]; then echo "FAIL - $t not installed (REQUIRE_REAL_HUB=1)"; fail=1
+  else echo "skip - $t not installed"; fi
+done
+[ -n "$tools" ] || { [ "$fail" = 0 ] && echo "no real hub tool installed: SKIP test-review-checks-real-hub"; exit "$fail"; }
+T=$(mktemp -d "${TMPDIR:-/tmp}/realhub.XXXXXX") && [ -d "$T" ] || { echo "FAIL - mktemp" >&2; exit 1; }
+trap 'cd /; rm -r -- "$T"' EXIT
+mkdir "$T/repo" && cd "$T/repo" || exit 1
+G() { git -c user.email=t@example.com -c user.name=t "$@"; }
+G init -q . && printf 'x\n' >README.md && G add . && G commit -q -m base && G branch base
+mkdir -p infra playbooks app
+printf 'resource "aws_s3_bucket" "acme" {\n  bucket = "acme-canary"\n}\n' >infra/main.tf
+printf -- '---\n- hosts: all\n  tasks:\n    - ansible.builtin.shell: echo hello\n' >playbooks/site.yml
+printf 'eval(params)\n' >app/canary.rb  # Security/Eval skips a plain string literal argument (cop source: pattern $!str), so the argument must not be one
+printf 'AWSTemplateFormatVersion: "2010-09-09"\nResources:\n  Acme:\n    Type: AWS::S3::Bucket\n    Properties:\n      NoSuchProperty: 1\n' >infra/stack.yaml
+G add . && G commit -q -m canaries
+rc=0; bash "$RC" --base base >"$T/out" 2>"$T/err" || rc=$?
+echo "review_checks exit $rc; stderr tail:"; tail -n 15 "$T/err"
+# tool -> rule id regex (rubocop and ansible-lint ids are exact; tfsec and cfn-lint match the rule family, unpinned ids)
+want() { case $1 in tfsec) echo '^aws-s3-' ;; ansible-lint) echo '^no-changed-when' ;; rubocop) echo '^Security/Eval$' ;; cfn-lint) echo '^E3' ;; esac; }
+# tool -> its canary path (a row must be on this file, not just carry the rule id)
+cany() { case $1 in tfsec) echo infra/main.tf ;; ansible-lint) echo playbooks/site.yml ;; rubocop) echo app/canary.rb ;; cfn-lint) echo infra/stack.yaml ;; esac; }
+# the hub's own capture is private to its run: on a failure, rerun the tool's hub argv here and show its raw output head
+raw() { case $1 in tfsec) (cd infra && tfsec . --format sarif --no-colour --soft-fail --no-module-downloads) ;; ansible-lint) ansible-lint --offline --nocolor -q -f sarif ./playbooks/site.yml ;;
+  rubocop) rubocop --force-default-config --fail-level fatal --format json --cache false -- ./app/canary.rb ;; cfn-lint) cfn-lint --format json -- ./infra/stack.yaml ;; esac; }
+for t in $tools; do
+  re=$(want "$t"); cf=$(cany "$t")
+  if grep -qE "analyzers ran:.*\b$t\b" "$T/err" && ! grep -q "not run: $t" "$T/err" && ! grep -q "NOT CHECKED: $t" "$T/err" \
+     && jq -e --arg t "$t" --arg re "$re" --arg cf "$cf" 'any(.[]; .tool==$t and (.rule|test($re)) and (.file|tostring|endswith($cf)))' "$T/out" >/dev/null; then
+    echo "ok - $t: RAN, rule matching $re reported"
+  else
+    echo "FAIL - $t: not RAN or no rule matching $re; its rows: $(jq -c --arg t "$t" '[.[]|select(.tool==$t)|{file,rule}]' "$T/out" 2>&1 | head -c 400)"; fail=1
+    echo "--- raw $t output (first 40 lines, exit via rerun) ---"; { raw "$t" 2>&1 || echo "[exit $?]"; } | head -n 40 || true; echo "--- end raw $t ---"
+  fi
+done
+[ "$fail" = 0 ] && echo "PASS: test-review-checks-real-hub"
+exit "$fail"