diff --git a/.claude/skills/deep-code-review/scripts/review_checks.sh b/.claude/skills/deep-code-review/scripts/review_checks.sh index be0175d25..552782ce9 100755 --- a/.claude/skills/deep-code-review/scripts/review_checks.sh +++ b/.claude/skills/deep-code-review/scripts/review_checks.sh @@ -1000,7 +1000,7 @@ hubcmd() { # cargo-audit: --no-fetch never touches the network, --stale accepts an old local DB; exit 1 = vulnerabilities found (also failure: the guard needs a row), 2 = error; the JSON report goes to stdout cargo-audit) cmd=(cargo-audit audit --no-fetch --stale --json --file Cargo.lock --db "${CARGO_HOME:-$HOME/.cargo}/advisory-db"); okrc="1" ;; kube-score) cmd=(kube-score score -o json "$@"); okrc="1" ;; # exit 1 = a critical check failed, but also every error: a crash prints no JSON, so sjson turns it into not-run (renderer/json_v2/api.go: top-level array) - kubescape) jf=$rep; : >"$rep"; cmd=(kubescape scan framework nsa,cis-v1.23-t1.0.1 "$1" --format json --output "$rep" --keep-local --logger error --use-artifacts-from "${KUBESCAPE_ARTIFACTS:-$HOME/.kubescape}") ;; # --keep-local: no report to a backend; --use-artifacts-from: offline frameworks, no download; a crash writes no JSON, so sjson turns it into not-run + kubescape) jf=$rep.json; : >"$jf"; cmd=(kubescape scan framework nsa,cis-v1.23-t1.0.1 "$1" --format json --output "$jf" --keep-local --logger error --use-artifacts-from "${KUBESCAPE_ARTIFACTS:-$HOME/.kubescape}") ;; # the report path ends in .json because real kubescape appends .json to an --output without it (a bare path would stay empty: not-run); --keep-local: no report to a backend; --use-artifacts-from: offline frameworks, no download; a crash writes no JSON, so sjson turns it into not-run retire) jf=$rep; : >"$rep"; cmd=(retire --path . --ignore node_modules --outputformat json --outputpath "$rep" --jsrepo "$REVIEW_RETIRE_JSREPO"); okrc="13" ;; # --jsrepo with a local file: no advisory download; exit 13 = vulnerabilities found (default --exitwith), but the guard below needs a row tflint) cmd=(tflint --format=json --no-color --call-module-type=none "--chdir=${1%/*}" "--filter=${1##*/}"); okrc="2" ;; # exit 2 = issues found, 1 = error (cmd/cli.go); no --init: plugins are never downloaded kube-linter) cmd=(kube-linter lint --format json "$@"); okrc="1" ;; # exit 1 = lint reports (command.go returns "found N lint errors") diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 236fb39d0..3a34c8a52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -515,8 +515,8 @@ jobs: - name: Allowlist guard run: REQUIRE_GITLEAKS=1 bash scripts/test_gitleaks_allowlist.sh - # The review_checks.sh stub tests never run a real scanner. This job installs the real tfsec, ansible-lint, rubocop, cfn-lint, pmd, solhint, hadolint, bandit, ruff, actionlint, zizmor, gosec, staticcheck, kube-linter, flawfinder, detect-secrets, gitleaks, trufflehog, checkov, kube-score, semgrep, njsscan, tflint, kics, cppcheck, devskim, govulncheck and grype - # (pinned, hash-locked where the ecosystem allows; no advisory-DB download, except govulncheck and grype, which fetch their vulnerability DB online) and checks each is RAN with its rule id on one canary. trivy, bundler-audit, kubescape and retire need a downloaded DB or rule set, so they stay stub-only; syft is an inventory the hub never counts as ran. + # The review_checks.sh stub tests never run a real scanner. This job installs the real tfsec, ansible-lint, rubocop, cfn-lint, pmd, solhint, hadolint, bandit, ruff, actionlint, zizmor, gosec, staticcheck, kube-linter, flawfinder, detect-secrets, gitleaks, trufflehog, checkov, kube-score, kubescape, semgrep, njsscan, tflint, kics, cppcheck, devskim, govulncheck and grype + # (pinned, hash-locked where the ecosystem allows; no advisory-DB download, except govulncheck and grype, which fetch their vulnerability DB online) and checks each is RAN with its rule id on one canary. trivy, bundler-audit and retire need a downloaded DB or rule set, so they stay stub-only; syft is an inventory the hub never counts as ran. real-hub: runs-on: ubuntu-24.04 # pinned: the cppcheck .deb below is the noble build and its runtime deps come from noble timeout-minutes: 25 @@ -626,6 +626,20 @@ jobs: curl --proto '=https' --tlsv1.2 -fsSL -O https://github.com/zegl/kube-score/releases/download/v1.20.0/kube-score_1.20.0_linux_amd64.tar.gz echo "ff777b4b3f45905b2d22c69e93c57a7a170e61529a79d54e59cb95cca4bc2a55 kube-score_1.20.0_linux_amd64.tar.gz" | sha256sum -c - tar -xzf kube-score_1.20.0_linux_amd64.tar.gz -C "$RUNNER_TEMP/bin" kube-score + - name: Install kubescape 3.0.48 and its offline frameworks (pinned sha256; the frameworks are regolibrary v1.0.316 assets, which publish no checksums, so their hashes were recorded at pin time) + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/bin" "$RUNNER_TEMP/kubescape-artifacts" + cd "$RUNNER_TEMP" + f() { curl --proto '=https' --tlsv1.2 -fsSL --retry 3 --retry-delay 5 --retry-all-errors -o "$1" "$2"; } + f kubescape_3.0.48_linux_amd64.tar.gz https://github.com/kubescape/kubescape/releases/download/v3.0.48/kubescape_3.0.48_linux_amd64.tar.gz + echo "d7bb9a8faf4f943835c31573f015b0c03386fc038107a53bece9cc4750ddb401 kubescape_3.0.48_linux_amd64.tar.gz" | sha256sum -c - + tar -xzf kubescape_3.0.48_linux_amd64.tar.gz -C "$RUNNER_TEMP/bin" kubescape + f kubescape-artifacts/nsa.json https://github.com/kubescape/regolibrary/releases/download/v1.0.316/nsa + f kubescape-artifacts/cis-v1.23-t1.0.1.json https://github.com/kubescape/regolibrary/releases/download/v1.0.316/cis-v1.23-t1.0.1 + echo "9f927eba6d9bca745dff7cf107f0633e2c25da907e9c1d46edf92f5aa88b9986 kubescape-artifacts/nsa.json" | sha256sum -c - + echo "24b84094cb34e644a547e386f1aadd9af49e34bb0853c946bcfc970d30373c4b kubescape-artifacts/cis-v1.23-t1.0.1.json" | sha256sum -c - + echo "KUBESCAPE_ARTIFACTS=$RUNNER_TEMP/kubescape-artifacts" >> "$GITHUB_ENV" - name: Install tflint 0.66.0 (pinned sha256 from the release checksums.txt; the bundled terraform ruleset, no plugin download) run: | set -euo pipefail diff --git a/changelog.d/hub-real-smoke-8.md b/changelog.d/hub-real-smoke-8.md new file mode 100644 index 000000000..22868c06c --- /dev/null +++ b/changelog.d/hub-real-smoke-8.md @@ -0,0 +1,5 @@ +### Fixed +- `review_checks.sh` now writes the kubescape report to a path ending in `.json` and reads that same path. Real kubescape appends `.json` to an `--output` path that lacks it, so the hub read an empty file and reported kubescape as not run ("unparseable output") on every real run. The stub in `scripts/test-review-checks-hub-k8s-js.sh` now mimics that suffix rule, so the test fails on the old code. + +### Changed +- The real-hub CI job now also runs real kubescape 3.0.48 through `review_checks.sh` on a fictional privileged-pod canary, and fails unless kubescape ran and reported `C-0057` on that file with no not-run or NOT CHECKED line. The binary is pinned by sha256 from the release `checksums.sha256`. The offline `nsa` and `cis-v1.23-t1.0.1` frameworks are regolibrary v1.0.316 release assets, pinned by sha256 recorded at pin time because that release publishes no checksums; the fetches retry up to 3 times. The frameworks are a fixed release, not the latest rules, so rule drift is only picked up when the pin is bumped. diff --git a/scripts/test-review-checks-hub-k8s-js.sh b/scripts/test-review-checks-hub-k8s-js.sh index 64357e4c4..7954972f0 100644 --- a/scripts/test-review-checks-hub-k8s-js.sh +++ b/scripts/test-review-checks-hub-k8s-js.sh @@ -23,6 +23,7 @@ cat >"$T/bin/kubescape" <<'S' echo "kubescape $*" >>"$HUB_LOG" [ "${HUB_MODE:-}" != crash ] || exit 9 t=${4#./}; o=; while [ $# -gt 0 ]; do [ "$1" != --output ] || o=$2; shift; done +case $o in *.json) ;; *) o=$o.json ;; esac # real kubescape appends .json to an --output path that lacks it printf '%s' "${J_ks//@F@/$t}" >"$o" S cat >"$T/bin/retire" <<'S' diff --git a/scripts/test-review-checks-real-hub.sh b/scripts/test-review-checks-real-hub.sh index a0d430860..efca6e673 100755 --- a/scripts/test-review-checks-real-hub.sh +++ b/scripts/test-review-checks-real-hub.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -# Real-tool smoke for the review_checks.sh hub: runs the REAL tfsec, ansible-lint, rubocop, cfn-lint, pmd, solhint, hadolint, bandit, ruff (ruff-security), actionlint, zizmor, gosec, staticcheck, kube-linter, flawfinder, detect-secrets, gitleaks, trufflehog, checkov, kube-score, semgrep (repo rules) and njsscan, tflint, kics, cppcheck, devskim, govulncheck and grype (no stubs) on one tiny +# Real-tool smoke for the review_checks.sh hub: runs the REAL tfsec, ansible-lint, rubocop, cfn-lint, pmd, solhint, hadolint, bandit, ruff (ruff-security), actionlint, zizmor, gosec, staticcheck, kube-linter, flawfinder, detect-secrets, gitleaks, trufflehog, checkov, kube-score, kubescape (pinned local frameworks), semgrep (repo rules) and njsscan, tflint, kics, cppcheck, devskim, govulncheck and grype (no stubs) on one tiny # known-bad canary each, in a throwaway repo, and asserts each is reported as RAN (stderr "analyzers ran:", no "not run:" line) # with its expected rule id in the JSON rows. The stub tests (scripts/test-review-checks-hub-*.sh) pin routing and failure -# handling; this pins the real flags, SARIF/JSON shapes and exit codes. Not covered: syft (an inventory the hub never counts as ran), trivy (its vuln scan needs the downloaded DB, so the hub reports it NOT CHECKED), bundler-audit, kubescape and retire need a +# handling; this pins the real flags, SARIF/JSON shapes and exit codes. Not covered: syft (an inventory the hub never counts as ran), trivy (its vuln scan needs the downloaded DB, so the hub reports it NOT CHECKED), bundler-audit and retire need a # network-fetched advisory DB or rule set (they are NOT CHECKED without one), so a real run would need a download. # A tool absent from PATH is skipped (exit 0) unless REQUIRE_REAL_HUB=1 (the ci.yml real-hub job sets it), then it FAILs. # Read-only on the repo; writes only under a temp dir. Fixtures are fictional canaries. @@ -11,7 +11,7 @@ ROOT=$(cd "$(dirname "$0")/.." && pwd) RC=${REVIEW_CHECKS:-$ROOT/.claude/skills/deep-code-review/scripts/review_checks.sh} fail=0; tools= bin() { [ "$1" = ruff-security ] && echo ruff || echo "$1"; } # the hub tool name ruff-security runs the ruff binary -for t in tfsec ansible-lint rubocop cfn-lint pmd solhint hadolint bandit ruff-security actionlint zizmor gosec staticcheck kube-linter flawfinder detect-secrets gitleaks trufflehog checkov kube-score semgrep njsscan tflint kics cppcheck devskim govulncheck grype; do +for t in tfsec ansible-lint rubocop cfn-lint pmd solhint hadolint bandit ruff-security actionlint zizmor gosec staticcheck kube-linter flawfinder detect-secrets gitleaks trufflehog checkov kube-score kubescape semgrep njsscan tflint kics cppcheck devskim govulncheck grype; do if command -v "$(bin "$t")" >/dev/null 2>&1; then tools+=" $t" elif [ "${REQUIRE_REAL_HUB:-}" = 1 ]; then echo "FAIL - $t not installed (REQUIRE_REAL_HUB=1)"; fail=1 else echo "skip - $t not installed"; fi @@ -36,6 +36,7 @@ printf 'on: push\njobs:\n acme:\n runs-on: ubuntu-latest\n steps:\n printf 'module example.com/canary\n\ngo 1.21\n' >go.mod # stdlib only: gosec and staticcheck run with GOPROXY=off printf 'package canary\n\nimport "crypto/md5"\n\nfunc F(a int) bool {\n\t_ = md5.New()\n\treturn a == a\n}\n' >gocanary/canary.go # gosec G401/G501 (md5), staticcheck SA4000 (identical operands) printf 'apiVersion: v1\nkind: Pod\nmetadata:\n name: acme\nspec:\n containers:\n - name: app\n image: example.com/acme/app:latest\n' >k8s/pod.yaml # kube-linter latest-tag +printf 'apiVersion: v1\nkind: Pod\nmetadata:\n name: acme-ks\nspec:\n containers:\n - name: app\n image: example.com/acme/app:1.0\n securityContext:\n privileged: true\n' >k8s/ks.yaml # kubescape C-0057 (privileged container) printf '#include \nvoid f(char *s) {\n char b[8];\n strcpy(b, s);\n}\n' >app/canary.c pw=Zq7xV2mK9pLw; printf 'url = %s://admin:%s@example.com/x\n' https "$pw" >app/notes.txt # Basic Auth Credentials (detect-secrets); a URL secret, not a provider-key shape the privacy grep bans tk=$(printf 'gh%s_%s' p a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8); printf 'token = %s\n' "$tk" >app/token.cfg # GitHub-token shape (gitleaks, trufflehog), joined at runtime so no source line matches the privacy grep @@ -62,14 +63,14 @@ echo "review_checks exit $rc; stderr tail:"; tail -n 15 "$T/err" want() { case $1 in tfsec) echo '^aws-s3-' ;; ansible-lint) echo '^no-changed-when' ;; rubocop) echo '^Security/Eval$' ;; cfn-lint) echo '^E3' ;; pmd) echo '^(HardCodedCryptoKey|InsecureCryptoIv)$' ;; solhint) echo '^avoid-tx-origin$' ;; hadolint) echo '^DL3007$' ;; bandit) echo '^B602$' ;; ruff-security) echo '^S602$' ;; actionlint) echo '^syntax-check$' ;; zizmor) echo '^unpinned-uses$' ;; gosec) echo '^G(401|501)$' ;; staticcheck) echo '^SA4000$' ;; kube-linter) echo '^latest-tag$' ;; flawfinder) echo '^FF1013$' ;; detect-secrets) echo '^Basic Auth Credentials$' ;; - gitleaks) echo '^github-pat$' ;; trufflehog) echo '^Github$' ;; checkov) echo '^CKV_DOCKER_(3|7)$' ;; kube-score) echo '^container-image-tag$' ;; semgrep) echo '^acme-canary$' ;; njsscan) echo '^eval_nodejs$' ;; + gitleaks) echo '^github-pat$' ;; trufflehog) echo '^Github$' ;; checkov) echo '^CKV_DOCKER_(3|7)$' ;; kube-score) echo '^container-image-tag$' ;; kubescape) echo '^C-0057$' ;; semgrep) echo '^acme-canary$' ;; njsscan) echo '^eval_nodejs$' ;; tflint) echo '^terraform_unused_declarations$' ;; kics) echo "^Image Version Using 'latest'\$" ;; cppcheck) echo '^arrayIndexOutOfBounds$' ;; devskim) echo '^DS126858$' ;; govulncheck) echo '^GO-(2020-0036|2021-0061|2022-0956)$' ;; grype) echo '^(GHSA-(35jh-r3h4-6jhm|p6mc-m468-83gw)|CVE-2021-23337|CVE-2020-8203)$' ;; esac; } # tool -> its canary path (a row must be on this file, not just carry the rule id) cany() { case $1 in tfsec) echo infra/main.tf ;; ansible-lint) echo playbooks/site.yml ;; rubocop) echo app/canary.rb ;; cfn-lint) echo infra/stack.yaml ;; pmd) echo app/Canary.java ;; solhint) echo contracts/Canary.sol ;; hadolint) echo docker/Dockerfile ;; bandit|ruff-security) echo app/canary.py ;; actionlint) echo .github/workflows/canary.yml ;; zizmor) echo .github/workflows/zcanary.yml ;; gosec|staticcheck) echo gocanary/canary.go ;; kube-linter) echo k8s/pod.yaml ;; flawfinder) echo app/canary.c ;; detect-secrets) echo app/notes.txt ;; gitleaks|trufflehog) echo app/token.cfg ;; - checkov) echo docker/Dockerfile ;; kube-score) echo k8s/pod.yaml ;; semgrep) echo app/sg.txt ;; njsscan) echo app/canary.js ;; + checkov) echo docker/Dockerfile ;; kube-score) echo k8s/pod.yaml ;; kubescape) echo k8s/ks.yaml ;; semgrep) echo app/sg.txt ;; njsscan) echo app/canary.js ;; tflint) echo tfl/main.tf ;; kics) echo docker/Dockerfile ;; cppcheck) echo app/cpp_canary.c ;; devskim) echo app/ds_canary.py ;; govulncheck) echo go.mod ;; grype) echo sbom/bom.json ;; esac; } # the hub's own capture is private to its run: on a failure, rerun the tool's hub argv here and show its raw output head raw() { case $1 in tfsec) (cd infra && tfsec . --format sarif --no-colour --soft-fail --no-module-downloads) ;; ansible-lint) ansible-lint --offline --nocolor -q -f sarif ./playbooks/site.yml ;; @@ -80,7 +81,7 @@ raw() { case $1 in tfsec) (cd infra && tfsec . --format sarif --no-colour --soft zizmor) zizmor --format=json --offline ./.github/workflows/zcanary.yml ;; gosec) GOPROXY=off gosec -fmt=json -no-fail ./... ;; staticcheck) GOPROXY=off GOTOOLCHAIN=local GOFLAGS=-mod=readonly GOWORK=off staticcheck -f json ./... ;; kube-linter) kube-linter lint --format json ./k8s/pod.yaml ;; flawfinder) flawfinder --sarif --quiet ./app/canary.c ;; detect-secrets) detect-secrets scan --no-verify ./app/notes.txt ;; gitleaks) gitleaks detect --no-git --no-banner --exit-code 0 -f json -r /dev/stdout -s ./app/token.cfg ;; trufflehog) trufflehog filesystem --no-verification --no-update --sarif ./app/token.cfg ;; - checkov) CKV_SKIP_PACKAGE_UPDATE_CHECK=true checkov -f ./docker/Dockerfile -o json --quiet --skip-download --soft-fail ;; kube-score) kube-score score -o json ./k8s/pod.yaml ;; + checkov) CKV_SKIP_PACKAGE_UPDATE_CHECK=true checkov -f ./docker/Dockerfile -o json --quiet --skip-download --soft-fail ;; kube-score) kube-score score -o json ./k8s/pod.yaml ;; kubescape) kubescape scan framework nsa,cis-v1.23-t1.0.1 ./k8s/ks.yaml --format json --output "$T/ks-raw.json" --keep-local --logger error --use-artifacts-from "${KUBESCAPE_ARTIFACTS:-$HOME/.kubescape}" >/dev/null && cat "$T/ks-raw.json" ;; semgrep) HOME="$T" semgrep --config .semgrep.yml --metrics=off --disable-version-check --json --quiet ./app/sg.txt ;; njsscan) njsscan --json ./app/canary.js ;; tflint) tflint --format=json --no-color --call-module-type=none --chdir=./tfl --filter=main.tf ;; kics) kics scan -p ./docker/Dockerfile --no-progress --disable-secrets --report-formats json -o "$T/kics-raw"; cat "$T/kics-raw/results.json" ;; cppcheck) cppcheck --xml-version=2 --quiet ./app/cpp_canary.c ;; devskim) devskim analyze --source-code ./app/ds_canary.py --file-format sarif ;;