diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc87de837..236fb39d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -515,11 +515,11 @@ jobs: - name: Allowlist guard run: REQUIRE_GITLEAKS=1 bash scripts/test_gitleaks_allowlist.sh - # The review_checks.sh stub tests never run a real scanner. This job installs the real tfsec, ansible-lint, rubocop, cfn-lint, pmd, solhint, hadolint, bandit, ruff, actionlint, zizmor, gosec, staticcheck, kube-linter, flawfinder, detect-secrets, gitleaks, trufflehog, checkov, kube-score, semgrep, njsscan, tflint, kics, cppcheck and devskim - # (pinned, hash-locked where the ecosystem allows; no advisory-DB download) and checks each is RAN with its rule id on one canary. trivy, bundler-audit, kubescape and retire need a downloaded DB or rule set, so they stay stub-only; syft is an inventory the hub never counts as ran. + # The review_checks.sh stub tests never run a real scanner. This job installs the real tfsec, ansible-lint, rubocop, cfn-lint, pmd, solhint, hadolint, bandit, ruff, actionlint, zizmor, gosec, staticcheck, kube-linter, flawfinder, detect-secrets, gitleaks, trufflehog, checkov, kube-score, semgrep, njsscan, tflint, kics, cppcheck, devskim, govulncheck and grype + # (pinned, hash-locked where the ecosystem allows; no advisory-DB download, except govulncheck and grype, which fetch their vulnerability DB online) and checks each is RAN with its rule id on one canary. trivy, bundler-audit, kubescape and retire need a downloaded DB or rule set, so they stay stub-only; syft is an inventory the hub never counts as ran. real-hub: runs-on: ubuntu-24.04 # pinned: the cppcheck .deb below is the noble build and its runtime deps come from noble - timeout-minutes: 15 + timeout-minutes: 25 permissions: contents: read steps: @@ -670,6 +670,27 @@ jobs: if dotnet nuget verify --help >/dev/null 2>&1; then dotnet nuget verify "$RUNNER_TEMP/nupkg/microsoft.cst.devskim.cli.1.0.100.nupkg"; fi dotnet tool install Microsoft.CST.DevSkim.CLI --version 1.0.100 --tool-path "$RUNNER_TEMP/devskim" --add-source "$RUNNER_TEMP/nupkg" echo "$RUNNER_TEMP/devskim" >> "$GITHUB_PATH" + - name: Install govulncheck 1.9.0 (source build on the pinned Go 1.26.11; module hash pinned, checksum DB verified; the vuln DB vuln.go.dev is fetched online at run time, not pinned) + env: + GOTOOLCHAIN: local + GOBIN: ${{ runner.temp }}/bin + run: | + set -euo pipefail + # bounded retry (3 tries, 10s then 20s backoff) for a transient proxy error + for i in 1 2 3; do if go install golang.org/x/vuln/cmd/govulncheck@v1.9.0; then break; fi; [ "$i" -lt 3 ] || exit 1; sleep $((i * 10)); done + go version -m "$RUNNER_TEMP/bin/govulncheck" | grep -qE "golang.org/x/vuln[[:space:]]+v1.9.0[[:space:]]+h1:12VaqnI9kJfqwRoPptxoqsUhQtdfrSbdksT7X/KY/sc=" + - name: Install grype 0.121.3 (pinned sha256 from the release checksums file; vulnerability DB fetched online, checksum-verified by grype, not pinned) + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/bin" + cd "$RUNNER_TEMP" + curl --proto '=https' --tlsv1.2 -fsSL -O https://github.com/anchore/grype/releases/download/v0.121.3/grype_0.121.3_checksums.txt + curl --proto '=https' --tlsv1.2 -fsSL -O https://github.com/anchore/grype/releases/download/v0.121.3/grype_0.121.3_linux_amd64.tar.gz + test "$(grep -cF ' grype_0.121.3_linux_amd64.tar.gz' grype_0.121.3_checksums.txt)" = 1 + grep -F ' grype_0.121.3_linux_amd64.tar.gz' grype_0.121.3_checksums.txt | sha256sum -c - + echo "e9da18cc1429e786722e32161288103600efffd8e483c90761f90485bd1bec85 grype_0.121.3_linux_amd64.tar.gz" | sha256sum -c - + tar -xzf grype_0.121.3_linux_amd64.tar.gz -C "$RUNNER_TEMP/bin" grype + for i in 1 2 3; do if GRYPE_CHECK_FOR_APP_UPDATE=false "$RUNNER_TEMP/bin/grype" db update; then break; fi; [ "$i" -lt 3 ] || exit 1; sleep $((i * 10)); done - name: Real-tool canaries run: REQUIRE_REAL_HUB=1 bash scripts/test-review-checks-real-hub.sh diff --git a/changelog.d/hub-real-smoke-7.md b/changelog.d/hub-real-smoke-7.md new file mode 100644 index 000000000..9929a5648 --- /dev/null +++ b/changelog.d/hub-real-smoke-7.md @@ -0,0 +1,4 @@ +### Changed +- The real-hub CI job now also runs real govulncheck 1.9.0 and grype 0.121.3 through `review_checks.sh`, each on a fictional canary; the job fails unless the tool ran and reported its expected rule on that file with no not-run or NOT CHECKED line. grype is pinned by sha256 from the release `grype_0.121.3_checksums.txt` (the linux_amd64 archive is also checked against a literal hash). govulncheck ships no release binaries, so it is built from source with `go install` on the pinned Go 1.26.11 (`GOTOOLCHAIN=local`), the module hash `h1:` is asserted from `go version -m`, and the Go checksum database verifies the source. The canaries: govulncheck calls `yaml.Unmarshal` of the real module gopkg.in/yaml.v2 2.2.2 (GO-2020-0036, GO-2021-0061, GO-2022-0956; `go mod tidy` fills `go.sum` and the module cache, then the hub runs with `GOPROXY=off`); grype scans a one-component CycloneDX SBOM (lodash 4.17.15, GHSA-35jh-r3h4-6jhm), because the hub points grype at SBOM files only, never at a lockfile. +- Both vulnerability databases are fetched online and are not pinned: govulncheck's `-db` flag has no env override, so it reads vuln.go.dev at run time (no offline pin is possible without changing the hub command), and the CI step runs `grype db update` (grype verifies the archive against the listing checksum; a dated archive pin would rot when upstream rotates it out). The canary advisories are old, so a DB refresh does not remove them. +- Follow-up: the real-hub job timeout is 25 minutes, the govulncheck install, `grype db update`, `go mod tidy` and the hub run (while govulncheck or grype is not run) each retry up to 3 times with backoff, and the grype canary also accepts the CVE aliases of its advisories. The canary `go.mod` stays at the repo root because the hub runs govulncheck only from a root `go.mod`; the test asserts gosec and staticcheck report nothing on the govulncheck canary package. diff --git a/scripts/test-review-checks-real-hub.sh b/scripts/test-review-checks-real-hub.sh index 11059c2b8..a0d430860 100755 --- a/scripts/test-review-checks-real-hub.sh +++ b/scripts/test-review-checks-real-hub.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Real-tool smoke for the review_checks.sh hub: runs the REAL tfsec, ansible-lint, rubocop, cfn-lint, pmd, solhint, hadolint, bandit, ruff (ruff-security), actionlint, zizmor, gosec, staticcheck, kube-linter, flawfinder, detect-secrets, gitleaks, trufflehog, checkov, kube-score, semgrep (repo rules) and njsscan, tflint, kics, cppcheck and devskim (no stubs) on one tiny +# Real-tool smoke for the review_checks.sh hub: runs the REAL tfsec, ansible-lint, rubocop, cfn-lint, pmd, solhint, hadolint, bandit, ruff (ruff-security), actionlint, zizmor, gosec, staticcheck, kube-linter, flawfinder, detect-secrets, gitleaks, trufflehog, checkov, kube-score, semgrep (repo rules) and njsscan, tflint, kics, cppcheck, devskim, govulncheck and grype (no stubs) on one tiny # known-bad canary each, in a throwaway repo, and asserts each is reported as RAN (stderr "analyzers ran:", no "not run:" line) # with its expected rule id in the JSON rows. The stub tests (scripts/test-review-checks-hub-*.sh) pin routing and failure # handling; this pins the real flags, SARIF/JSON shapes and exit codes. Not covered: syft (an inventory the hub never counts as ran), trivy (its vuln scan needs the downloaded DB, so the hub reports it NOT CHECKED), bundler-audit, kubescape and retire need a @@ -11,7 +11,7 @@ ROOT=$(cd "$(dirname "$0")/.." && pwd) RC=${REVIEW_CHECKS:-$ROOT/.claude/skills/deep-code-review/scripts/review_checks.sh} fail=0; tools= bin() { [ "$1" = ruff-security ] && echo ruff || echo "$1"; } # the hub tool name ruff-security runs the ruff binary -for t in tfsec ansible-lint rubocop cfn-lint pmd solhint hadolint bandit ruff-security actionlint zizmor gosec staticcheck kube-linter flawfinder detect-secrets gitleaks trufflehog checkov kube-score semgrep njsscan tflint kics cppcheck devskim; do +for t in tfsec ansible-lint rubocop cfn-lint pmd solhint hadolint bandit ruff-security actionlint zizmor gosec staticcheck kube-linter flawfinder detect-secrets gitleaks trufflehog checkov kube-score semgrep njsscan tflint kics cppcheck devskim govulncheck grype; do if command -v "$(bin "$t")" >/dev/null 2>&1; then tools+=" $t" elif [ "${REQUIRE_REAL_HUB:-}" = 1 ]; then echo "FAIL - $t not installed (REQUIRE_REAL_HUB=1)"; fail=1 else echo "skip - $t not installed"; fi @@ -45,21 +45,32 @@ printf "var express = require('express');\nvar app = express();\napp.get('/', fu printf 'variable "acme_unused" {\n type = string\n}\n' >tfl/main.tf # tflint terraform_unused_declarations (bundled ruleset, recommended preset, no --init) printf 'void f(void) {\n char a[4];\n a[10] = 0;\n}\n' >app/cpp_canary.c # cppcheck arrayIndexOutOfBounds (default checks) printf 'import hashlib\n\nh = hashlib.md5(b"acme")\n' >app/ds_canary.py # devskim DS126858 weak hash +# govulncheck (CI online runs: the Go modules and vuln.go.dev are fetched) needs a called symbol of a vulnerable module: yaml.v2 2.2.2 (GO-2020-0036, GO-2021-0061, GO-2022-0956). Added only when govulncheck is under test, so the other Go canaries stay stdlib-only offline +case " $tools " in *" govulncheck "*) + mkdir vulncanary && printf 'package vulncanary\n\nimport "gopkg.in/yaml.v2"\n\nfunc F(b []byte) error {\n\tvar v interface{}\n\treturn yaml.Unmarshal(b, &v)\n}\n' >vulncanary/canary.go + printf 'module example.com/canary\n\ngo 1.21\n\nrequire gopkg.in/yaml.v2 v2.2.2\n' >go.mod + # go.sum from the module proxy and checksum DB, module cache filled; the hub run itself is GOPROXY=off + for i in 1 2 3; do GOFLAGS=-mod=mod go mod tidy >"$T/tidy" 2>&1 && break; [ "$i" -lt 3 ] || { cat "$T/tidy"; echo "FAIL - go mod tidy for the govulncheck canary"; exit 1; }; sleep $((i * 10)); done ;; +esac +# grype scans an SBOM (the hub never points it at a lockfile): a CycloneDX file with one known-vulnerable package (lodash 4.17.15, GHSA-35jh-r3h4-6jhm); its vulnerability DB is fetched online by the CI step before this runs +mkdir sbom && printf '{"bomFormat":"CycloneDX","specVersion":"1.5","version":1,"components":[{"type":"library","name":"lodash","version":"4.17.15","purl":"pkg:npm/lodash@4.17.15"}]}\n' >sbom/bom.json G add . && G commit -q -m canaries -rc=0; bash "$RC" --base base >"$T/out" 2>"$T/err" || rc=$? +# govulncheck (vuln.go.dev) and grype are online: a transient fetch error reads as not run, so retry the hub run up to 3 times while either is not run +for i in 1 2 3; do rc=0; bash "$RC" --base base >"$T/out" 2>"$T/err" || rc=$?; grep -qE 'not run:.*\b(govulncheck|grype)\b' "$T/err" && [ "$i" -lt 3 ] || break; sleep $((i * 10)); done echo "review_checks exit $rc; stderr tail:"; tail -n 15 "$T/err" # tool -> rule id regex (rubocop and ansible-lint ids are exact; tfsec and cfn-lint match the rule family, unpinned ids) want() { case $1 in tfsec) echo '^aws-s3-' ;; ansible-lint) echo '^no-changed-when' ;; rubocop) echo '^Security/Eval$' ;; cfn-lint) echo '^E3' ;; pmd) echo '^(HardCodedCryptoKey|InsecureCryptoIv)$' ;; solhint) echo '^avoid-tx-origin$' ;; hadolint) echo '^DL3007$' ;; bandit) echo '^B602$' ;; ruff-security) echo '^S602$' ;; actionlint) echo '^syntax-check$' ;; zizmor) echo '^unpinned-uses$' ;; gosec) echo '^G(401|501)$' ;; staticcheck) echo '^SA4000$' ;; kube-linter) echo '^latest-tag$' ;; flawfinder) echo '^FF1013$' ;; detect-secrets) echo '^Basic Auth Credentials$' ;; gitleaks) echo '^github-pat$' ;; trufflehog) echo '^Github$' ;; checkov) echo '^CKV_DOCKER_(3|7)$' ;; kube-score) echo '^container-image-tag$' ;; semgrep) echo '^acme-canary$' ;; njsscan) echo '^eval_nodejs$' ;; - tflint) echo '^terraform_unused_declarations$' ;; kics) echo "^Image Version Using 'latest'\$" ;; cppcheck) echo '^arrayIndexOutOfBounds$' ;; devskim) echo '^DS126858$' ;; esac; } + tflint) echo '^terraform_unused_declarations$' ;; kics) echo "^Image Version Using 'latest'\$" ;; cppcheck) echo '^arrayIndexOutOfBounds$' ;; devskim) echo '^DS126858$' ;; + govulncheck) echo '^GO-(2020-0036|2021-0061|2022-0956)$' ;; grype) echo '^(GHSA-(35jh-r3h4-6jhm|p6mc-m468-83gw)|CVE-2021-23337|CVE-2020-8203)$' ;; esac; } # tool -> its canary path (a row must be on this file, not just carry the rule id) cany() { case $1 in tfsec) echo infra/main.tf ;; ansible-lint) echo playbooks/site.yml ;; rubocop) echo app/canary.rb ;; cfn-lint) echo infra/stack.yaml ;; pmd) echo app/Canary.java ;; solhint) echo contracts/Canary.sol ;; hadolint) echo docker/Dockerfile ;; bandit|ruff-security) echo app/canary.py ;; actionlint) echo .github/workflows/canary.yml ;; zizmor) echo .github/workflows/zcanary.yml ;; gosec|staticcheck) echo gocanary/canary.go ;; kube-linter) echo k8s/pod.yaml ;; flawfinder) echo app/canary.c ;; detect-secrets) echo app/notes.txt ;; gitleaks|trufflehog) echo app/token.cfg ;; checkov) echo docker/Dockerfile ;; kube-score) echo k8s/pod.yaml ;; semgrep) echo app/sg.txt ;; njsscan) echo app/canary.js ;; - tflint) echo tfl/main.tf ;; kics) echo docker/Dockerfile ;; cppcheck) echo app/cpp_canary.c ;; devskim) echo app/ds_canary.py ;; esac; } + tflint) echo tfl/main.tf ;; kics) echo docker/Dockerfile ;; cppcheck) echo app/cpp_canary.c ;; devskim) echo app/ds_canary.py ;; govulncheck) echo go.mod ;; grype) echo sbom/bom.json ;; esac; } # the hub's own capture is private to its run: on a failure, rerun the tool's hub argv here and show its raw output head raw() { case $1 in tfsec) (cd infra && tfsec . --format sarif --no-colour --soft-fail --no-module-downloads) ;; ansible-lint) ansible-lint --offline --nocolor -q -f sarif ./playbooks/site.yml ;; rubocop) rubocop --force-default-config --fail-level fatal --format json --cache false -- ./app/canary.rb ;; cfn-lint) cfn-lint --format json -- ./infra/stack.yaml ;; @@ -72,7 +83,8 @@ raw() { case $1 in tfsec) (cd infra && tfsec . --format sarif --no-colour --soft checkov) CKV_SKIP_PACKAGE_UPDATE_CHECK=true checkov -f ./docker/Dockerfile -o json --quiet --skip-download --soft-fail ;; kube-score) kube-score score -o json ./k8s/pod.yaml ;; semgrep) HOME="$T" semgrep --config .semgrep.yml --metrics=off --disable-version-check --json --quiet ./app/sg.txt ;; njsscan) njsscan --json ./app/canary.js ;; tflint) tflint --format=json --no-color --call-module-type=none --chdir=./tfl --filter=main.tf ;; kics) kics scan -p ./docker/Dockerfile --no-progress --disable-secrets --report-formats json -o "$T/kics-raw"; cat "$T/kics-raw/results.json" ;; - cppcheck) cppcheck --xml-version=2 --quiet ./app/cpp_canary.c ;; devskim) devskim analyze --source-code ./app/ds_canary.py --file-format sarif ;; esac; } + cppcheck) cppcheck --xml-version=2 --quiet ./app/cpp_canary.c ;; devskim) devskim analyze --source-code ./app/ds_canary.py --file-format sarif ;; + govulncheck) GOPROXY=off GOTOOLCHAIN=local govulncheck -format json ./... ;; grype) GRYPE_DB_AUTO_UPDATE=false GRYPE_CHECK_FOR_APP_UPDATE=false grype sbom:./sbom/bom.json -o json ;; esac; } for t in $tools; do re=$(want "$t"); cf=$(cany "$t") if grep -qE "analyzers ran:.*\b$t\b" "$T/err" && ! grep -q "not run: $t" "$T/err" && ! grep -q "NOT CHECKED: $t" "$T/err" \ @@ -83,5 +95,7 @@ for t in $tools; do echo "--- raw $t output (first 40 lines, exit via rerun) ---"; { raw "$t" 2>&1 || echo "[exit $?]"; } >"$T/raw" || true; head -n 40 "$T/raw"; echo "--- end raw $t ---" fi done +# the govulncheck canary package must not feed the other Go canaries (its go.mod has to stay at the repo root: the hub runs govulncheck only there) +jq -e 'any(.[]; (.tool=="gosec" or .tool=="staticcheck") and (.file|tostring|test("vulncanary")))' "$T/out" >/dev/null && { echo "FAIL - gosec/staticcheck reported a row on vulncanary"; fail=1; } [ "$fail" = 0 ] && echo "PASS: test-review-checks-real-hub" exit "$fail"