CI: real-tool smoke for trivy, with a digest-pinned vuln DB

.github/workflows/ci.yml

@@ -515,8 +515,8 @@ jobs:
       - name: Allowlist guard
         run: REQUIRE_GITLEAKS=1 bash scripts/test_gitleaks_allowlist.sh
 
-  # The review_checks.sh stub tests never run a real scanner. This job installs the real tfsec, ansible-lint, rubocop, cfn-lint, pmd, solhint, hadolint, bandit, ruff, actionlint, zizmor, gosec, staticcheck, kube-linter, flawfinder, detect-secrets, gitleaks, trufflehog, checkov, kube-score, kubescape, semgrep, njsscan, tflint, kics, cppcheck, devskim, govulncheck and grype
-  # (pinned, hash-locked where the ecosystem allows; no advisory-DB download, except govulncheck and grype, which fetch their vulnerability DB online) and checks each is RAN with its rule id on one canary. trivy, bundler-audit and retire need a downloaded DB or rule set, so they stay stub-only; syft is an inventory the hub never counts as ran.
+  # The review_checks.sh stub tests never run a real scanner. This job installs the real tfsec, ansible-lint, rubocop, cfn-lint, pmd, solhint, hadolint, bandit, ruff, actionlint, zizmor, gosec, staticcheck, kube-linter, flawfinder, detect-secrets, gitleaks, trufflehog, checkov, kube-score, kubescape, semgrep, njsscan, tflint, kics, cppcheck, devskim, govulncheck, grype and trivy
+  # (pinned, hash-locked where the ecosystem allows; no advisory-DB download, except govulncheck, grype and trivy, which fetch their vulnerability DB online) and checks each is RAN with its rule id on one canary. bundler-audit and retire need a downloaded DB or rule set, so they stay stub-only; syft is an inventory the hub never counts as ran.
   real-hub:
     runs-on: ubuntu-24.04  # pinned: the cppcheck .deb below is the noble build and its runtime deps come from noble
     timeout-minutes: 25
@@ -705,6 +705,33 @@ jobs:
           echo "e9da18cc1429e786722e32161288103600efffd8e483c90761f90485bd1bec85  grype_0.121.3_linux_amd64.tar.gz" | sha256sum -c -
           tar -xzf grype_0.121.3_linux_amd64.tar.gz -C "$RUNNER_TEMP/bin" grype
           for i in 1 2 3; do if GRYPE_CHECK_FOR_APP_UPDATE=false "$RUNNER_TEMP/bin/grype" db update; then break; fi; [ "$i" -lt 3 ] || exit 1; sleep $((i * 10)); done
+      - name: Install trivy 0.76.2 (pinned sha256 from the release checksums file; vuln DB pulled by OCI digest; misconfig checks = the bundle embedded in the pinned binary)
+        run: |
+          set -euo pipefail
+          mkdir -p "$RUNNER_TEMP/bin"
+          cd "$RUNNER_TEMP"
+          curl --proto '=https' --tlsv1.2 -fsSL -O https://github.com/aquasecurity/trivy/releases/download/v0.76.2/trivy_0.76.2_checksums.txt
+          curl --proto '=https' --tlsv1.2 -fsSL -O https://github.com/aquasecurity/trivy/releases/download/v0.76.2/trivy_0.76.2_checksums.txt.sigstore.json
+          curl --proto '=https' --tlsv1.2 -fsSL -O https://github.com/aquasecurity/trivy/releases/download/v0.76.2/trivy_0.76.2_Linux-64bit.tar.gz
+          # cosign 3.1.4 pinned by sha256 (the release asset digest); it verifies the checksums file's sigstore bundle against the trivy release workflow identity
+          curl --proto '=https' --tlsv1.2 -fsSL -o cosign https://github.com/sigstore/cosign/releases/download/v3.1.4/cosign-linux-amd64
+          echo "424e2c97d0b45fd7a6b608a87f090f35633f4c23ece5f092d48bba11aaf4bbfa  cosign" | sha256sum -c -
+          chmod +x cosign
+          ./cosign verify-blob --bundle trivy_0.76.2_checksums.txt.sigstore.json \
+            --certificate-identity "https://github.com/aquasecurity/trivy/.github/workflows/reusable-release.yaml@refs/tags/v0.76.2" \
+            --certificate-oidc-issuer "https://token.actions.githubusercontent.com" trivy_0.76.2_checksums.txt
+          grep -F ' trivy_0.76.2_Linux-64bit.tar.gz' trivy_0.76.2_checksums.txt | sha256sum -c -
+          echo "bc0fc726e017b8927a62b2d3531d7a84ee01769daf5c99ed866dced54089fdbf  trivy_0.76.2_Linux-64bit.tar.gz" | sha256sum -c -
+          tar -xzf trivy_0.76.2_Linux-64bit.tar.gz -C "$RUNNER_TEMP/bin" trivy
+          # vuln DB: the ghcr trivy-db:2 manifest digest recorded 2026-11-13 (the tag moves daily; a digest the registry later garbage-collects fails this step loudly, bump it then). The hub run itself is offline.
+          export TRIVY_DISABLE_TELEMETRY=true
+          dg=ghcr.io/aquasecurity/trivy-db@sha256:f3fc0ea9b0d550c458eff73a503f206a861256dff2a2e4e0513419a3d79fbe7c
+          for i in 1 2 3; do if "$RUNNER_TEMP/bin/trivy" --cache-dir "$RUNNER_TEMP/trivy-cache" image --download-db-only --db-repository "$dg" >"$RUNNER_TEMP/dbdl.log" 2>&1; then break; fi; cat "$RUNNER_TEMP/dbdl.log"; [ "$i" -lt 3 ] || exit 1; sleep $((i * 10)); done
+          cat "$RUNNER_TEMP/dbdl.log"
+          # the cache stores no digest, so assert trivy fetched the pinned digest reference (a digest form it ignored would log the default tag) and wrote a DB
+          grep -qF "repo=\"$dg\"" "$RUNNER_TEMP/dbdl.log"
+          test -s "$RUNNER_TEMP/trivy-cache/db/trivy.db"
+          echo "REVIEW_TRIVY_CACHE=$RUNNER_TEMP/trivy-cache" >> "$GITHUB_ENV"
       - name: Real-tool canaries
         run: REQUIRE_REAL_HUB=1 bash scripts/test-review-checks-real-hub.sh
 

changelog.d/hub-real-smoke-9.md

@@ -0,0 +1,3 @@
+### Changed
+- The real-hub CI job now also runs real trivy 0.76.2 through `review_checks.sh` on two fictional canaries: a Dockerfile (`FROM ubuntu:latest`, misconfig `DS-0001`/`DS-0002`) and an npm lockfile pinning lodash 4.17.15 (a vulnerability row). The job fails unless trivy ran as a full scan (no `not run`, no `NOT CHECKED`, no secret-only fallback) and reported both. Pins: the binary by sha256 from the release checksums file, which is itself verified with `cosign verify-blob` (cosign 3.1.4 pinned by sha256) against its sigstore bundle, the trivy release workflow identity and the GitHub Actions OIDC issuer; the vulnerability DB by OCI digest (`ghcr.io/aquasecurity/trivy-db@sha256:...`, bounded retries); the misconfig checks are the bundle embedded in the pinned binary (`TRIVY_SKIP_CHECK_UPDATE=true`), so no check download happens at run time. A pinned DB digest that the registry later garbage-collects fails the install step loudly and needs a bump.
+- Not added: codeql. Its CLI bundle is a multi-hundred-MB download plus a database build per language, too heavy for a per-push job.

scripts/test-review-checks-real-hub.sh

@@ -1,17 +1,17 @@
 #!/usr/bin/env bash
-# Real-tool smoke for the review_checks.sh hub: runs the REAL tfsec, ansible-lint, rubocop, cfn-lint, pmd, solhint, hadolint, bandit, ruff (ruff-security), actionlint, zizmor, gosec, staticcheck, kube-linter, flawfinder, detect-secrets, gitleaks, trufflehog, checkov, kube-score, kubescape (pinned local frameworks), semgrep (repo rules) and njsscan, tflint, kics, cppcheck, devskim, govulncheck and grype (no stubs) on one tiny
+# Real-tool smoke for the review_checks.sh hub: runs the REAL tfsec, ansible-lint, rubocop, cfn-lint, pmd, solhint, hadolint, bandit, ruff (ruff-security), actionlint, zizmor, gosec, staticcheck, kube-linter, flawfinder, detect-secrets, gitleaks, trufflehog, checkov, kube-score, kubescape (pinned local frameworks), semgrep (repo rules) and njsscan, tflint, kics, cppcheck, devskim, govulncheck, grype and trivy (no stubs) on one tiny
 # known-bad canary each, in a throwaway repo, and asserts each is reported as RAN (stderr "analyzers ran:", no "not run:" line)
 # with its expected rule id in the JSON rows. The stub tests (scripts/test-review-checks-hub-*.sh) pin routing and failure
-# handling; this pins the real flags, SARIF/JSON shapes and exit codes. Not covered: syft (an inventory the hub never counts as ran), trivy (its vuln scan needs the downloaded DB, so the hub reports it NOT CHECKED), bundler-audit and retire need a
-# network-fetched advisory DB or rule set (they are NOT CHECKED without one), so a real run would need a download.
+# handling; this pins the real flags, SARIF/JSON shapes and exit codes. Not covered: syft (an inventory the hub never counts as ran), bundler-audit and retire need a
+# network-fetched advisory DB or rule set (they are NOT CHECKED without one), so a real run would need a download. codeql is dropped: its CLI bundle is a multi-hundred-MB download plus per-language database builds, too heavy for this per-push job.
 # A tool absent from PATH is skipped (exit 0) unless REQUIRE_REAL_HUB=1 (the ci.yml real-hub job sets it), then it FAILs.
 # Read-only on the repo; writes only under a temp dir. Fixtures are fictional canaries.
 set -euo pipefail
 ROOT=$(cd "$(dirname "$0")/.." && pwd)
 RC=${REVIEW_CHECKS:-$ROOT/.claude/skills/deep-code-review/scripts/review_checks.sh}
 fail=0; tools=
 bin() { [ "$1" = ruff-security ] && echo ruff || echo "$1"; }  # the hub tool name ruff-security runs the ruff binary
-for t in tfsec ansible-lint rubocop cfn-lint pmd solhint hadolint bandit ruff-security actionlint zizmor gosec staticcheck kube-linter flawfinder detect-secrets gitleaks trufflehog checkov kube-score kubescape semgrep njsscan tflint kics cppcheck devskim govulncheck grype; do
+for t in tfsec ansible-lint rubocop cfn-lint pmd solhint hadolint bandit ruff-security actionlint zizmor gosec staticcheck kube-linter flawfinder detect-secrets gitleaks trufflehog checkov kube-score kubescape semgrep njsscan tflint kics cppcheck devskim govulncheck grype trivy; do
   if command -v "$(bin "$t")" >/dev/null 2>&1; then tools+=" $t"
   elif [ "${REQUIRE_REAL_HUB:-}" = 1 ]; then echo "FAIL - $t not installed (REQUIRE_REAL_HUB=1)"; fail=1
   else echo "skip - $t not installed"; fi
@@ -55,23 +55,25 @@ case " $tools " in *" govulncheck "*)
 esac
 # grype scans an SBOM (the hub never points it at a lockfile): a CycloneDX file with one known-vulnerable package (lodash 4.17.15, GHSA-35jh-r3h4-6jhm); its vulnerability DB is fetched online by the CI step before this runs
 mkdir sbom && printf '{"bomFormat":"CycloneDX","specVersion":"1.5","version":1,"components":[{"type":"library","name":"lodash","version":"4.17.15","purl":"pkg:npm/lodash@4.17.15"}]}\n' >sbom/bom.json
+# trivy: misconfig DS-0001/DS-0002 on docker/Dockerfile (above) plus a vuln on a lockfile (lodash 4.17.15). The vuln DB comes from REVIEW_TRIVY_CACHE (the CI step pulls it by digest); TRIVY_SKIP_CHECK_UPDATE keeps the misconfig checks to the bundle embedded in the sha-pinned binary
+case " $tools " in *" trivy "*) printf '{"name":"acme-canary","lockfileVersion":3,"requires":true,"packages":{"":{"name":"acme-canary","version":"1.0.0","dependencies":{"lodash":"4.17.15"}},"node_modules/lodash":{"version":"4.17.15"}}}\n' >package-lock.json; export TRIVY_SKIP_CHECK_UPDATE=true ;; esac
 G add . && G commit -q -m canaries
 # govulncheck (vuln.go.dev) and grype are online: a transient fetch error reads as not run, so retry the hub run up to 3 times while either is not run
-for i in 1 2 3; do rc=0; bash "$RC" --base base >"$T/out" 2>"$T/err" || rc=$?; grep -qE 'not run:.*\b(govulncheck|grype)\b' "$T/err" && [ "$i" -lt 3 ] || break; sleep $((i * 10)); done
+for i in 1 2 3; do rc=0; bash "$RC" --base base >"$T/out" 2>"$T/err" || rc=$?; grep -qE 'not run:.*\b(govulncheck|grype|trivy)\b' "$T/err" && [ "$i" -lt 3 ] || break; sleep $((i * 10)); done
 echo "review_checks exit $rc; stderr tail:"; tail -n 15 "$T/err"
 # tool -> rule id regex (rubocop and ansible-lint ids are exact; tfsec and cfn-lint match the rule family, unpinned ids)
 want() { case $1 in tfsec) echo '^aws-s3-' ;; ansible-lint) echo '^no-changed-when' ;; rubocop) echo '^Security/Eval$' ;; cfn-lint) echo '^E3' ;;
   pmd) echo '^(HardCodedCryptoKey|InsecureCryptoIv)$' ;; solhint) echo '^avoid-tx-origin$' ;; hadolint) echo '^DL3007$' ;; bandit) echo '^B602$' ;; ruff-security) echo '^S602$' ;; actionlint) echo '^syntax-check$' ;; zizmor) echo '^unpinned-uses$' ;; gosec) echo '^G(401|501)$' ;; staticcheck) echo '^SA4000$' ;;
   kube-linter) echo '^latest-tag$' ;; flawfinder) echo '^FF1013$' ;; detect-secrets) echo '^Basic Auth Credentials$' ;;
   gitleaks) echo '^github-pat$' ;; trufflehog) echo '^Github$' ;; checkov) echo '^CKV_DOCKER_(3|7)$' ;; kube-score) echo '^container-image-tag$' ;; kubescape) echo '^C-0057$' ;; semgrep) echo '^acme-canary$' ;; njsscan) echo '^eval_nodejs$' ;;
   tflint) echo '^terraform_unused_declarations$' ;; kics) echo "^Image Version Using 'latest'\$" ;; cppcheck) echo '^arrayIndexOutOfBounds$' ;; devskim) echo '^DS126858$' ;;
-  govulncheck) echo '^GO-(2020-0036|2021-0061|2022-0956)$' ;; grype) echo '^(GHSA-(35jh-r3h4-6jhm|p6mc-m468-83gw)|CVE-2021-23337|CVE-2020-8203)$' ;; esac; }
+  govulncheck) echo '^GO-(2020-0036|2021-0061|2022-0956)$' ;; grype) echo '^(GHSA-(35jh-r3h4-6jhm|p6mc-m468-83gw)|CVE-2021-23337|CVE-2020-8203)$' ;; trivy) echo '^DS-000[12]$' ;; esac; }
 # tool -> its canary path (a row must be on this file, not just carry the rule id)
 cany() { case $1 in tfsec) echo infra/main.tf ;; ansible-lint) echo playbooks/site.yml ;; rubocop) echo app/canary.rb ;; cfn-lint) echo infra/stack.yaml ;;
   pmd) echo app/Canary.java ;; solhint) echo contracts/Canary.sol ;; hadolint) echo docker/Dockerfile ;; bandit|ruff-security) echo app/canary.py ;; actionlint) echo .github/workflows/canary.yml ;; zizmor) echo .github/workflows/zcanary.yml ;; gosec|staticcheck) echo gocanary/canary.go ;;
   kube-linter) echo k8s/pod.yaml ;; flawfinder) echo app/canary.c ;; detect-secrets) echo app/notes.txt ;; gitleaks|trufflehog) echo app/token.cfg ;;
   checkov) echo docker/Dockerfile ;; kube-score) echo k8s/pod.yaml ;; kubescape) echo k8s/ks.yaml ;; semgrep) echo app/sg.txt ;; njsscan) echo app/canary.js ;;
-  tflint) echo tfl/main.tf ;; kics) echo docker/Dockerfile ;; cppcheck) echo app/cpp_canary.c ;; devskim) echo app/ds_canary.py ;; govulncheck) echo go.mod ;; grype) echo sbom/bom.json ;; esac; }
+  tflint) echo tfl/main.tf ;; kics) echo docker/Dockerfile ;; cppcheck) echo app/cpp_canary.c ;; devskim) echo app/ds_canary.py ;; govulncheck) echo go.mod ;; grype) echo sbom/bom.json ;; trivy) echo docker/Dockerfile ;; esac; }
 # the hub's own capture is private to its run: on a failure, rerun the tool's hub argv here and show its raw output head
 raw() { case $1 in tfsec) (cd infra && tfsec . --format sarif --no-colour --soft-fail --no-module-downloads) ;; ansible-lint) ansible-lint --offline --nocolor -q -f sarif ./playbooks/site.yml ;;
   rubocop) rubocop --force-default-config --fail-level fatal --format json --cache false -- ./app/canary.rb ;; cfn-lint) cfn-lint --format json -- ./infra/stack.yaml ;;
@@ -85,7 +87,8 @@ raw() { case $1 in tfsec) (cd infra && tfsec . --format sarif --no-colour --soft
   semgrep) HOME="$T" semgrep --config .semgrep.yml --metrics=off --disable-version-check --json --quiet ./app/sg.txt ;; njsscan) njsscan --json ./app/canary.js ;;
   tflint) tflint --format=json --no-color --call-module-type=none --chdir=./tfl --filter=main.tf ;; kics) kics scan -p ./docker/Dockerfile --no-progress --disable-secrets --report-formats json -o "$T/kics-raw"; cat "$T/kics-raw/results.json" ;;
   cppcheck) cppcheck --xml-version=2 --quiet ./app/cpp_canary.c ;; devskim) devskim analyze --source-code ./app/ds_canary.py --file-format sarif ;;
-  govulncheck) GOPROXY=off GOTOOLCHAIN=local govulncheck -format json ./... ;; grype) GRYPE_DB_AUTO_UPDATE=false GRYPE_CHECK_FOR_APP_UPDATE=false grype sbom:./sbom/bom.json -o json ;; esac; }
+  govulncheck) GOPROXY=off GOTOOLCHAIN=local govulncheck -format json ./... ;; grype) GRYPE_DB_AUTO_UPDATE=false GRYPE_CHECK_FOR_APP_UPDATE=false grype sbom:./sbom/bom.json -o json ;;
+  trivy) trivy ${REVIEW_TRIVY_CACHE:+--cache-dir "$REVIEW_TRIVY_CACHE"} fs --quiet --format json --skip-version-check --skip-db-update --offline-scan --scanners vuln,misconfig . ;; esac; }
 for t in $tools; do
   re=$(want "$t"); cf=$(cany "$t")
   if grep -qE "analyzers ran:.*\b$t\b" "$T/err" && ! grep -q "not run: $t" "$T/err" && ! grep -q "NOT CHECKED: $t" "$T/err" \
@@ -96,6 +99,11 @@ for t in $tools; do
     echo "--- raw $t output (first 40 lines, exit via rerun) ---"; { raw "$t" 2>&1 || echo "[exit $?]"; } >"$T/raw" || true; head -n 40 "$T/raw"; echo "--- end raw $t ---"
   fi
 done
+# trivy: the full scan ran (not the secret-only fallback) and its vuln scan reported the lockfile canary
+case " $tools " in *" trivy "*)
+  grep -q 'secret-only' "$T/err" && { echo "FAIL - trivy fell back to secret-only"; fail=1; }
+  for id in DS-0001 DS-0002; do jq -e --arg id "$id" 'any(.[]; .tool=="trivy" and .rule==$id and (.file|tostring|endswith("docker/Dockerfile")))' "$T/out" >/dev/null || { echo "FAIL - trivy: no $id row on docker/Dockerfile"; fail=1; }; done
+  jq -e 'any(.[]; .tool=="trivy" and (.rule|test("^(CVE-2020-8203|CVE-2021-23337|CVE-2020-28500|GHSA-(35jh-r3h4-6jhm|29mw-wpgm-hmr9|p6mc-m468-83gw))$")) and (.file|tostring|endswith("package-lock.json")))' "$T/out" >/dev/null || { echo "FAIL - trivy: no lodash vuln row on package-lock.json"; fail=1; } ;; esac
 # the govulncheck canary package must not feed the other Go canaries (its go.mod has to stay at the repo root: the hub runs govulncheck only there)
 jq -e 'any(.[]; (.tool=="gosec" or .tool=="staticcheck") and (.file|tostring|test("vulncanary")))' "$T/out" >/dev/null && { echo "FAIL - gosec/staticcheck reported a row on vulncanary"; fail=1; }
 [ "$fail" = 0 ] && echo "PASS: test-review-checks-real-hub"