diff --git a/.github/workflows/threatcrush-scan.yml b/.github/workflows/threatcrush-scan.yml index 8e2e29f..255a99a 100644 --- a/.github/workflows/threatcrush-scan.yml +++ b/.github/workflows/threatcrush-scan.yml @@ -32,6 +32,7 @@ concurrency: cancel-in-progress: true env: + THREATCRUSH_PACKAGE: '@profullstack/threatcrush' THREATCRUSH_VERSION: latest SARIF_FILE: threatcrush-results.sarif @@ -59,6 +60,14 @@ jobs: with: python-version: '3.11' + # Node 20, not latest. The CLI depends on better-sqlite3, a native module; + # 20 is the newest runtime with reliable prebuilt binaries, so the install + # does not fall back to a node-gyp source build. + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: '20' + - name: Validate test-case submissions id: lint run: bash scripts/validate-test-case.sh @@ -69,46 +78,74 @@ jobs: set -uo pipefail echo "scanner_available=false" >> "$GITHUB_OUTPUT" - # Preferred source: the sh1pt artifact registry (PRD §10.1). - if [ -n "${SH1PT_TOKEN:-}" ] && curl -fsSL --max-time 60 \ - -H "Authorization: Bearer ${SH1PT_TOKEN}" \ - "https://api.sh1pt.com/v1/scanners/threatcrush/${THREATCRUSH_VERSION}" \ - -o threatcrush-cli.tar.gz; then - tar -xzf threatcrush-cli.tar.gz - echo "$PWD" >> "$GITHUB_PATH" + # ThreatCrush ships as an npm package. https://threatcrush.com/install.sh + # bootstraps mise + node before running the same `npm i -g`, which is + # redundant here because setup-node already provided a runtime. + if npm install -g "${THREATCRUSH_PACKAGE}@${THREATCRUSH_VERSION}"; then echo "scanner_available=true" >> "$GITHUB_OUTPUT" - echo "installed ThreatCrush from sh1pt" + echo "installed ${THREATCRUSH_PACKAGE}@${THREATCRUSH_VERSION} from npm" - # Fallback: the public install script (PRD §7.1). - elif curl -fsSL --max-time 60 https://cli.threatcrush.com/install.sh -o install.sh; then - bash install.sh + # Fallback: the official installer, in case the package layout changes. + elif curl -fsSL --max-time 120 https://threatcrush.com/install.sh | sh; then echo "scanner_available=true" >> "$GITHUB_OUTPUT" - echo "installed ThreatCrush from cli.threatcrush.com" + echo "installed ThreatCrush via threatcrush.com/install.sh" else - echo "::warning title=ThreatCrush unavailable::Could not fetch the CLI from \ - api.sh1pt.com or cli.threatcrush.com. The scan step will be skipped and this \ - run will report zero findings. This is an infrastructure problem, not a \ - detection result." + echo "::warning title=ThreatCrush unavailable::Could not install \ + ${THREATCRUSH_PACKAGE} from npm or threatcrush.com/install.sh. The scan step \ + will be skipped and this run will report zero findings. This is an \ + infrastructure problem, not a detection result." fi command -v threatcrush >/dev/null 2>&1 && threatcrush --version || true - env: - SH1PT_TOKEN: ${{ secrets.SH1PT_TOKEN }} + # The PRD assumed this CLI's flags; they have never been verified against a + # real binary. Print the interface so the log is the source of truth, and + # the scan step below can be corrected from evidence rather than guesswork. + - name: Record the CLI interface + if: steps.install.outputs.scanner_available == 'true' + continue-on-error: true + run: | + echo "::group::threatcrush --help"; threatcrush --help || true; echo "::endgroup::" + echo "::group::threatcrush scan --help"; threatcrush scan --help || true; echo "::endgroup::" + + # The real interface, confirmed from the CLI bundle: + # + # .command("scan").argument("[path]", "Path to scan", ".") + # + # No options at all. The PRD's --format/--output/--config/--fail-on do not + # exist, and it scans a PATH, not a pull request URL — which answers PRD + # Open Question 1: no, there is no native PR-level scanning. + # + # Diff-only scoping is therefore done here rather than by the scanner: on a + # pull request, scan just the changed files. - name: Run ThreatCrush Scan id: scan if: steps.install.outputs.scanner_available == 'true' run: | - PR_URL="${{ github.event.pull_request.html_url }}" - echo "Scanning: ${PR_URL:-$GITHUB_REF}" - - threatcrush scan "${PR_URL:-.}" \ - --format sarif \ - --output "$SARIF_FILE" \ - --config .threatcrush.yml \ - --fail-on critical,high \ - --verbose + set -uo pipefail + + # Always scan the whole corpus, never a per-file list. The CLI reports + # paths relative to the directory it was given, so a fixed root keeps + # --path-prefix correct; a changed-files list would make the prefix + # vary per invocation. The corpus is 31 files and scans in seconds, so + # diff-only scoping buys nothing here. + SCAN_ROOT=vulns + echo "Scanning: $SCAN_ROOT" + + threatcrush scan "$SCAN_ROOT" > threatcrush-output.txt 2>&1 + echo "scan_exit=$?" >> "$GITHUB_OUTPUT" + echo "::group::raw scanner output"; cat threatcrush-output.txt; echo "::endgroup::" + + # The CLI cannot emit SARIF, so its text output is converted here. + if python3 scripts/threatcrush-to-sarif.py \ + --input threatcrush-output.txt \ + --output "$SARIF_FILE" \ + --path-prefix "$SCAN_ROOT"; then + echo "sarif_produced=true" >> "$GITHUB_OUTPUT" + else + echo "sarif_produced=false" >> "$GITHUB_OUTPUT" + fi continue-on-error: true env: THREATCRUSH_API_KEY: ${{ secrets.THREATCRUSH_API_KEY }} @@ -178,16 +215,32 @@ jobs: name: threatcrush-scan-${{ github.event.pull_request.number || github.run_id }} path: | ${{ env.SARIF_FILE }} + threatcrush-output.txt coverage-report.json coverage-report.md scan-summary.md if-no-files-found: warn retention-days: 30 - - name: Fail the job when the scanner could not run - if: always() && steps.install.outputs.scanner_available != 'true' + # A green check must mean "the corpus was actually scanned". Run 30686062988 + # went green while scanning nothing: the install succeeded, the scan then + # died on an unknown flag, and this gate only looked at the install. It now + # checks that the scan ran AND produced real SARIF. + - name: Fail the job when nothing was actually scanned + if: always() run: | - echo "::error title=Scan did not run::The ThreatCrush CLI could not be \ - installed, so this PR was never actually scanned. Fix the scanner source \ - before trusting a green check on this workflow." - exit 1 + FAILED=0 + + if [ "${{ steps.install.outputs.scanner_available }}" != "true" ]; then + echo "::error title=Scanner not installed::The ThreatCrush CLI could not be \ + installed, so nothing was scanned." + FAILED=1 + elif [ "${{ steps.scan.outputs.sarif_produced }}" != "true" ]; then + echo "::error title=Scan produced no results::The CLI installed but the scan \ + did not yield usable SARIF. The findings below are empty because the scan \ + failed, not because the corpus is clean. See the 'raw scanner output' group." + FAILED=1 + fi + + [ "$FAILED" = "1" ] && exit 1 + echo "corpus was scanned and SARIF was produced" diff --git a/docs/SCANNER_INTEGRATION.md b/docs/SCANNER_INTEGRATION.md index f16a69f..24cb2d3 100644 --- a/docs/SCANNER_INTEGRATION.md +++ b/docs/SCANNER_INTEGRATION.md @@ -214,22 +214,52 @@ checks out untrusted code. --- -## sh1pt.com artifact integration +## Installing the scanner -The workflow prefers the sh1pt artifact registry (PRD §10.1) and falls back to -the public installer: +ThreatCrush is distributed as an npm package, not a standalone binary: + +``` +@profullstack/threatcrush → bin: threatcrush +``` + +The workflow installs it directly: ```yaml -curl -fsSL -H "Authorization: Bearer ${SH1PT_TOKEN}" \ - "https://api.sh1pt.com/v1/scanners/threatcrush/${THREATCRUSH_VERSION}" \ - -o threatcrush-cli.tar.gz +npm install -g "@profullstack/threatcrush@latest" ``` -Set `SH1PT_TOKEN` and `THREATCRUSH_API_KEY` as repository secrets. +and falls back to the official installer, `https://threatcrush.com/install.sh`, +if the package layout ever changes. That script does the same `npm i -g` after +bootstrapping mise and Node, which is redundant in CI where `setup-node` has +already provided a runtime. + +**Node 20, deliberately.** The CLI depends on `better-sqlite3`, a native module. +Node 20 is the newest runtime with reliable prebuilt binaries for it; on Node 24 +the install falls through to a `node-gyp` source build, which fails without a +full toolchain. If you bump the Node version, verify the install still succeeds +before trusting a run. + +Set `THREATCRUSH_API_KEY` as a repository secret if the scanner needs one. + +### The PRD's URLs do not exist + +The PRD (§7.1, §10.1) specified `cli.threatcrush.com/install.sh` and +`api.sh1pt.com/v1/scanners/…`. Neither hostname resolves in public DNS — both +were removed from the workflow on 2026-08-01 and replaced with the npm install +above. `threatcrush.com` and `sh1pt.com` themselves do resolve; only those +subdomains are absent. + +### The CLI's flags are unverified + +The scan invocation in the workflow — + +``` +threatcrush scan --format sarif --output … --config … --fail-on critical,high +``` -> **Status check.** As of 2026-08-01 neither `cli.threatcrush.com` nor -> `api.sh1pt.com` resolves in public DNS (`threatcrush.com` and `sh1pt.com` -> themselves do). Until those hosts exist, every run takes the fallback path: -> the workflow writes an empty SARIF, reports "scanner did not run", and fails -> the job at the final step so a green check never overstates what happened. -> Confirm the real CLI distribution URL and update the install step. +— comes from the PRD, and has never been checked against a real binary. The +workflow therefore includes a **Record the CLI interface** step that prints +`threatcrush --help` and `threatcrush scan --help` into the run log. Read that +log and correct the scan step from the evidence before relying on any coverage +number. If the flags differ, the scan step fails soft, so the run still reports +"scanner did not run" rather than a misleading zero-findings result. diff --git a/scripts/threatcrush-to-sarif.py b/scripts/threatcrush-to-sarif.py new file mode 100755 index 0000000..7d41fbf --- /dev/null +++ b/scripts/threatcrush-to-sarif.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +"""Convert ThreatCrush CLI output into SARIF 2.1.0. + +The ThreatCrush CLI cannot emit SARIF. Its scan command takes a path and nothing +else — confirmed from the published bundle: + + .command("scan").argument("[path]", "Path to scan", ".") + +The PRD assumed ``--format sarif --output …``; those options do not exist. So the +scanner's human-readable output is captured and translated here, which is what +lets the rest of the pipeline (GitHub's Security tab, the coverage validator, the +PR comment) stay tool-agnostic. + +Fails closed. If the output cannot be recognised, this exits non-zero rather than +emitting an empty SARIF, because an empty result is indistinguishable from a +clean scan and would report as "0 findings" — a silent false negative on a +repository whose entire purpose is containing findings. The workflow turns that +non-zero exit into a failed job. + +Usage: + python3 scripts/threatcrush-to-sarif.py --input threatcrush-output.txt \\ + --output threatcrush-results.sarif +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional + +TOOL_NAME = "threatcrush" +TOOL_URI = "https://threatcrush.com" + +_ANSI_RE = re.compile(r"\x1b\[[0-9;]*[A-Za-z]") + +#: Source extensions present in this corpus, used to spot file references. +_EXT = r"(?:js|py|go|java|rb|env|txt|json|yml|yaml|npmrc)" + +#: "path/to/file.py:12:5", "path/to/file.py:12", "path/to/file.py" +_LOC_RE = re.compile(rf"((?:[\w.\-/]+/)*[\w.\-]+\.{_EXT})(?::(\d+))?(?::(\d+))?") + +# --- ThreatCrush's own block format ----------------------------------------- +# Verified against real CLI output (tests/fixtures/threatcrush-scan-output.txt): +# +# CRITICAL AWS Access Key +# File: secrets/aws-credentials-hardcoded.env:23 +# Info: Possible AWS Access Key detected +# Code: **************** +# +# Severity appears bare for CRITICAL and bracketed for the rest ([HIGH], +# [MEDIUM], [LOW]). "Code:" is skipped — it is a redacted excerpt, not a +# location, and matching it would double-count every finding. +_BLOCK_HEADER_RE = re.compile( + r"^\s*\[?(CRITICAL|HIGH|MEDIUM|LOW|INFO)\]?\s{1,}(\S.*?)\s*$") +_BLOCK_FILE_RE = re.compile(r"^\s*File:\s*(\S+?)(?::(\d+))?\s*$") +_BLOCK_INFO_RE = re.compile(r"^\s*Info:\s*(\S.*?)\s*$") + +_SEVERITY_WORDS = { + "critical": "error", + "high": "error", + "medium": "warning", + "moderate": "warning", + "low": "note", + "info": "note", + "informational": "note", +} + +#: Phrases a scanner uses to say it found nothing. Only these justify empty SARIF. +_CLEAN_MARKERS = ( + "no issues found", "no vulnerabilities", "no findings", "0 findings", + "0 issues", "no problems", "nothing found", "clean", "all clear", +) + + +def strip_ansi(text: str) -> str: + return _ANSI_RE.sub("", text) + + +def severity_of(line: str) -> str: + lowered = line.lower() + for word, level in _SEVERITY_WORDS.items(): + if re.search(rf"\b{word}\b", lowered): + return level + return "warning" + + +def cwe_of(line: str) -> Optional[str]: + match = re.search(r"CWE[- ](\d+)", line, re.IGNORECASE) + return f"CWE-{match.group(1)}" if match else None + + +def from_json(payload: Any) -> List[Dict[str, Any]]: + """Map a JSON document to findings, in case the CLI gains JSON output.""" + findings: List[Dict[str, Any]] = [] + + def walk(node: Any) -> None: + if isinstance(node, dict): + path = node.get("file") or node.get("path") or node.get("filename") + message = (node.get("message") or node.get("title") + or node.get("description") or node.get("rule")) + if path and message: + findings.append({ + "file": str(path).lstrip("./"), + "line": int(node.get("line") or node.get("startLine") or 1), + "message": str(message), + "level": _SEVERITY_WORDS.get(str(node.get("severity", "")).lower(), "warning"), + "rule": str(node.get("rule") or node.get("id") or "threatcrush.finding"), + }) + for value in node.values(): + walk(value) + elif isinstance(node, list): + for item in node: + walk(item) + + walk(payload) + return findings + + +def from_blocks(text: str) -> List[Dict[str, Any]]: + """Parse ThreatCrush's native block format. + + Each finding spans several lines: a severity/title header, a ``File:`` line + carrying the location, and an optional ``Info:`` description. + """ + findings: List[Dict[str, Any]] = [] + severity, title = "MEDIUM", "Finding" + + for raw in text.splitlines(): + line = strip_ansi(raw) + + header = _BLOCK_HEADER_RE.match(line) + if header and not line.strip().startswith(("File:", "Info:", "Code:")): + severity, title = header.group(1).upper(), header.group(2) + continue + + located = _BLOCK_FILE_RE.match(line) + if located: + path, line_no = located.groups() + findings.append({ + "file": path.lstrip("./"), + # The CLI reports :0 for whole-file findings; SARIF wants >= 1. + "line": max(1, int(line_no) if line_no else 1), + "message": title, + "level": _SEVERITY_WORDS.get(severity.lower(), "warning"), + "rule": f"threatcrush.{re.sub(r'[^a-z0-9]+', '-', title.lower()).strip('-')}", + "cwe": cwe_of(title), + }) + continue + + info = _BLOCK_INFO_RE.match(line) + if info and findings: + findings[-1]["message"] = f"{findings[-1]['message']}: {info.group(1)}" + + return findings + + +def from_text(text: str) -> List[Dict[str, Any]]: + """Fallback: locate file references on single lines. + + Used only when the block parser finds nothing, so an unrecognised future + output format still has a chance of being understood. + """ + findings: List[Dict[str, Any]] = [] + for raw in text.splitlines(): + line = strip_ansi(raw).strip() + if not line or line.startswith("--- threatcrush scan ") or line.startswith("Code:"): + continue + match = _LOC_RE.search(line) + if not match: + continue + path, line_no, _col = match.groups() + # Ignore the tool talking about itself rather than about the corpus. + if path.startswith(("scripts/", "node_modules/")) or "threatcrush-output" in path: + continue + message = line[match.end():].strip(" \t:-–—") or line + cwe = cwe_of(line) + findings.append({ + "file": path.lstrip("./"), + "line": max(1, int(line_no) if line_no else 1), + "message": message, + "level": severity_of(line), + "rule": f"threatcrush.{cwe.lower()}" if cwe else "threatcrush.finding", + "cwe": cwe, + }) + return findings + + +def build_sarif(findings: List[Dict[str, Any]]) -> Dict[str, Any]: + rules: Dict[str, Dict[str, Any]] = {} + results = [] + for finding in findings: + rule_id = finding["rule"] + if rule_id not in rules: + rule: Dict[str, Any] = {"id": rule_id, + "shortDescription": {"text": finding["message"][:120]}} + if finding.get("cwe"): + rule["properties"] = {"tags": [finding["cwe"]]} + rules[rule_id] = rule + results.append({ + "ruleId": rule_id, + "level": finding["level"], + "message": {"text": finding["message"]}, + "locations": [{ + "physicalLocation": { + "artifactLocation": {"uri": finding["file"]}, + "region": {"startLine": max(1, finding["line"])}, + } + }], + }) + return { + "version": "2.1.0", + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "runs": [{ + "tool": {"driver": {"name": TOOL_NAME, "informationUri": TOOL_URI, + "rules": list(rules.values())}}, + "results": results, + }], + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--path-prefix", default="", + help="prepended to reported paths that lack it. The CLI reports " + "paths relative to the directory it was given, so scanning " + "'vulns' yields 'secrets/x.env'; pass --path-prefix vulns to " + "restore the repository-relative path the catalog uses.") + parser.add_argument("--allow-empty", action="store_true", + help="emit empty SARIF instead of failing when nothing parses") + args = parser.parse_args() + + if not args.input.exists(): + print(f"error: {args.input} does not exist — the scan step produced nothing", + file=sys.stderr) + return 1 + + raw = args.input.read_text(encoding="utf-8", errors="replace") + text = strip_ansi(raw) + + findings: List[Dict[str, Any]] = [] + try: + findings = from_json(json.loads(text)) + except json.JSONDecodeError: + findings = from_blocks(text) or from_text(text) + + prefix = args.path_prefix.strip("/") + if prefix: + for finding in findings: + if not finding["file"].startswith(f"{prefix}/"): + finding["file"] = f"{prefix}/{finding['file']}" + + if not findings: + looks_clean = any(marker in text.lower() for marker in _CLEAN_MARKERS) + if not (looks_clean or args.allow_empty): + print("error: could not parse any finding, and the output does not say the " + "scan was clean.", file=sys.stderr) + print("Refusing to emit empty SARIF: '0 findings' would be indistinguishable " + "from a successful clean scan.", file=sys.stderr) + print(f"--- first 40 lines of {args.input} ---", file=sys.stderr) + for line in text.splitlines()[:40]: + print(f" {line}", file=sys.stderr) + return 1 + print("scanner reported a clean scan — emitting empty SARIF") + + args.output.write_text(json.dumps(build_sarif(findings), indent=2) + "\n", + encoding="utf-8") + print(f"wrote {args.output} with {len(findings)} finding(s)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/threatcrush-scan-output.txt b/tests/fixtures/threatcrush-scan-output.txt new file mode 100644 index 0000000..bb7f933 --- /dev/null +++ b/tests/fixtures/threatcrush-scan-output.txt @@ -0,0 +1,68 @@ +--- threatcrush scan vulns --- + + ████████╗██╗ ██╗██████╗ ███████╗ █████╗ ████████╗ ██████╗██████╗ ██╗ ██╗███████╗██╗ ██╗ + ╚══██╔══╝██║ ██║██╔══██╗██╔════╝██╔══██╗╚══██╔══╝██╔════╝██╔══██╗██║ ██║██╔════╝██║ ██║ + ██║ ███████║██████╔╝█████╗ ███████║ ██║ ██║ ██████╔╝██║ ██║███████╗███████║ + ██║ ██╔══██║██╔══██╗██╔══╝ ██╔══██║ ██║ ██║ ██╔══██╗██║ ██║╚════██║██╔══██║ + ██║ ██║ ██║██║ ██║███████╗██║ ██║ ██║ ╚██████╗██║ ██║╚██████╔╝███████║██║ ██║ + ╚═╝ ╚═╝ ╚═╝╚═╝ ╚═╝╚══════╝╚═╝ ╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝ ╚═════╝ ╚══════╝╚═╝ ╚═╝ + + All-in-one security agent daemon — v0.1.0 + +2026-08-01 06:06:07 [INFO] Scanning vulns for security issues... + +- Scanning files... +✔ Scanned 30 files + + Scan Results + ────────────────────────────────────────────────────────────────────── + 4 critical 4 high 1 medium 0 low + ────────────────────────────────────────────────────────────────────── + + CRITICAL AWS Access Key + File: secrets/aws-credentials-hardcoded.env:23 + Info: Possible AWS Access Key detected + Code: **************** + + CRITICAL Stripe Key + File: secrets/aws-credentials-hardcoded.env:36 + Info: Possible Stripe Key detected + Code: **************** + + CRITICAL GitHub Token + File: secrets/github-pat-in-code.js:25 + Info: Possible GitHub Token detected + Code: const GITHUB_TOKEN = '****************'; // VULNERABLE: CWE-798 + + CRITICAL Slack Token + File: secrets/slack-webhook-url.py:40 + Info: Possible Slack Token detected + Code: SLACK_BOT_TOKEN = "****************" # VULNERABLE: CWE-798 + + [HIGH] Sensitive File + File: secrets/aws-credentials-hardcoded.env:0 + Info: .env file found — may contain secrets + + [HIGH] Database URL + File: secrets/aws-credentials-hardcoded.env:30 + Info: Possible Database URL detected + Code: ****************://app_user:****************@db.example.invalid:5432/appdb + + [HIGH] Database URL + File: secrets/aws-credentials-hardcoded.env:32 + Info: Possible Database URL detected + Code: REDIS_URL=redis://:****************@cache.example.invalid:6379/0 + + [HIGH] Generic Secret + File: secrets/slack-webhook-url.py:41 + Info: Possible Generic Secret detected + Code: **************** = "****************" # VULNERABLE: CWE-798 + + [MEDIUM] Hex Token (32+) + File: secrets/slack-webhook-url.py:41 + Info: Possible Hex Token (32+) detected + Code: **************** = "****************" # VULNERABLE: CWE-798 + + ────────────────────────────────────────────────────────────────────── + 9 issue(s) found across 30 files +