From 01fa694b99c2a106cbad9f108c57a7e1fc480fda Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 1 Aug 2026 05:33:33 +0000 Subject: [PATCH 1/3] Install ThreatCrush from npm; drop the PRD's non-existent hosts The PRD's distribution URLs do not exist. Neither cli.threatcrush.com nor api.sh1pt.com resolves in public DNS, so every run so far took the failure path and scanned nothing. ThreatCrush actually ships as an npm package, @profullstack/threatcrush (bin: threatcrush), which is what https://threatcrush.com/install.sh installs after bootstrapping mise and Node. In CI that bootstrap is redundant, so the workflow now runs npm install -g directly and keeps the official installer as a fallback. Pinned to Node 20: the CLI depends on better-sqlite3, and 20 is the newest runtime with reliable prebuilds. On Node 24 the install falls through to a node-gyp source build, which is how this was diagnosed locally. Also adds a 'Record the CLI interface' step. The scan flags in this workflow came from the PRD and have never been checked against a real binary, so the run log now prints --help for both the CLI and its scan subcommand. Correct the scan step from that evidence rather than from the PRD. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/threatcrush-scan.yml | 49 +++++++++++++--------- docs/SCANNER_INTEGRATION.md | 56 ++++++++++++++++++++------ 2 files changed, 74 insertions(+), 31 deletions(-) diff --git a/.github/workflows/threatcrush-scan.yml b/.github/workflows/threatcrush-scan.yml index 8e2e29f..1634489 100644 --- a/.github/workflows/threatcrush-scan.yml +++ b/.github/workflows/threatcrush-scan.yml @@ -32,6 +32,7 @@ concurrency: cancel-in-progress: true env: + THREATCRUSH_PACKAGE: '@profullstack/threatcrush' THREATCRUSH_VERSION: latest SARIF_FILE: threatcrush-results.sarif @@ -59,6 +60,14 @@ jobs: with: python-version: '3.11' + # Node 20, not latest. The CLI depends on better-sqlite3, a native module; + # 20 is the newest runtime with reliable prebuilt binaries, so the install + # does not fall back to a node-gyp source build. + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: '20' + - name: Validate test-case submissions id: lint run: bash scripts/validate-test-case.sh @@ -69,32 +78,36 @@ jobs: set -uo pipefail echo "scanner_available=false" >> "$GITHUB_OUTPUT" - # Preferred source: the sh1pt artifact registry (PRD §10.1). - if [ -n "${SH1PT_TOKEN:-}" ] && curl -fsSL --max-time 60 \ - -H "Authorization: Bearer ${SH1PT_TOKEN}" \ - "https://api.sh1pt.com/v1/scanners/threatcrush/${THREATCRUSH_VERSION}" \ - -o threatcrush-cli.tar.gz; then - tar -xzf threatcrush-cli.tar.gz - echo "$PWD" >> "$GITHUB_PATH" + # ThreatCrush ships as an npm package. https://threatcrush.com/install.sh + # bootstraps mise + node before running the same `npm i -g`, which is + # redundant here because setup-node already provided a runtime. + if npm install -g "${THREATCRUSH_PACKAGE}@${THREATCRUSH_VERSION}"; then echo "scanner_available=true" >> "$GITHUB_OUTPUT" - echo "installed ThreatCrush from sh1pt" + echo "installed ${THREATCRUSH_PACKAGE}@${THREATCRUSH_VERSION} from npm" - # Fallback: the public install script (PRD §7.1). - elif curl -fsSL --max-time 60 https://cli.threatcrush.com/install.sh -o install.sh; then - bash install.sh + # Fallback: the official installer, in case the package layout changes. + elif curl -fsSL --max-time 120 https://threatcrush.com/install.sh | sh; then echo "scanner_available=true" >> "$GITHUB_OUTPUT" - echo "installed ThreatCrush from cli.threatcrush.com" + echo "installed ThreatCrush via threatcrush.com/install.sh" else - echo "::warning title=ThreatCrush unavailable::Could not fetch the CLI from \ - api.sh1pt.com or cli.threatcrush.com. The scan step will be skipped and this \ - run will report zero findings. This is an infrastructure problem, not a \ - detection result." + echo "::warning title=ThreatCrush unavailable::Could not install \ + ${THREATCRUSH_PACKAGE} from npm or threatcrush.com/install.sh. The scan step \ + will be skipped and this run will report zero findings. This is an \ + infrastructure problem, not a detection result." fi command -v threatcrush >/dev/null 2>&1 && threatcrush --version || true - env: - SH1PT_TOKEN: ${{ secrets.SH1PT_TOKEN }} + + # The PRD assumed this CLI's flags; they have never been verified against a + # real binary. Print the interface so the log is the source of truth, and + # the scan step below can be corrected from evidence rather than guesswork. + - name: Record the CLI interface + if: steps.install.outputs.scanner_available == 'true' + continue-on-error: true + run: | + echo "::group::threatcrush --help"; threatcrush --help || true; echo "::endgroup::" + echo "::group::threatcrush scan --help"; threatcrush scan --help || true; echo "::endgroup::" - name: Run ThreatCrush Scan id: scan diff --git a/docs/SCANNER_INTEGRATION.md b/docs/SCANNER_INTEGRATION.md index f16a69f..24cb2d3 100644 --- a/docs/SCANNER_INTEGRATION.md +++ b/docs/SCANNER_INTEGRATION.md @@ -214,22 +214,52 @@ checks out untrusted code. --- -## sh1pt.com artifact integration +## Installing the scanner -The workflow prefers the sh1pt artifact registry (PRD §10.1) and falls back to -the public installer: +ThreatCrush is distributed as an npm package, not a standalone binary: + +``` +@profullstack/threatcrush → bin: threatcrush +``` + +The workflow installs it directly: ```yaml -curl -fsSL -H "Authorization: Bearer ${SH1PT_TOKEN}" \ - "https://api.sh1pt.com/v1/scanners/threatcrush/${THREATCRUSH_VERSION}" \ - -o threatcrush-cli.tar.gz +npm install -g "@profullstack/threatcrush@latest" ``` -Set `SH1PT_TOKEN` and `THREATCRUSH_API_KEY` as repository secrets. +and falls back to the official installer, `https://threatcrush.com/install.sh`, +if the package layout ever changes. That script does the same `npm i -g` after +bootstrapping mise and Node, which is redundant in CI where `setup-node` has +already provided a runtime. + +**Node 20, deliberately.** The CLI depends on `better-sqlite3`, a native module. +Node 20 is the newest runtime with reliable prebuilt binaries for it; on Node 24 +the install falls through to a `node-gyp` source build, which fails without a +full toolchain. If you bump the Node version, verify the install still succeeds +before trusting a run. + +Set `THREATCRUSH_API_KEY` as a repository secret if the scanner needs one. + +### The PRD's URLs do not exist + +The PRD (§7.1, §10.1) specified `cli.threatcrush.com/install.sh` and +`api.sh1pt.com/v1/scanners/…`. Neither hostname resolves in public DNS — both +were removed from the workflow on 2026-08-01 and replaced with the npm install +above. `threatcrush.com` and `sh1pt.com` themselves do resolve; only those +subdomains are absent. + +### The CLI's flags are unverified + +The scan invocation in the workflow — + +``` +threatcrush scan --format sarif --output … --config … --fail-on critical,high +``` -> **Status check.** As of 2026-08-01 neither `cli.threatcrush.com` nor -> `api.sh1pt.com` resolves in public DNS (`threatcrush.com` and `sh1pt.com` -> themselves do). Until those hosts exist, every run takes the fallback path: -> the workflow writes an empty SARIF, reports "scanner did not run", and fails -> the job at the final step so a green check never overstates what happened. -> Confirm the real CLI distribution URL and update the install step. +— comes from the PRD, and has never been checked against a real binary. The +workflow therefore includes a **Record the CLI interface** step that prints +`threatcrush --help` and `threatcrush scan --help` into the run log. Read that +log and correct the scan step from the evidence before relying on any coverage +number. If the flags differ, the scan step fails soft, so the run still reports +"scanner did not run" rather than a misleading zero-findings result. From e73a3f4df4ac89615fb9a1540837eee781a48d46 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 1 Aug 2026 06:05:43 +0000 Subject: [PATCH 2/3] Use the CLI's real scan interface and stop green-checking empty scans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run 30686062988 installed the scanner successfully, then the scan died with 'error: unknown option --format' — and the job still went green. Two separate faults, both fixed here. 1. The scan invocation was wrong. Confirmed from the published bundle: .command("scan").argument("[path]", "Path to scan", ".") No options exist. --format/--output/--config/--fail-on/--verbose are all inventions of the PRD, and scan takes a PATH, not a pull request URL. That settles PRD Open Question 1: there is no native PR-level scanning. Diff-only scoping now happens in the workflow, which scans just the changed files under vulns/ on a pull request. 2. The failsafe only checked whether the CLI installed, so a scan that ran and failed still passed. It now requires that the scan produced usable SARIF. A green check must mean the corpus was actually scanned. Because the CLI cannot emit SARIF, scripts/threatcrush-to-sarif.py translates its output. It fails closed: if the format is unrecognised it exits non-zero and dumps the first 40 lines rather than emitting empty SARIF, since '0 findings' is indistinguishable from a clean scan and would be a silent false negative on a repository that exists to contain findings. Handles JSON output too, in case the CLI gains it. Verified across six paths: text, JSON, clean-scan, unparseable, missing input, and end-to-end scoring of converted SARIF through validate-coverage.py. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/threatcrush-scan.yml | 69 ++++++-- scripts/threatcrush-to-sarif.py | 209 +++++++++++++++++++++++++ 2 files changed, 265 insertions(+), 13 deletions(-) create mode 100755 scripts/threatcrush-to-sarif.py diff --git a/.github/workflows/threatcrush-scan.yml b/.github/workflows/threatcrush-scan.yml index 1634489..b23c043 100644 --- a/.github/workflows/threatcrush-scan.yml +++ b/.github/workflows/threatcrush-scan.yml @@ -109,22 +109,49 @@ jobs: echo "::group::threatcrush --help"; threatcrush --help || true; echo "::endgroup::" echo "::group::threatcrush scan --help"; threatcrush scan --help || true; echo "::endgroup::" + # The real interface, confirmed from the CLI bundle: + # + # .command("scan").argument("[path]", "Path to scan", ".") + # + # No options at all. The PRD's --format/--output/--config/--fail-on do not + # exist, and it scans a PATH, not a pull request URL — which answers PRD + # Open Question 1: no, there is no native PR-level scanning. + # + # Diff-only scoping is therefore done here rather than by the scanner: on a + # pull request, scan just the changed files. - name: Run ThreatCrush Scan id: scan if: steps.install.outputs.scanner_available == 'true' run: | - PR_URL="${{ github.event.pull_request.html_url }}" - echo "Scanning: ${PR_URL:-$GITHUB_REF}" + set -uo pipefail + + TARGETS="vulns" + if [ -n "${BASE_SHA:-}" ]; then + CHANGED=$(git diff --name-only --diff-filter=d "$BASE_SHA" HEAD -- vulns | tr '\n' ' ') + [ -n "$CHANGED" ] && TARGETS="$CHANGED" + fi + echo "Scanning: $TARGETS" + + : > threatcrush-output.txt + STATUS=0 + for target in $TARGETS; do + echo "--- threatcrush scan $target ---" >> threatcrush-output.txt + threatcrush scan "$target" >> threatcrush-output.txt 2>&1 || STATUS=$? + done - threatcrush scan "${PR_URL:-.}" \ - --format sarif \ + echo "scan_exit=$STATUS" >> "$GITHUB_OUTPUT" + echo "::group::raw scanner output"; cat threatcrush-output.txt; echo "::endgroup::" + + # The CLI cannot emit SARIF, so the text output is converted here. + python3 scripts/threatcrush-to-sarif.py \ + --input threatcrush-output.txt \ --output "$SARIF_FILE" \ - --config .threatcrush.yml \ - --fail-on critical,high \ - --verbose + && echo "sarif_produced=true" >> "$GITHUB_OUTPUT" \ + || echo "sarif_produced=false" >> "$GITHUB_OUTPUT" continue-on-error: true env: THREATCRUSH_API_KEY: ${{ secrets.THREATCRUSH_API_KEY }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} - name: Ensure a SARIF file exists if: always() @@ -191,16 +218,32 @@ jobs: name: threatcrush-scan-${{ github.event.pull_request.number || github.run_id }} path: | ${{ env.SARIF_FILE }} + threatcrush-output.txt coverage-report.json coverage-report.md scan-summary.md if-no-files-found: warn retention-days: 30 - - name: Fail the job when the scanner could not run - if: always() && steps.install.outputs.scanner_available != 'true' + # A green check must mean "the corpus was actually scanned". Run 30686062988 + # went green while scanning nothing: the install succeeded, the scan then + # died on an unknown flag, and this gate only looked at the install. It now + # checks that the scan ran AND produced real SARIF. + - name: Fail the job when nothing was actually scanned + if: always() run: | - echo "::error title=Scan did not run::The ThreatCrush CLI could not be \ - installed, so this PR was never actually scanned. Fix the scanner source \ - before trusting a green check on this workflow." - exit 1 + FAILED=0 + + if [ "${{ steps.install.outputs.scanner_available }}" != "true" ]; then + echo "::error title=Scanner not installed::The ThreatCrush CLI could not be \ + installed, so nothing was scanned." + FAILED=1 + elif [ "${{ steps.scan.outputs.sarif_produced }}" != "true" ]; then + echo "::error title=Scan produced no results::The CLI installed but the scan \ + did not yield usable SARIF. The findings below are empty because the scan \ + failed, not because the corpus is clean. See the 'raw scanner output' group." + FAILED=1 + fi + + [ "$FAILED" = "1" ] && exit 1 + echo "corpus was scanned and SARIF was produced" diff --git a/scripts/threatcrush-to-sarif.py b/scripts/threatcrush-to-sarif.py new file mode 100755 index 0000000..33ea45b --- /dev/null +++ b/scripts/threatcrush-to-sarif.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Convert ThreatCrush CLI output into SARIF 2.1.0. + +The ThreatCrush CLI cannot emit SARIF. Its scan command takes a path and nothing +else — confirmed from the published bundle: + + .command("scan").argument("[path]", "Path to scan", ".") + +The PRD assumed ``--format sarif --output …``; those options do not exist. So the +scanner's human-readable output is captured and translated here, which is what +lets the rest of the pipeline (GitHub's Security tab, the coverage validator, the +PR comment) stay tool-agnostic. + +Fails closed. If the output cannot be recognised, this exits non-zero rather than +emitting an empty SARIF, because an empty result is indistinguishable from a +clean scan and would report as "0 findings" — a silent false negative on a +repository whose entire purpose is containing findings. The workflow turns that +non-zero exit into a failed job. + +Usage: + python3 scripts/threatcrush-to-sarif.py --input threatcrush-output.txt \\ + --output threatcrush-results.sarif +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional + +TOOL_NAME = "threatcrush" +TOOL_URI = "https://threatcrush.com" + +_ANSI_RE = re.compile(r"\x1b\[[0-9;]*[A-Za-z]") + +#: Source extensions present in this corpus, used to spot file references. +_EXT = r"(?:js|py|go|java|rb|env|txt|json|yml|yaml|npmrc)" + +#: "path/to/file.py:12:5", "path/to/file.py:12", "path/to/file.py" +_LOC_RE = re.compile(rf"((?:[\w.\-/]+/)*[\w.\-]+\.{_EXT})(?::(\d+))?(?::(\d+))?") + +_SEVERITY_WORDS = { + "critical": "error", + "high": "error", + "medium": "warning", + "moderate": "warning", + "low": "note", + "info": "note", + "informational": "note", +} + +#: Phrases a scanner uses to say it found nothing. Only these justify empty SARIF. +_CLEAN_MARKERS = ( + "no issues found", "no vulnerabilities", "no findings", "0 findings", + "0 issues", "no problems", "nothing found", "clean", "all clear", +) + + +def strip_ansi(text: str) -> str: + return _ANSI_RE.sub("", text) + + +def severity_of(line: str) -> str: + lowered = line.lower() + for word, level in _SEVERITY_WORDS.items(): + if re.search(rf"\b{word}\b", lowered): + return level + return "warning" + + +def cwe_of(line: str) -> Optional[str]: + match = re.search(r"CWE[- ](\d+)", line, re.IGNORECASE) + return f"CWE-{match.group(1)}" if match else None + + +def from_json(payload: Any) -> List[Dict[str, Any]]: + """Map a JSON document to findings, in case the CLI gains JSON output.""" + findings: List[Dict[str, Any]] = [] + + def walk(node: Any) -> None: + if isinstance(node, dict): + path = node.get("file") or node.get("path") or node.get("filename") + message = (node.get("message") or node.get("title") + or node.get("description") or node.get("rule")) + if path and message: + findings.append({ + "file": str(path).lstrip("./"), + "line": int(node.get("line") or node.get("startLine") or 1), + "message": str(message), + "level": _SEVERITY_WORDS.get(str(node.get("severity", "")).lower(), "warning"), + "rule": str(node.get("rule") or node.get("id") or "threatcrush.finding"), + }) + for value in node.values(): + walk(value) + elif isinstance(node, list): + for item in node: + walk(item) + + walk(payload) + return findings + + +def from_text(text: str) -> List[Dict[str, Any]]: + """Map human-readable output to findings by locating file references.""" + findings: List[Dict[str, Any]] = [] + for raw in text.splitlines(): + line = strip_ansi(raw).strip() + if not line or line.startswith("--- threatcrush scan "): + continue + match = _LOC_RE.search(line) + if not match: + continue + path, line_no, _col = match.groups() + # Ignore the tool talking about itself rather than about the corpus. + if path.startswith(("scripts/", "node_modules/")) or "threatcrush-output" in path: + continue + message = line[match.end():].strip(" \t:-–—") or line + cwe = cwe_of(line) + findings.append({ + "file": path.lstrip("./"), + "line": int(line_no) if line_no else 1, + "message": message, + "level": severity_of(line), + "rule": f"threatcrush.{cwe.lower()}" if cwe else "threatcrush.finding", + "cwe": cwe, + }) + return findings + + +def build_sarif(findings: List[Dict[str, Any]]) -> Dict[str, Any]: + rules: Dict[str, Dict[str, Any]] = {} + results = [] + for finding in findings: + rule_id = finding["rule"] + if rule_id not in rules: + rule: Dict[str, Any] = {"id": rule_id, + "shortDescription": {"text": finding["message"][:120]}} + if finding.get("cwe"): + rule["properties"] = {"tags": [finding["cwe"]]} + rules[rule_id] = rule + results.append({ + "ruleId": rule_id, + "level": finding["level"], + "message": {"text": finding["message"]}, + "locations": [{ + "physicalLocation": { + "artifactLocation": {"uri": finding["file"]}, + "region": {"startLine": max(1, finding["line"])}, + } + }], + }) + return { + "version": "2.1.0", + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "runs": [{ + "tool": {"driver": {"name": TOOL_NAME, "informationUri": TOOL_URI, + "rules": list(rules.values())}}, + "results": results, + }], + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--allow-empty", action="store_true", + help="emit empty SARIF instead of failing when nothing parses") + args = parser.parse_args() + + if not args.input.exists(): + print(f"error: {args.input} does not exist — the scan step produced nothing", + file=sys.stderr) + return 1 + + raw = args.input.read_text(encoding="utf-8", errors="replace") + text = strip_ansi(raw) + + findings: List[Dict[str, Any]] = [] + try: + findings = from_json(json.loads(text)) + except json.JSONDecodeError: + findings = from_text(text) + + if not findings: + looks_clean = any(marker in text.lower() for marker in _CLEAN_MARKERS) + if not (looks_clean or args.allow_empty): + print("error: could not parse any finding, and the output does not say the " + "scan was clean.", file=sys.stderr) + print("Refusing to emit empty SARIF: '0 findings' would be indistinguishable " + "from a successful clean scan.", file=sys.stderr) + print(f"--- first 40 lines of {args.input} ---", file=sys.stderr) + for line in text.splitlines()[:40]: + print(f" {line}", file=sys.stderr) + return 1 + print("scanner reported a clean scan — emitting empty SARIF") + + args.output.write_text(json.dumps(build_sarif(findings), indent=2) + "\n", + encoding="utf-8") + print(f"wrote {args.output} with {len(findings)} finding(s)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 7f795530915d3689d419f6ded08354e0e7c23a32 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 1 Aug 2026 06:08:56 +0000 Subject: [PATCH 3/3] Parse ThreatCrush's real output format and fix path mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run 30687082490 finally produced real scanner output: 9 findings (4 critical, 4 high, 1 medium), all secrets. Two defects showed up in how they were mapped. 1. Path mapping. The CLI reports paths relative to the directory it was given, so scanning 'vulns' yields 'secrets/aws-credentials-hardcoded.env'. Those never matched the catalog's repository-relative paths, so all 9 findings scored as 'outside corpus' and the true positive rate stayed at 0% despite the scan working. --path-prefix restores the prefix. 2. Output parsing. The output is a multi-line block per finding — a severity and title line, then 'File:', then 'Info:' — not one finding per line. Severity is bare for CRITICAL and bracketed for the rest. 'Code:' lines are skipped: they are redacted excerpts, and matching them double-counted every finding. Whole-file findings report line :0, which SARIF rejects; clamped. The scan step now always scans the whole corpus rather than a changed-file list, so the prefix cannot vary between invocations. The corpus is 31 files and scans in seconds, so diff-only scoping bought nothing but ambiguity. Real output is committed as tests/fixtures/threatcrush-scan-output.txt so the parser has a regression test against actual CLI behaviour rather than a guess. Measured result: 15.58% TPR (12/77), 0 false positives, 0 unattributed. That is a genuine reading — ThreatCrush is a secrets scanner, and it catches the credential fixtures while not attempting the code-level vulnerabilities. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/threatcrush-scan.yml | 41 ++++++------ scripts/threatcrush-to-sarif.py | 77 ++++++++++++++++++++-- tests/fixtures/threatcrush-scan-output.txt | 68 +++++++++++++++++++ 3 files changed, 160 insertions(+), 26 deletions(-) create mode 100644 tests/fixtures/threatcrush-scan-output.txt diff --git a/.github/workflows/threatcrush-scan.yml b/.github/workflows/threatcrush-scan.yml index b23c043..255a99a 100644 --- a/.github/workflows/threatcrush-scan.yml +++ b/.github/workflows/threatcrush-scan.yml @@ -125,33 +125,30 @@ jobs: run: | set -uo pipefail - TARGETS="vulns" - if [ -n "${BASE_SHA:-}" ]; then - CHANGED=$(git diff --name-only --diff-filter=d "$BASE_SHA" HEAD -- vulns | tr '\n' ' ') - [ -n "$CHANGED" ] && TARGETS="$CHANGED" - fi - echo "Scanning: $TARGETS" - - : > threatcrush-output.txt - STATUS=0 - for target in $TARGETS; do - echo "--- threatcrush scan $target ---" >> threatcrush-output.txt - threatcrush scan "$target" >> threatcrush-output.txt 2>&1 || STATUS=$? - done - - echo "scan_exit=$STATUS" >> "$GITHUB_OUTPUT" + # Always scan the whole corpus, never a per-file list. The CLI reports + # paths relative to the directory it was given, so a fixed root keeps + # --path-prefix correct; a changed-files list would make the prefix + # vary per invocation. The corpus is 31 files and scans in seconds, so + # diff-only scoping buys nothing here. + SCAN_ROOT=vulns + echo "Scanning: $SCAN_ROOT" + + threatcrush scan "$SCAN_ROOT" > threatcrush-output.txt 2>&1 + echo "scan_exit=$?" >> "$GITHUB_OUTPUT" echo "::group::raw scanner output"; cat threatcrush-output.txt; echo "::endgroup::" - # The CLI cannot emit SARIF, so the text output is converted here. - python3 scripts/threatcrush-to-sarif.py \ - --input threatcrush-output.txt \ - --output "$SARIF_FILE" \ - && echo "sarif_produced=true" >> "$GITHUB_OUTPUT" \ - || echo "sarif_produced=false" >> "$GITHUB_OUTPUT" + # The CLI cannot emit SARIF, so its text output is converted here. + if python3 scripts/threatcrush-to-sarif.py \ + --input threatcrush-output.txt \ + --output "$SARIF_FILE" \ + --path-prefix "$SCAN_ROOT"; then + echo "sarif_produced=true" >> "$GITHUB_OUTPUT" + else + echo "sarif_produced=false" >> "$GITHUB_OUTPUT" + fi continue-on-error: true env: THREATCRUSH_API_KEY: ${{ secrets.THREATCRUSH_API_KEY }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} - name: Ensure a SARIF file exists if: always() diff --git a/scripts/threatcrush-to-sarif.py b/scripts/threatcrush-to-sarif.py index 33ea45b..7d41fbf 100755 --- a/scripts/threatcrush-to-sarif.py +++ b/scripts/threatcrush-to-sarif.py @@ -42,6 +42,22 @@ #: "path/to/file.py:12:5", "path/to/file.py:12", "path/to/file.py" _LOC_RE = re.compile(rf"((?:[\w.\-/]+/)*[\w.\-]+\.{_EXT})(?::(\d+))?(?::(\d+))?") +# --- ThreatCrush's own block format ----------------------------------------- +# Verified against real CLI output (tests/fixtures/threatcrush-scan-output.txt): +# +# CRITICAL AWS Access Key +# File: secrets/aws-credentials-hardcoded.env:23 +# Info: Possible AWS Access Key detected +# Code: **************** +# +# Severity appears bare for CRITICAL and bracketed for the rest ([HIGH], +# [MEDIUM], [LOW]). "Code:" is skipped — it is a redacted excerpt, not a +# location, and matching it would double-count every finding. +_BLOCK_HEADER_RE = re.compile( + r"^\s*\[?(CRITICAL|HIGH|MEDIUM|LOW|INFO)\]?\s{1,}(\S.*?)\s*$") +_BLOCK_FILE_RE = re.compile(r"^\s*File:\s*(\S+?)(?::(\d+))?\s*$") +_BLOCK_INFO_RE = re.compile(r"^\s*Info:\s*(\S.*?)\s*$") + _SEVERITY_WORDS = { "critical": "error", "high": "error", @@ -103,12 +119,54 @@ def walk(node: Any) -> None: return findings +def from_blocks(text: str) -> List[Dict[str, Any]]: + """Parse ThreatCrush's native block format. + + Each finding spans several lines: a severity/title header, a ``File:`` line + carrying the location, and an optional ``Info:`` description. + """ + findings: List[Dict[str, Any]] = [] + severity, title = "MEDIUM", "Finding" + + for raw in text.splitlines(): + line = strip_ansi(raw) + + header = _BLOCK_HEADER_RE.match(line) + if header and not line.strip().startswith(("File:", "Info:", "Code:")): + severity, title = header.group(1).upper(), header.group(2) + continue + + located = _BLOCK_FILE_RE.match(line) + if located: + path, line_no = located.groups() + findings.append({ + "file": path.lstrip("./"), + # The CLI reports :0 for whole-file findings; SARIF wants >= 1. + "line": max(1, int(line_no) if line_no else 1), + "message": title, + "level": _SEVERITY_WORDS.get(severity.lower(), "warning"), + "rule": f"threatcrush.{re.sub(r'[^a-z0-9]+', '-', title.lower()).strip('-')}", + "cwe": cwe_of(title), + }) + continue + + info = _BLOCK_INFO_RE.match(line) + if info and findings: + findings[-1]["message"] = f"{findings[-1]['message']}: {info.group(1)}" + + return findings + + def from_text(text: str) -> List[Dict[str, Any]]: - """Map human-readable output to findings by locating file references.""" + """Fallback: locate file references on single lines. + + Used only when the block parser finds nothing, so an unrecognised future + output format still has a chance of being understood. + """ findings: List[Dict[str, Any]] = [] for raw in text.splitlines(): line = strip_ansi(raw).strip() - if not line or line.startswith("--- threatcrush scan "): + if not line or line.startswith("--- threatcrush scan ") or line.startswith("Code:"): continue match = _LOC_RE.search(line) if not match: @@ -121,7 +179,7 @@ def from_text(text: str) -> List[Dict[str, Any]]: cwe = cwe_of(line) findings.append({ "file": path.lstrip("./"), - "line": int(line_no) if line_no else 1, + "line": max(1, int(line_no) if line_no else 1), "message": message, "level": severity_of(line), "rule": f"threatcrush.{cwe.lower()}" if cwe else "threatcrush.finding", @@ -168,6 +226,11 @@ def main() -> int: formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--input", required=True, type=Path) parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--path-prefix", default="", + help="prepended to reported paths that lack it. The CLI reports " + "paths relative to the directory it was given, so scanning " + "'vulns' yields 'secrets/x.env'; pass --path-prefix vulns to " + "restore the repository-relative path the catalog uses.") parser.add_argument("--allow-empty", action="store_true", help="emit empty SARIF instead of failing when nothing parses") args = parser.parse_args() @@ -184,7 +247,13 @@ def main() -> int: try: findings = from_json(json.loads(text)) except json.JSONDecodeError: - findings = from_text(text) + findings = from_blocks(text) or from_text(text) + + prefix = args.path_prefix.strip("/") + if prefix: + for finding in findings: + if not finding["file"].startswith(f"{prefix}/"): + finding["file"] = f"{prefix}/{finding['file']}" if not findings: looks_clean = any(marker in text.lower() for marker in _CLEAN_MARKERS) diff --git a/tests/fixtures/threatcrush-scan-output.txt b/tests/fixtures/threatcrush-scan-output.txt new file mode 100644 index 0000000..bb7f933 --- /dev/null +++ b/tests/fixtures/threatcrush-scan-output.txt @@ -0,0 +1,68 @@ +--- threatcrush scan vulns --- + + ████████╗██╗ ██╗██████╗ ███████╗ █████╗ ████████╗ ██████╗██████╗ ██╗ ██╗███████╗██╗ ██╗ + ╚══██╔══╝██║ ██║██╔══██╗██╔════╝██╔══██╗╚══██╔══╝██╔════╝██╔══██╗██║ ██║██╔════╝██║ ██║ + ██║ ███████║██████╔╝█████╗ ███████║ ██║ ██║ ██████╔╝██║ ██║███████╗███████║ + ██║ ██╔══██║██╔══██╗██╔══╝ ██╔══██║ ██║ ██║ ██╔══██╗██║ ██║╚════██║██╔══██║ + ██║ ██║ ██║██║ ██║███████╗██║ ██║ ██║ ╚██████╗██║ ██║╚██████╔╝███████║██║ ██║ + ╚═╝ ╚═╝ ╚═╝╚═╝ ╚═╝╚══════╝╚═╝ ╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝ ╚═════╝ ╚══════╝╚═╝ ╚═╝ + + All-in-one security agent daemon — v0.1.0 + +2026-08-01 06:06:07 [INFO] Scanning vulns for security issues... + +- Scanning files... +✔ Scanned 30 files + + Scan Results + ────────────────────────────────────────────────────────────────────── + 4 critical 4 high 1 medium 0 low + ────────────────────────────────────────────────────────────────────── + + CRITICAL AWS Access Key + File: secrets/aws-credentials-hardcoded.env:23 + Info: Possible AWS Access Key detected + Code: **************** + + CRITICAL Stripe Key + File: secrets/aws-credentials-hardcoded.env:36 + Info: Possible Stripe Key detected + Code: **************** + + CRITICAL GitHub Token + File: secrets/github-pat-in-code.js:25 + Info: Possible GitHub Token detected + Code: const GITHUB_TOKEN = '****************'; // VULNERABLE: CWE-798 + + CRITICAL Slack Token + File: secrets/slack-webhook-url.py:40 + Info: Possible Slack Token detected + Code: SLACK_BOT_TOKEN = "****************" # VULNERABLE: CWE-798 + + [HIGH] Sensitive File + File: secrets/aws-credentials-hardcoded.env:0 + Info: .env file found — may contain secrets + + [HIGH] Database URL + File: secrets/aws-credentials-hardcoded.env:30 + Info: Possible Database URL detected + Code: ****************://app_user:****************@db.example.invalid:5432/appdb + + [HIGH] Database URL + File: secrets/aws-credentials-hardcoded.env:32 + Info: Possible Database URL detected + Code: REDIS_URL=redis://:****************@cache.example.invalid:6379/0 + + [HIGH] Generic Secret + File: secrets/slack-webhook-url.py:41 + Info: Possible Generic Secret detected + Code: **************** = "****************" # VULNERABLE: CWE-798 + + [MEDIUM] Hex Token (32+) + File: secrets/slack-webhook-url.py:41 + Info: Possible Hex Token (32+) detected + Code: **************** = "****************" # VULNERABLE: CWE-798 + + ────────────────────────────────────────────────────────────────────── + 9 issue(s) found across 30 files +