diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index e791e0380..40bfeb17e 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -313,11 +313,13 @@ overrides, without starting Codex or contacting the network. `core.hooksPath`, does not replace an existing hook, and blocks high-severity findings or failed scans. Set `--fail-on-severity` to change the threshold. -`--path` scopes a scan to one or more paths, `--diff` scans committed changes, -and `--working-tree` scans staged and unstaged changes. Deep scans support -repository and path targets. The output directory must be outside the scanned -directory and any enclosing Git worktree. When SARIF is produced, it is written -to +`--path` selects one or more files or directories. A completed scoped scan +covers those paths, not the rest of the repository. Human-readable output +JSON-quotes ambiguous path names; JSON results retain the original strings. +`--diff` scans committed changes, and `--working-tree` scans staged and +unstaged changes. Deep scans support repository and path targets. The output +directory must be outside the scanned directory and any enclosing Git worktree. +When SARIF is produced, it is written to `/exports/results.sarif`. Working-tree snapshots include files from untracked nested Git repositories. @@ -378,10 +380,17 @@ the destination without moving files. Scans are report-only by default. Use `--fail-on-severity` in CI to exit 1 when a completed scan contains a finding at or above the selected severity. -Incomplete coverage and CLI/runtime errors exit 2 so they cannot be mistaken -for a passing policy. Incomplete scans still write the available human or JSON -result to stdout and a coverage warning to stderr, including in report-only -mode. +Incomplete or unknown coverage of the requested scope and CLI/runtime errors +exit 2. A completed `scan . --path src/parser` exits 0 in report-only mode, +even if other paths were not selected. `coverage.deferred` records unfinished +requested work; `coverage.openQuestions` records optional follow-up. A +static-only review can be complete without live reproduction. + +Exit 2 can still include a valid partial report. Incomplete scans write the +available human or JSON result to stdout and a coverage warning to stderr, +including in report-only mode. Automation should check the manifest status, +`coverage.mode`, `coverage.completeness`, scope paths, and deferred work +instead of treating every exit 2 as a missing report. Use `--patch` to fix and verify confirmed findings after a complete scan. `--patch-severity high` selects high and critical findings; the default is low diff --git a/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json b/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json index 76688fe96..93934fc55 100644 --- a/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json +++ b/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "codex-security", - "version": "0.1.22", + "version": "0.1.23", "description": "Codex Security workflows for security scans, analysis, and investigation.", "author": { "name": "OpenAI" diff --git a/sdk/typescript/_bundled_plugin/examples/completed-scan/report.md b/sdk/typescript/_bundled_plugin/examples/completed-scan/report.md index 58834b65e..97be80153 100644 --- a/sdk/typescript/_bundled_plugin/examples/completed-scan/report.md +++ b/sdk/typescript/_bundled_plugin/examples/completed-scan/report.md @@ -10,7 +10,7 @@ The scan reviewed the canonical include paths and exclusions listed below. - Revision: deadbeef - Snapshot digest: codex-security-snapshot/v1:sha256:ed88f96a4c1a06603a41b3f261f59c3de2555c367ef6ad3bb8b9e483495d34eb - Inventory strategy: repository -- Included paths: . +- Included paths: `.` - Excluded paths: none - Runtime or test status: not recorded @@ -21,7 +21,7 @@ The scan reviewed the canonical include paths and exclusions listed below. | Reportable findings | 1 | | Severity mix | high: 1 | | Confidence mix | high: 1 | -| Coverage | complete | +| Coverage | complete for requested scope | | Validation mode | not recorded | Canonical artifacts: `scan-manifest.json`, `findings.json`, and `coverage.json`. This report is a deterministic projection of those files. diff --git a/sdk/typescript/_bundled_plugin/mcp/server.mjs.br.part-000 b/sdk/typescript/_bundled_plugin/mcp/server.mjs.br.part-000 index 0d8b425ad..427e3f2bf 100644 Binary files a/sdk/typescript/_bundled_plugin/mcp/server.mjs.br.part-000 and b/sdk/typescript/_bundled_plugin/mcp/server.mjs.br.part-000 differ diff --git a/sdk/typescript/_bundled_plugin/mcp/server.mjs.br.part-001 b/sdk/typescript/_bundled_plugin/mcp/server.mjs.br.part-001 index d43827a78..4277194fc 100644 Binary files a/sdk/typescript/_bundled_plugin/mcp/server.mjs.br.part-001 and b/sdk/typescript/_bundled_plugin/mcp/server.mjs.br.part-001 differ diff --git a/sdk/typescript/_bundled_plugin/references/core-scan.md b/sdk/typescript/_bundled_plugin/references/core-scan.md index 5e66bfe21..9b2792c50 100644 --- a/sdk/typescript/_bundled_plugin/references/core-scan.md +++ b/sdk/typescript/_bundled_plugin/references/core-scan.md @@ -9,9 +9,9 @@ Perform one complete, evidence-backed security audit of the exact supplied repos 3. While the baseline runs, read `threat-model.md` once and obtain its independent architecture review within the available worker allowance. Verify its resource rows against their actual consumers, use the returned canonical `threatModel` as the generated model, and build source-backed investigation packets from it. Carry that object and its evidence into the final result instead of reconstructing a shorter summary. Preserve any user-supplied threat model unchanged as the authoritative security assumptions; map its real surfaces and controls without replacing it. 4. Group related source-backed security questions into investigation packets. Each group shares its plausible attacker, protected asset, entry points, expected controls, sensitive operations, component relationships, and actual repository-relative source anchors. Keep each question concrete, preserve distinct attacker boundaries and security mechanisms, and let investigators establish the detailed dataflow. 5. Launch focused investigator subagents with `fork_turns: "none"` as soon as useful packet groups exist. Choose their number and assignments from the amount, complexity, and independence of source-backed work, bounded by the supplied available subagent allowance; use fewer for related packets and more only when distinct surfaces justify them. Keep mapping other surfaces while they run. Send each only its focused-investigator prompt below, assigned packets, investigator perspective, repository path, authorized scope, any supplied scoped-source inventory, exact user context, supplied threat model, applicable packet-specific security guidance and its resolver command, the optional authoritative knowledge-base location, and verified search command. Do not include this reference or another worker's prompt. Supporting code may be outside a requested path, but an affected entry point, control, or operation must be in scope. -6. Reconcile source coverage before combining findings. Union only the baseline and focused investigators' `fully_reviewed_files` with files the parent fully security-audited, then intersect that set with the supplied authorized inventory or an inventory of the selected current scope. Architecture mapping alone and supporting files outside that inventory do not count toward completed audit coverage. Finish the remaining in-scope files in coherent groups, reusing available investigators within the same allowance. Inspect implementation-owning generated or compressed code as data. Do not add overlapping worker counts or claim that a search hit completed a file. Keep this one transient set; do not create a separate progress ledger or receipt format. If a user limit or unavailable source prevents completion, identify the actual remaining paths and report partial coverage. Then combine baseline and investigator findings once. Group observations only when they share the same broken security control and effective remediation; preserve every affected route, operation, sink, and supporting source location. Never merge different security failures solely because they share a CWE. +6. Reconcile source coverage before combining findings. Union only the baseline and focused investigators' `fully_reviewed_files` with files the parent fully security-audited, then intersect that set with the supplied authorized inventory or an inventory of the selected current scope. Architecture mapping alone and supporting files outside that inventory do not count toward completed audit coverage. Finish the remaining in-scope files in coherent groups, reusing available investigators within the same allowance. Inspect implementation-owning generated or compressed code as data. Do not add overlapping worker counts or claim that a search hit completed a file. Keep this one transient set; do not create a separate progress ledger or receipt format. If a user limit or unavailable source prevents review of the requested inventory, identify the actual remaining paths and report partial coverage. Then combine baseline and investigator findings once. Group observations only when they share the same broken security control and effective remediation; preserve every affected route, operation, sink, and supporting source location. Never merge different security failures solely because they share a CWE. 7. Independently validate each unique finding against local source once. Establish its attacker, entry point, trust boundary, attacker-controlled dataflow, transformations, broken control, sensitive operation, prerequisites, effective mitigations, strongest counterevidence, and concrete impact. Record concise, source-backed `rootCause.summary`, `validation.summary`, `attackPath.dataflow.summary`, and `attackPath.reachability.summary` alongside their supporting facts; determine impact, likelihood, and severity from those established facts. State optional configuration, dependency-version, or deployment prerequisites; do not require proof of a real deployment or runtime reproduction. A public library or parser boundary is sufficient when callers control the input. Reject only with source-backed counterevidence, preserve valid baseline findings, record material unresolved proof gaps, and apply the severity rules below. -8. Assemble complete semantic `scope`, `threatModel`, `findings`, and `coverage` using the plugin's `examples/completed-scan/` and `schemas/` as shape references, never as values to copy. Use the canonical field mapping and scenario reconciliation in `threat-model.md`, preserving supplied models unchanged and retaining source-backed architecture, capability, deployment, and uncertainty facts. Give each finding a stable lowercase vulnerability-family `ruleId`, its precise `taxonomy.category` and `taxonomy.cwe` values, genuine `provenance.source`, an instance when separately reported findings would otherwise collide, a `root_control` location when identifiable, all materially affected locations, calibrated severity and rationale, confidence and rationale, verified nonempty source evidence, attacker-to-sink reachability, and practical remediation. Use actual coverage surface labels and dispositions; report reviewed surfaces, explicit exclusions, deferred work, and unresolved questions honestly, and mark coverage `complete` only when the requested source scope was actually reviewed. Preserve every genuine finding, evidence item, user-supplied assumption, and unresolved proof gap in the caller's complete semantic result. +8. Assemble complete semantic `scope`, `threatModel`, `findings`, and `coverage` using the plugin's `examples/completed-scan/` and `schemas/` as shape references, never as values to copy. Use the canonical field mapping and scenario reconciliation in `threat-model.md`, preserving supplied models unchanged and retaining source-backed architecture, capability, deployment, and uncertainty facts. Give each finding a stable lowercase vulnerability-family `ruleId`, its precise `taxonomy.category` and `taxonomy.cwe` values, genuine `provenance.source`, an instance when separately reported findings would otherwise collide, a `root_control` location when identifiable, all materially affected locations, calibrated severity and rationale, confidence and rationale, verified nonempty source evidence, attacker-to-sink reachability, and practical remediation. Use actual coverage surface labels and dispositions. Coverage is `complete` when the requested source was reviewed and no essential in-scope proof gap remains. Unselected source does not make a scoped review partial. Put unfinished requested work in `coverage.deferred` or a `needs_follow_up` surface. Put optional deployment checks, broader investigations, and additional assurance in `coverage.openQuestions`; record the agreed validation method and limitations in `scope`. Never reclassify a real blocker as optional to claim completion. Preserve every genuine finding, evidence item, user-supplied assumption, and unresolved proof gap in the caller's complete semantic result. Keep discovery, validation, and attack-path reasoning within this one self-contained audit; do not invoke separate phase skills. Do not create ranking phases, per-file or per-candidate ledgers, separate phase worker pools, repeated phase reports, or receipt files. diff --git a/sdk/typescript/_bundled_plugin/references/final-report.md b/sdk/typescript/_bundled_plugin/references/final-report.md index 03dcd6880..c53f6f1c6 100644 --- a/sdk/typescript/_bundled_plugin/references/final-report.md +++ b/sdk/typescript/_bundled_plugin/references/final-report.md @@ -20,7 +20,7 @@ Every scan mode uses the same final report pipeline. Workbench-owned Standard an For each finding, supply an evidence-supported lowercase vulnerability-family `ruleId`; `taxonomy: { category, cwe }` using its exact known CWEs; verified locations; and `provenance.source`, using `"local_plugin"` only when this plugin actually discovered the finding. Preserve genuine worker or source provenance and any existing canonical candidate identity in the finding extensions. A finding with no known CWE retains `cwe: []`; never invent a classification. Include optional `codeEvidence` only when its actual code is nonempty and every referenced evidence ID is present. -Supply semantic coverage as `{ completeness, surfaces, explicitExclusions, deferred }`, with each surface using the actual `label` and one existing `disposition`. Mark coverage `partial` when a deferred item or `needs_follow_up` surface remains; preserve its real reason and supporting context. Each deferred item needs a meaningful reason; preserve any existing `id` or `candidateId`. The workbench derives a missing ID from its candidate identity or stable deferred-work details. Open questions may be nonempty strings or `{ question, followUpPrompt? }` objects. The workbench derives target and scope metadata, scope include and exclude paths, coverage mode and inventory strategy, finding identities and fingerprints, and surface IDs. Do not put those workbench-owned values or top-level coverage receipt references into the semantic draft. +Supply semantic coverage as `{ completeness, surfaces, explicitExclusions, deferred, openQuestions? }`, with each surface using the actual `label` and one existing `disposition`. Completeness applies to the authorized requested review. A finished path-scoped review is complete for those paths. Deferred items and `needs_follow_up` surfaces mean requested work or an essential in-scope proof gap remains, so coverage must be `partial`. Preserve their reasons and supporting context. Record optional follow-up, such as deployment confirmation or a broader review, in `openQuestions`. Record the agreed validation method, runtime status, and limitations in `scan.scope`. Do not invent exclusions or move blockers to `openQuestions` to claim completion. Each deferred item needs a meaningful reason; preserve any existing `id` or `candidateId`. The workbench derives a missing ID from its candidate identity or stable deferred-work details. Open questions may be nonempty strings or `{ question, followUpPrompt? }` objects. The workbench derives target and scope metadata, scope include and exclude paths, coverage mode and inventory strategy, finding identities and fingerprints, and surface IDs. Do not put those workbench-owned values or top-level coverage receipt references into the semantic draft. After a workbench-owned Standard or workbench-backed diff draft is accepted, or the Deep coordinator returns its parent scan's canonical manifest, call `complete_codex_security_scan({ scanId, handoffClaimToken? })` and use its returned completion metadata. An SDK-owned scan returns its unsealed canonical files without calling a completion tool or finalizer; the SDK owns completion and report generation. Read full canonical results only when explicitly requested. For a terminal/chat workflow without a `scanId` or completion tool, retain `python /scripts/finalize_scan_contract.py --scan-dir --source-root ` after writing the canonical JSON. Outside the SDK path, do not mark the scan goal complete until finalization succeeds and the generated report exists. @@ -150,11 +150,11 @@ Recommended outcomes: - `No issue found`: reviewed and no credible issue survived. - `Rejected`: plausible-looking candidate was ruled out with specific counterevidence. - `Not applicable`: the risk class does not apply to that surface. -- `Needs follow-up`: plausible but not fully closed because of a concrete blocker or proof gap. +- `Needs follow-up`: essential in-scope review remains unresolved because of a concrete blocker or proof gap; this blocks complete coverage. Write the same content, or a slightly more detailed version, to `/reviewed_surfaces.md`. -For broad scans where the completed coverage is useful for triage but too large for high-precision review, include a concise `## Open Questions And Follow Up` section near the end of the report. Use concrete, copyable prompt ideas that narrow the next review to individual commits from the current scan. Do not include this section for precise scans where the requested scope was already sufficient. +The generated `## Incomplete Requested Work` section lists deferred work and unresolved in-scope surfaces. Put optional investigations in `## Open Questions And Follow Up`, including for an otherwise complete scoped scan. Use concrete, copyable prompts grounded in the selected source and findings. Do not invent follow-ups or broaden the current scan. Follow-up prompts should be tailored to the actual scan results: diff --git a/sdk/typescript/_bundled_plugin/schemas/coverage.schema.json b/sdk/typescript/_bundled_plugin/schemas/coverage.schema.json index 13138398e..7b856b56e 100644 --- a/sdk/typescript/_bundled_plugin/schemas/coverage.schema.json +++ b/sdk/typescript/_bundled_plugin/schemas/coverage.schema.json @@ -43,7 +43,8 @@ "complete", "partial", "unknown" - ] + ], + "description": "Completion of the requested review within the authorized scope, not whole-repository coverage or proof that no vulnerabilities exist. Use partial when requested work or an essential in-scope proof gap remains; use unknown when completion cannot be established." }, "inventoryStrategy": { "enum": [ @@ -92,7 +93,8 @@ "rejected", "not_applicable", "needs_follow_up" - ] + ], + "description": "The evidence-supported result for this in-scope surface. Use needs_follow_up only when an essential part of its requested review remains unresolved." }, "receiptRefs": { "type": "array", @@ -131,7 +133,8 @@ "minLength": 1 } } - } + }, + "description": "Source excluded by the requested scope or applicable authorization and policy. Do not invent exclusions to hide unfinished requested work." }, "deferred": { "type": "array", @@ -165,7 +168,8 @@ } } } - } + }, + "description": "Unfinished requested work or an essential unresolved in-scope proof gap. Every item blocks complete coverage. Put optional additional assurance in openQuestions instead; never move a real blocker there to claim completion." }, "openQuestions": { "type": "array", @@ -184,7 +188,8 @@ "minLength": 1 } } - } + }, + "description": "Optional additional assurance or later investigations that do not block completion of the requested review. Preserve essential unresolved in-scope proof gaps in deferred instead." } }, "allOf": [ diff --git a/sdk/typescript/_bundled_plugin/schemas/tools/scan-draft.schema.json b/sdk/typescript/_bundled_plugin/schemas/tools/scan-draft.schema.json index d32892223..c2be95ea2 100644 --- a/sdk/typescript/_bundled_plugin/schemas/tools/scan-draft.schema.json +++ b/sdk/typescript/_bundled_plugin/schemas/tools/scan-draft.schema.json @@ -667,7 +667,7 @@ "description": "The meaningful name of the reviewed security surface." }, "disposition": { - "description": "The evidence-supported review result for this surface.", + "description": "The evidence-supported result for this in-scope surface. Use needs_follow_up only when an essential part of its requested review remains unresolved.", "enum": [ "reported", "no_issue_found", @@ -708,7 +708,7 @@ "excludePaths": false, "receiptRefs": false, "completeness": { - "description": "Use partial if any work is deferred or any surface needs follow-up; use complete only when no such work remains.", + "description": "Completion of the requested review within the authorized scope, not whole-repository coverage or proof that no vulnerabilities exist. Use partial when requested work or an essential in-scope proof gap remains; use unknown when completion cannot be established.", "enum": [ "complete", "partial", @@ -739,7 +739,8 @@ "reason" ], "additionalProperties": true - } + }, + "description": "Source excluded by the requested scope or applicable authorization and policy. Do not invent exclusions to hide unfinished requested work." }, "deferred": { "type": "array", @@ -766,7 +767,8 @@ "reason" ], "additionalProperties": true - } + }, + "description": "Unfinished requested work or an essential unresolved in-scope proof gap. Every item blocks complete coverage. Put optional additional assurance in openQuestions instead; never move a real blocker there to claim completion." }, "openQuestions": { "type": "array", @@ -791,7 +793,8 @@ "additionalProperties": true } ] - } + }, + "description": "Optional additional assurance or later investigations that do not block completion of the requested review. Preserve essential unresolved in-scope proof gaps in deferred instead." } }, "required": [ diff --git a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py index d99436dbf..d5caf952f 100644 --- a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py +++ b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py @@ -2292,9 +2292,17 @@ def build_sarif_projection( sarif = build_sarif(manifest, findings, source_root) run = sarif["runs"][0] completeness = coverage["completeness"] + run["properties"].update( + { + "codexSecurityCoverageMode": coverage["mode"], + "codexSecurityCoverageCompleteness": completeness, + "codexSecurityIncludePaths": coverage["includePaths"], + "codexSecurityExcludePaths": coverage["excludePaths"], + "codexSecurityExplicitExclusions": coverage["explicitExclusions"], + } + ) run["invocations"] = [{"executionSuccessful": completeness == "complete"}] if completeness != "complete": - run["properties"]["codexSecurityCoverageCompleteness"] = completeness reasons = [item["reason"] for item in coverage["deferred"]] or [ f"Scan coverage is {completeness}; results may be incomplete." ] diff --git a/sdk/typescript/_bundled_plugin/scripts/report_projection.py b/sdk/typescript/_bundled_plugin/scripts/report_projection.py index 6149d5f4f..6cf1c7534 100644 --- a/sdk/typescript/_bundled_plugin/scripts/report_projection.py +++ b/sdk/typescript/_bundled_plugin/scripts/report_projection.py @@ -4,7 +4,9 @@ from __future__ import annotations import argparse +import json import re +import unicodedata from collections import Counter from typing import Any @@ -20,6 +22,19 @@ } WRITEUP_REPORT_PATH_RE = re.compile(r"^findings/([a-z0-9][a-z0-9._-]*)/\1\.md$") +SCOPE_PATH_QUOTING_RE = re.compile(r"""[\s,;'"\\\x00-\x1f]""") +SCOPE_PATH_NON_ASCII_RE = re.compile(r"[^\x20-\x7e]") +# Unicode 17 General_Category=Format, Default_Ignorable_Code_Point, C1 controls, +# and line separators. +SCOPE_PATH_CONTROLS_RE = re.compile( + r"[\x7f-\x9f\u2028\u2029\u00ad\u034f\u0600-\u0605\u061c\u06dd\u070f" + r"\u0890-\u0891\u08e2\u115f-\u1160\u17b4-\u17b5" + r"\u180b-\u180f\u200b-\u200f\u202a-\u202e\u2060-\u206f\u3164\ufe00-\ufe0f" + r"\ufeff\uffa0\ufff0-\ufffb\U000110bd\U000110cd\U00013430-\U0001343f" + r"\U0001bca0-\U0001bca3\U0001d173-\U0001d17a" + r"\U000e0000-\U000e0fff]" +) + class ReportProjectionError(ValueError): """Raised when a canonical scan cannot be projected into a valid report.""" @@ -59,6 +74,39 @@ def _strings(value: Any) -> list[str]: return normalized +def _scope_path(value: Any, fallback: str = "unspecified") -> str: + path = value if isinstance(value, str) else fallback + rendered = path + normalization_sensitive = unicodedata.normalize("NFC", path) != path + if ( + not path + or normalization_sensitive + or SCOPE_PATH_QUOTING_RE.search(path) + or SCOPE_PATH_CONTROLS_RE.search(path) + ): + escaped_characters = SCOPE_PATH_NON_ASCII_RE + if not normalization_sensitive: + escaped_characters = SCOPE_PATH_CONTROLS_RE + rendered = escaped_characters.sub( + lambda match: json.dumps(match.group(0))[1:-1], + json.dumps(path, ensure_ascii=False), + ) + longest_run = max( + (len(match.group(0)) for match in re.finditer(r"`+", rendered)), default=0 + ) + fence = "`" * (longest_run + 1) + padding = " " if rendered.startswith("`") or rendered.endswith("`") else "" + return f"{fence}{padding}{rendered}{padding}{fence}" + + +def _scope_paths(value: Any) -> list[str]: + if isinstance(value, str): + value = [value] + if not isinstance(value, list): + return [] + return [_scope_path(item) for item in value if isinstance(item, str)] + + def _cell(value: Any) -> str: return _text(value, "none").replace("|", "\\|").replace("\n", "
") @@ -780,6 +828,22 @@ def _linked_finding_section(number: int, finding: dict[str, Any], report_path: s return lines +def _append_questions(lines: list[str], heading: str, questions: list[Any]) -> None: + if not questions: + return + lines.extend(["", heading, ""]) + for question in questions: + if not isinstance(question, dict): + continue + lines.append(f"- {_text(question.get('question'), 'Unspecified open question.')}") + prompt = _text(question.get("followUpPrompt"), "") + if prompt: + lines.append(f" - Follow-up prompt: {prompt}") + paths = _scope_paths(question.get("paths")) + if paths: + lines.append(" - Paths: " + ", ".join(paths)) + + def build_report_markdown( manifest: dict[str, Any], findings_document: dict[str, Any], coverage: dict[str, Any] ) -> str: @@ -809,8 +873,8 @@ def build_report_markdown( deep_presentation = _uses_deep_presentation(coverage, findings) deep_finding_groups = _deep_finding_groups(findings, writeup_paths) if deep_presentation else [] hardening_portfolio_path = _hardening_portfolio_path(scan) - include_paths = _strings(coverage.get("includePaths", scope.get("includePaths", []))) - exclude_paths = _strings(coverage.get("excludePaths", scope.get("excludePaths", []))) + include_paths = _scope_paths(coverage.get("includePaths", scope.get("includePaths", []))) + exclude_paths = _scope_paths(coverage.get("excludePaths", scope.get("excludePaths", []))) limitations = _strings(scope.get("limitations")) explicit_exclusions = coverage.get("explicitExclusions", []) lines = [ @@ -830,7 +894,7 @@ def build_report_markdown( f"- Excluded paths: {', '.join(exclude_paths) or 'none'}", f"- Runtime or test status: {_text(scope.get('runtimeStatus'), 'not recorded')}", ] - artifacts_reviewed = _strings(scope.get("artifactsReviewed")) + artifacts_reviewed = _scope_paths(scope.get("artifactsReviewed")) if artifacts_reviewed: lines.extend(["- Artifacts reviewed: " + ", ".join(artifacts_reviewed)]) context = _text(scope.get("context"), "") @@ -839,7 +903,7 @@ def build_report_markdown( for exclusion in explicit_exclusions: if isinstance(exclusion, dict): limitations.append( - f"Excluded {_text(exclusion.get('pattern'), 'unspecified')}: " + f"Excluded {_scope_path(exclusion.get('pattern'))}: " f"{_text(exclusion.get('reason'), 'reason not recorded')}" ) if limitations: @@ -866,7 +930,7 @@ def build_report_markdown( "| Field | Value |", "| --- | --- |", *summary_count_lines, - f"| Coverage | {coverage['completeness']} |", + f"| Coverage | {coverage['completeness']} for requested scope |", f"| Validation mode | {_cell(scope.get('validationMode', 'not recorded'))} |", "", "Canonical artifacts: `scan-manifest.json`, `findings.json`, and `coverage.json`. This report is a deterministic projection of those files.", @@ -983,7 +1047,7 @@ def build_report_markdown( if no_source_review else ( "No findings were validated before the scan reached its cost limit. " - "Review the deferred candidates in Open Questions And Follow Up." + "Review the deferred candidates in Incomplete Requested Work." if budget_exhausted else "No reportable findings survived the canonical discovery, validation, " "and reportability gates." @@ -1003,6 +1067,7 @@ def build_report_markdown( ] ) surfaces = coverage.get("surfaces", []) + surfaces = surfaces if isinstance(surfaces, list) else [] if surfaces: lines.extend( [ @@ -1032,17 +1097,21 @@ def build_report_markdown( ) + " |" ) - open_questions = coverage.get("openQuestions", []) - questions = list(open_questions) if isinstance(open_questions, list) else [] deferred = coverage.get("deferred", []) - if isinstance(deferred, list): - questions.extend( + deferred = deferred if isinstance(deferred, list) else [] + blockers = [] + deferred_surface_ids = set() + for item in deferred: + if not isinstance(item, dict): + continue + deferred_surface_ids.update(item.get("surfaceIds", [])) + blockers.append( { - "question": item.get("reason", "Deferred review requires follow-up."), + "question": item.get("reason", "Requested review work remains unfinished."), + "paths": item.get("paths", []), "followUpPrompt": " ".join( ( f"Review deferred unit {item.get('id', 'unknown')} and close its stated proof gap.", - f"Paths: {', '.join(item.get('paths', []))}." if item.get("paths") else "", ( f"Surfaces: {', '.join(item.get('surfaceIds', []))}." if item.get("surfaceIds") @@ -1051,18 +1120,33 @@ def build_report_markdown( ) ).strip(), } - for item in deferred - if isinstance(item, dict) ) - if questions: - lines.extend(["", "## Open Questions And Follow Up", ""]) - for question in questions: - if not isinstance(question, dict): - continue - lines.append(f"- {_text(question.get('question'), 'Unspecified open question.')}") - prompt = _text(question.get("followUpPrompt"), "") - if prompt: - lines.append(f" - Follow-up prompt: {prompt}") + for surface in surfaces: + if ( + isinstance(surface, dict) + and surface.get("disposition") == "needs_follow_up" + and surface.get("id") not in deferred_surface_ids + ): + blockers.append( + { + "question": f"{surface.get('label', 'Unresolved surface')}: " + f"{surface.get('notes') or 'Essential in-scope review remains unfinished.'}" + } + ) + if not blockers and coverage["completeness"] != "complete": + blockers.append( + { + "question": f"Requested-scope coverage is {coverage['completeness']}; " + "no specific remaining work was recorded." + } + ) + _append_questions(lines, "## Incomplete Requested Work", blockers) + open_questions = coverage.get("openQuestions", []) + _append_questions( + lines, + "## Open Questions And Follow Up", + open_questions if isinstance(open_questions, list) else [], + ) return "\n".join(lines).rstrip() + "\n" diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py index 33332a490..82d950bbf 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py @@ -47,6 +47,7 @@ PRODUCER_NAME, ContractError, _prepare_scan_finalization, + _read_scan_local_json_bytes, _write_prepared_scan_finalization, csv_cell, finalize_scan, @@ -1793,6 +1794,56 @@ def get_scan_recipe(connection: sqlite3.Connection, args: argparse.Namespace) -> } +def coverage_summary_for_history(scan: sqlite3.Row) -> dict[str, Any]: + if scan["seal_manifest_digest"] is None: + raise SystemExit("Only sealed scans have coverage summaries.") + scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"])) + coverage_ref = ARTIFACTS["coverage"] + try: + # Completion already validated the full scan. History only needs the + # pinned manifest and its coverage artifact, not findings or receipts. + manifest, manifest_bytes = _read_scan_local_json_bytes( + scan_dir, ARTIFACTS["manifest"], ARTIFACTS["manifest"] + ) + if f"sha256:{hashlib.sha256(manifest_bytes).hexdigest()}" != scan["seal_manifest_digest"]: + raise ContractError("The sealed scan manifest changed after completion.") + sealed_scan = manifest["scan"] + if ( + sealed_scan["id"] != scan["id"] + or not sealed_scan.get("sealedAt") + or sealed_scan["coverageRef"] != coverage_ref + ): + raise ContractError("The sealed coverage does not belong to this scan.") + coverage, coverage_bytes = _read_scan_local_json_bytes( + scan_dir, coverage_ref, coverage_ref + ) + expected = next( + ( + artifact["sha256"] + for artifact in sealed_scan["artifacts"] + if artifact["path"] == coverage_ref + ), + None, + ) + if ( + hashlib.sha256(coverage_bytes).hexdigest() != expected + or coverage.get("scanId") != scan["id"] + ): + raise ContractError("The sealed coverage changed after completion.") + except ContractError as exc: + raise SystemExit(str(exc)) from exc + return { + key: coverage[key] + for key in ( + "mode", + "completeness", + "includePaths", + "excludePaths", + "explicitExclusions", + ) + } + + def coverage_for_comparison(scan: sqlite3.Row) -> dict[str, Any]: if scan["seal_manifest_digest"] is None: raise SystemExit("Only sealed scans can be compared.") @@ -3175,6 +3226,20 @@ def scan_context( return context +def get_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]: + result = scan_context(connection, args.scan_id, args.occurrence_id) + if result["scan"]["progress"]["status"] == "complete": + scan = require_scan(connection, args.scan_id) + try: + result["scan"] = { + **result["scan"], + "coverage": coverage_summary_for_history(scan), + } + except (OSError, RuntimeError, SystemExit): + pass # Historical artifacts may no longer be available or verifiable. + return result + + def list_findings(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]: scan = require_scan(connection, args.scan_id) backfill_legacy_finding_details(connection, scan) @@ -3874,7 +3939,7 @@ def main() -> None: elif args.command == "fail-deep-scan": result = deep_scan.fail_deep_scan(connection, args) elif args.command == "get-scan": - result = scan_context(connection, args.scan_id, args.occurrence_id) + result = get_scan(connection, args) elif args.command == "get-scan-feedback": result = get_scan_feedback(connection, require_scan(connection, args.scan_id)) elif args.command == "list-scans": diff --git a/sdk/typescript/_bundled_plugin/skills/security-scan/SKILL.md b/sdk/typescript/_bundled_plugin/skills/security-scan/SKILL.md index 423a68aa8..7a12b2e84 100644 --- a/sdk/typescript/_bundled_plugin/skills/security-scan/SKILL.md +++ b/sdk/typescript/_bundled_plugin/skills/security-scan/SKILL.md @@ -23,7 +23,7 @@ When an SDK or terminal host sets `CODEX_SECURITY_SCAN_ID`, emit its standalone 1. Resolve the repository, requested scope, and output scan directory from the host-provided scan context when available; otherwise use the requested output directory or `/codex-security-scans//`. Preserve the exact user context, supplied threat model, applicable inherited `SECURITY.md` guidance, and optional `CODEX_SECURITY_KNOWLEDGE_BASE` for the core audit. Resolve `` from the configured interpreter (`"$PYTHON"` in POSIX shells or `& "$env:PYTHON"` in PowerShell), otherwise use `python3` on Unix-like hosts or `python` on Windows. Only when `CODEX_SECURITY_TARGET_PATHS_FILE` is supplied, resolve every authorized source path before review with ` /scripts/generate_rank_input.py make-repo-scope-input --repo --scopes-file --out /scoped-source-input.jsonl`; use `"$CODEX_SECURITY_TARGET_PATHS_FILE"` in POSIX shells or `"$env:CODEX_SECURITY_TARGET_PATHS_FILE"` in PowerShell and honor repository ignore rules for directory descendants while retaining every directly requested file. Never print, modify, or treat the scope input as shell syntax; pass it to the core audit without widening the authorized target or scope. 2. Read `../../references/core-scan.md` once and perform its complete source-backed security audit against the resolved target, authorized scope, exact user context, supplied threat model, inherited security policy, optional knowledge base, available workers, and any resolved scoped-source inventory. Retain the resulting complete semantic `scope`, `threatModel`, `findings`, and `coverage`; preserve every finding's source evidence, calibrated severity, confidence, root cause, validation, attack path, and honest coverage. -3. For a host-backed scan, submit one accepted semantic draft with `record_codex_security_scan_draft({ scanId, handoffClaimToken?, scope?, threatModel, findings, coverage })`; let the workbench derive its authoritative target, scope, coverage metadata, surface IDs, finding identities, and fingerprints. If the draft is explicitly rejected before writing, correct only the identified fields without dropping valid findings or evidence and retry the same scan at most twice. For an SDK-owned or prompt-only headless scan, write unsealed canonical `scan-manifest.json`, `findings.json`, and `coverage.json`; use `scoped_path` for both coverage fields when a scope was requested, otherwise set `coverage.mode` to `repository` and `coverage.inventoryStrategy` to `directory` for a non-Git directory or `repository` for a Git-backed target. Omit `scan.sealedAt` and `scan.artifacts`; an SDK scan preserves its exact registered directory and all SDK-provided scan and target values. When `CODEX_SECURITY_TARGET_PATHS_FILE` is supplied on either file-authored path, bind its exact requested paths with ` /scripts/generate_rank_input.py bind-repo-scopes --scopes-file --manifest /scan-manifest.json --coverage /coverage.json`, using the same shell-specific target-paths reference. +3. For a host-backed scan, submit one accepted semantic draft with `record_codex_security_scan_draft({ scanId, handoffClaimToken?, scope?, threatModel, findings, coverage })`; let the workbench derive its authoritative target, scope, coverage metadata, surface IDs, finding identities, and fingerprints. If the draft is explicitly rejected before writing, correct only the identified fields without dropping valid findings or evidence and retry the same scan at most twice. For an SDK-owned or prompt-only headless scan, write unsealed canonical `scan-manifest.json`, `findings.json`, and `coverage.json`; use `scoped_path` for both coverage fields when a scope was requested and measure completeness against that requested scope, otherwise set `coverage.mode` to `repository` and `coverage.inventoryStrategy` to `directory` for a non-Git directory or `repository` for a Git-backed target. Omit `scan.sealedAt` and `scan.artifacts`; an SDK scan preserves its exact registered directory and all SDK-provided scan and target values. When `CODEX_SECURITY_TARGET_PATHS_FILE` is supplied on either file-authored path, bind its exact requested paths with ` /scripts/generate_rank_input.py bind-repo-scopes --scopes-file --manifest /scan-manifest.json --coverage /coverage.json`, using the same shell-specific target-paths reference. 4. Verify all three canonical JSON files exist. For an SDK-owned scan, return control without finalizing, sealing, generating `report.md`, or starting another scan; the SDK owns completion. For another host-backed scan, call `complete_codex_security_scan({ scanId, handoffClaimToken? })` once. For a prompt-only headless scan, run ` /scripts/finalize_scan_contract.py --scan-dir --source-root `. Outside the SDK path, return only after completion succeeds and the generated `report.md` exists; never write the report by hand or reread the complete canonical findings unless the user explicitly requests them. Report measured token counts when returned and label partial measurement or unavailable usage honestly. Keep discovery, validation, and attack-path reasoning within this Standard workflow; do not invoke separate phase skills or load Deep or diff references. Never call Deep-only tools. Do not create ranking phases, per-file or per-candidate ledgers, separate phase worker pools, repeated phase reports, or receipt files. diff --git a/sdk/typescript/scripts/check-package.mjs b/sdk/typescript/scripts/check-package.mjs index 05cd4c0b5..4b20f2ede 100644 --- a/sdk/typescript/scripts/check-package.mjs +++ b/sdk/typescript/scripts/check-package.mjs @@ -170,6 +170,7 @@ const distFiles = new Set( "contract", "cost", "cost-model", + "coverage-presentation", "custom-validation", "custom-validation-prompt", "errors", diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 5d23633e3..1b9acb5fb 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -74,6 +74,11 @@ import { type JsonValue, } from "./config.js"; import { formatUsd, type ScanCost } from "./cost.js"; +import { + formatCoverageCompleteness, + formatCoverageScope, + formatScopePath, +} from "./coverage-presentation.js"; import { CodexSecurityError, ConfigurationError, @@ -5590,7 +5595,7 @@ async function executeScan( reason: "Stop reason unavailable. See the report for details.", }; } - progress?.stage(`Scan complete · ${result.manifest.scan.id.slice(0, 8)}`); + progress?.stage(`Scan finished · ${result.manifest.scan.id.slice(0, 8)}`); printScanSummary( result, progress, @@ -5620,8 +5625,8 @@ async function executeScan( if (incomplete) { errorOutput.write( threshold === undefined - ? `codex-security: Scan coverage is ${result.coverage.completeness}; results may be incomplete.\n` - : `codex-security: Cannot evaluate the failure policy: coverage is ${result.coverage.completeness}.\n`, + ? `codex-security: Scan coverage is ${result.coverage.completeness} for the requested scope; see the report for unfinished work.\n` + : `codex-security: Cannot evaluate the failure policy: coverage is ${result.coverage.completeness} for the requested scope.\n`, ); return completedScan(2); } @@ -5838,7 +5843,7 @@ function scanScope(arguments_: ScanArguments): string | null { portable.startsWith("//") ? basename(portable) || portable : portable; - return errorMessage(scoped.replaceAll(/[\u0000-\u001F\u007F]/gu, " ")); + return errorMessage(formatScopePath(scoped)); }); return `${displayed.join(", ")}${arguments_.paths.length > displayed.length ? `, +${arguments_.paths.length - displayed.length} more` : ""}`; } @@ -5984,7 +5989,8 @@ function printScanSummary( errorOutput.write( `\n ${paint("REPORT", "1;36")} ${paint(errorMessage(result.reportPath), 4)}\n\n` + ` ${paint("FINDINGS", 1)} ${paint(`${findingCount}${findingSummary === "" ? "" : ` (${findingSummary})`}`, findingColor)}\n` + - ` ${paint("COVERAGE", 1)} ${result.coverage.completeness}\n` + + ` ${paint("SCOPE", 1)} ${formatCoverageScope(result.coverage)}\n` + + ` ${paint("COVERAGE", 1)} ${formatCoverageCompleteness(result.coverage.completeness)}\n` + (deepScanStop === undefined ? "" : ` ${paint("STOPPED", 1)} ${deepScanStop.reason}\n`) + diff --git a/sdk/typescript/src/coverage-presentation.ts b/sdk/typescript/src/coverage-presentation.ts new file mode 100644 index 000000000..3a578d526 --- /dev/null +++ b/sdk/typescript/src/coverage-presentation.ts @@ -0,0 +1,74 @@ +import type { CoverageCompleteness, CoverageDocument } from "./models.js"; + +export type CoverageSummary = Pick< + CoverageDocument, + "mode" | "completeness" | "includePaths" | "excludePaths" +> & + Partial>; + +export function formatScopePath(path: string): string { + const normalizationSensitive = path.normalize("NFC") !== path; + if ( + path.length > 0 && + !normalizationSensitive && + !/[\s,;'"\\\u0000-\u001f\u007f-\u009f\p{Cf}\p{Default_Ignorable_Code_Point}]/u.test( + path, + ) + ) { + return path; + } + return JSON.stringify(path).replace( + normalizationSensitive + ? /[^\u0020-\u007e]/gu + : /[\u007f-\u009f\u2028\u2029\p{Cf}\p{Default_Ignorable_Code_Point}]/gu, + (character) => + character + .split("") + .map((unit) => `\\u${unit.charCodeAt(0).toString(16).padStart(4, "0")}`) + .join(""), + ); +} + +function scopePathParts(paths: readonly string[], finalSuffix = ""): string[] { + return paths.map( + (path, index) => + `${formatScopePath(path)}${index < paths.length - 1 ? "," : finalSuffix}`, + ); +} + +export function formatCoverageScopeParts( + coverage: Omit, +): string[] { + const mode = + coverage.mode === "scoped_path" + ? "scoped paths" + : coverage.mode.replaceAll("_", " "); + const exclusions = [ + ...new Set([ + ...coverage.excludePaths, + ...(coverage.explicitExclusions ?? []).map(({ pattern }) => pattern), + ]), + ]; + const suffix = exclusions.length > 0 ? ";" : ""; + return [ + `${mode}:`, + ...(coverage.includePaths.length > 0 + ? scopePathParts(coverage.includePaths, suffix) + : [`(no included paths)${suffix}`]), + ...(exclusions.length > 0 + ? ["excluding", ...scopePathParts(exclusions)] + : []), + ]; +} + +export function formatCoverageScope( + coverage: Omit, +): string { + return formatCoverageScopeParts(coverage).join(" "); +} + +export function formatCoverageCompleteness( + completeness: CoverageCompleteness, +): string { + return `${completeness} for requested scope`; +} diff --git a/sdk/typescript/src/custom-validation-prompt.ts b/sdk/typescript/src/custom-validation-prompt.ts index ad8cc0128..8a175bc46 100644 --- a/sdk/typescript/src/custom-validation-prompt.ts +++ b/sdk/typescript/src/custom-validation-prompt.ts @@ -9,9 +9,9 @@ import { PLUGIN_NAME } from "./runtime.js"; // the ordinary validation sequence with a custom-validation request. const SOURCES = { "references/core-scan.md": - "806efc9432b3352e86c495466b00a700d77e338b6db297a67f3d78d1f7ec359e", + "c91f8d9b873cc63991d1e3c6da0e089b0c1267860d578126447c7491839e9f66", "skills/security-scan/SKILL.md": - "ef4d18fdf85667877f80f68be557214d5edca629c97075c85d7ccdefcc76fb8f", + "4aa63fa34d6a39097a4f4169cbf293d9e95c11202c587378c90749e97f8dbcbb", "skills/security-diff-scan/SKILL.md": "5c2bcdf8c862da4e7fb2c01b0fed8f0de61b65a479c6ae6897d7f12f11f9f87a", } as const; diff --git a/sdk/typescript/src/models.ts b/sdk/typescript/src/models.ts index b9d9fefd6..bd473b5f6 100644 --- a/sdk/typescript/src/models.ts +++ b/sdk/typescript/src/models.ts @@ -281,6 +281,9 @@ export interface CoverageDocument { | "branch_diff" | "working_tree" | "deep_repository"; + /** + * Completion of the requested review within the authorized scope, not whole-repository coverage or proof that no vulnerabilities exist. Use partial when requested work or an essential in-scope proof gap remains; use unknown when completion cannot be established. + */ completeness: "complete" | "partial" | "unknown"; inventoryStrategy: | "repository" @@ -293,6 +296,9 @@ export interface CoverageDocument { surfaces: { id: string; label: string; + /** + * The evidence-supported result for this in-scope surface. Use needs_follow_up only when an essential part of its requested review remains unresolved. + */ disposition: | "reported" | "no_issue_found" @@ -304,11 +310,17 @@ export interface CoverageDocument { notes?: string; [k: string]: unknown; }[]; + /** + * Source excluded by the requested scope or applicable authorization and policy. Do not invent exclusions to hide unfinished requested work. + */ explicitExclusions: { pattern: string; reason: string; [k: string]: unknown; }[]; + /** + * Unfinished requested work or an essential unresolved in-scope proof gap. Every item blocks complete coverage. Put optional additional assurance in openQuestions instead; never move a real blocker there to claim completion. + */ deferred: { id: string; reason: string; @@ -316,6 +328,9 @@ export interface CoverageDocument { surfaceIds?: string[]; [k: string]: unknown; }[]; + /** + * Optional additional assurance or later investigations that do not block completion of the requested review. Preserve essential unresolved in-scope proof gaps in deferred instead. + */ openQuestions?: { question: string; followUpPrompt?: string; diff --git a/sdk/typescript/src/scan-history-renderer.ts b/sdk/typescript/src/scan-history-renderer.ts index a95ebe9b8..4ff4c72e5 100644 --- a/sdk/typescript/src/scan-history-renderer.ts +++ b/sdk/typescript/src/scan-history-renderer.ts @@ -1,5 +1,10 @@ import { basename, relative } from "node:path"; import type { JsonObject } from "./config.js"; +import { + formatCoverageCompleteness, + formatCoverageScopeParts, + type CoverageSummary, +} from "./coverage-presentation.js"; export type HistoryCommand = | "list" @@ -80,11 +85,18 @@ export function renderScanHistory( ` ${accent("━".repeat(width - 4))}`, ]; - const wrap = (value: string, indent: number, prefix?: string): void => { + const wrap = ( + value: string | readonly string[], + indent: number, + prefix?: string, + ): void => { const available = width - indent - 2; let line = ""; let first = true; - for (const word of clean(value).split(/\s+/)) { + // Scope paths arrive as whole entries because whitespace can be part of a filename. + const words = + typeof value === "string" ? clean(value).split(/\s+/) : value.map(clean); + for (const word of words) { if (line.length > 0 && line.length + word.length + 1 > available) { lines.push(`${first && prefix ? prefix : " ".repeat(indent)}${line}`); first = false; @@ -184,7 +196,7 @@ export function renderScanHistory( ); if (wide) { lines.push( - ` ${strong("SCAN".padEnd(36))} ${strong("DATE".padEnd(10))} ${strong("MODE".padEnd(8))}${multipleRepositories ? ` ${strong("REPOSITORY".padEnd(18))}` : ""} ${strong("FINDINGS")} ${strong("STATUS")}`, + ` ${strong("SCAN".padEnd(36))} ${strong("DATE".padEnd(10))} ${strong("MODE".padEnd(8))}${multipleRepositories ? ` ${strong("REPOSITORY".padEnd(18))}` : ""} ${strong("FINDINGS")} ${strong("EXECUTION")}`, ); } for (const scan of scans) { @@ -192,7 +204,7 @@ export function renderScanHistory( const complete = status === "complete"; const statusColor = complete ? 32 : status === "running" ? 36 : 31; const statusLabel = paint( - `${complete ? "✓" : "●"} ${status.toUpperCase()}`, + complete ? "✓ FINISHED" : `● ${status.toUpperCase()}`, statusColor, ); const started = clean(scan["startedAt"]).slice(0, 10); @@ -227,8 +239,21 @@ export function renderScanHistory( status === "complete" ? 32 : status === "running" ? 36 : 31; lines.push( ` ${strong(clean(basename(result["targetPath"] as string)))} ${accent("·")} ${clean(result["scanId"])}`, - ` ${paint(`${status === "complete" ? "✓" : "●"} ${status.toUpperCase()}`, statusColor)} ${accent("·")} ${clean(result["mode"])}`, + ` ${paint(status === "complete" ? "✓ FINISHED" : `● ${status.toUpperCase()}`, statusColor)} ${accent("·")} ${clean(result["mode"])}`, ); + const canonicalCoverage = result["coverage"] as CoverageSummary | undefined; + if (canonicalCoverage) { + wrap( + formatCoverageScopeParts(canonicalCoverage), + 11, + ` ${strong("SCOPE")} `, + ); + lines.push( + ` ${strong("COVERAGE")} ${formatCoverageCompleteness(canonicalCoverage.completeness)}`, + ); + } else if (status === "complete") { + lines.push(` ${strong("COVERAGE")} not available`); + } if (result["failureMessage"]) { wrap(String(result["failureMessage"]), 11, ` ${paint("ERROR", 31)} `); } @@ -289,7 +314,7 @@ export function renderScanHistory( ]; if (parts.length > 0) { lines.push( - ` ${strong("COVERAGE")} ${parts.join(` ${accent("·")} `)}`, + ` ${strong("REVIEW PROGRESS")} ${parts.join(` ${accent("·")} `)}`, ); } } diff --git a/sdk/typescript/src/version.ts b/sdk/typescript/src/version.ts index 241666585..a6165b5ac 100644 --- a/sdk/typescript/src/version.ts +++ b/sdk/typescript/src/version.ts @@ -9,7 +9,7 @@ const PACKAGE_VERSIONS = packageVersions( export const VERSION = PACKAGE_VERSIONS.package; export const CODEX_SDK_VERSION = PACKAGE_VERSIONS.sdk; export const CODEX_EXECUTABLE_VERSION = PACKAGE_VERSIONS.executable; -export const BUNDLED_PLUGIN_VERSION = "0.1.22" as const; +export const BUNDLED_PLUGIN_VERSION = "0.1.23" as const; const PACKAGE_NAME = "@openai/codex-security"; diff --git a/sdk/typescript/tests-ts/cli-export.test.ts b/sdk/typescript/tests-ts/cli-export.test.ts index e8e58c8a4..ff8eecd70 100644 --- a/sdk/typescript/tests-ts/cli-export.test.ts +++ b/sdk/typescript/tests-ts/cli-export.test.ts @@ -415,7 +415,7 @@ describe("CLI", () => { completeness === "complete", ); expect(run.properties.codexSecurityCoverageCompleteness).toBe( - completeness === "complete" ? undefined : completeness, + completeness, ); if (completeness === "complete") { expect(invocation.toolExecutionNotifications).toBeUndefined(); diff --git a/sdk/typescript/tests-ts/cli.test.ts b/sdk/typescript/tests-ts/cli.test.ts index fbf7520e1..85647b568 100644 --- a/sdk/typescript/tests-ts/cli.test.ts +++ b/sdk/typescript/tests-ts/cli.test.ts @@ -1159,7 +1159,7 @@ describe("CLI", () => { expect(text).toContain("CODEX SECURITY"); expect(text).toContain("SCAN HISTORY"); expect(text).toContain("juice-shop"); - for (const heading of ["DATE", "STATUS", "FINDINGS", "MODE", "SCAN"]) { + for (const heading of ["DATE", "EXECUTION", "FINDINGS", "MODE", "SCAN"]) { expect(text).toContain(heading); } expect(text).toContain("failed-scan"); @@ -2906,7 +2906,7 @@ describe("CLI", () => { expect(JSON.parse(stdout.text())).toEqual(fakeResult().toJSON()); expect(stderr.text()).toContain("Preparing scan"); expect(stderr.text()).toContain("Running scan"); - expect(stderr.text()).toContain("Scan complete"); + expect(stderr.text()).toContain("Scan finished"); expect(captured.config).toEqual({ pluginPath: "plugin.zip", pythonPath: "/managed/python", @@ -4189,14 +4189,15 @@ describe("CLI", () => { ), ).toBe(0); expect(stdout.text()).toBe(""); - expect(stderr.text()).toContain("Scan complete · 12345678"); + expect(stderr.text()).toContain("Scan finished · 12345678"); expect(stderr.text()).not.toContain(result.manifest.scan.id); expect(stderr.text()).toContain( [ ` REPORT ${result.reportPath}`, "", " FINDINGS 1 (1 high)", - " COVERAGE complete", + " SCOPE repository: .", + " COVERAGE complete for requested scope", " ELAPSED 6m 37s", " TOKENS 1,250 input, 200 cached, 30 output", " COST $0.00625", @@ -4895,7 +4896,8 @@ describe("CLI", () => { "Worker delegation unavailable during file review; continuing without delegated workers.", ); expect(stdout.text()).toBe(""); - expect(stderr.text()).toContain("FINDINGS 0\n COVERAGE complete"); + expect(stderr.text()).toContain("FINDINGS 0"); + expect(stderr.text()).toContain("COVERAGE complete for requested scope"); }); test("validates a dry run without starting a scan", async () => { @@ -5116,6 +5118,144 @@ describe("CLI", () => { } }); + test("evaluates complete scoped scans against their requested scope", async () => { + for (const paths of [ + ["src/parser.ts"], + ["src/parser"], + ["src", "src/parser.ts", "tests"], + ]) { + for (const policy of [false, true]) { + const result = fakeResult(["high"]); + result.manifest.scan.scope.includePaths = paths; + result.manifest.scan.scope.validationMode = "static source review"; + result.manifest.scan.scope.limitations = [ + "Runtime testing was not requested.", + ]; + Object.assign(result.coverage, { + mode: "scoped_path", + inventoryStrategy: "scoped_path", + includePaths: paths, + openQuestions: [ + { question: "Consider a separate deployment review." }, + ], + }); + const stdout = capture(); + const stderr = capture(); + const exit = await main( + [ + "scan", + "--json", + ...paths.flatMap((path) => ["--path", path]), + ...(policy ? ["--fail-on-severity", "high"] : []), + ], + stdout.stream, + stderr.stream, + dependencies({ result }), + ); + expect(exit).toBe(policy ? 1 : 0); + expect(JSON.parse(stdout.text())).toEqual(result.toJSON()); + expect(stderr.text()).toContain(`scoped paths: ${paths.join(", ")}`); + expect(stderr.text()).toContain("complete for requested scope"); + expect(stderr.text()).not.toContain( + "Cannot evaluate the failure policy", + ); + } + } + }); + + test("includes canonical exclusions in a complete scan summary", async () => { + const result = fakeResult(); + Object.assign(result.manifest.scan.scope, { + includePaths: ["src"], + excludePaths: ["src/vendor"], + }); + Object.assign(result.coverage, { + mode: "scoped_path", + inventoryStrategy: "scoped_path", + includePaths: ["src"], + excludePaths: ["src/vendor"], + explicitExclusions: [ + { pattern: "src/vendor", reason: "Excluded dependency source." }, + { pattern: "src/generated/**", reason: "Excluded generated source." }, + ], + }); + const stdout = capture(); + const stderr = capture(); + expect( + await main( + ["scan", "--path", "src", "--json"], + stdout.stream, + stderr.stream, + dependencies({ result }), + ), + ).toBe(0); + expect(JSON.parse(stdout.text())).toEqual(result.toJSON()); + expect(stderr.text()).toContain( + "scoped paths: src; excluding src/vendor, src/generated/**", + ); + expect(stderr.text()).toContain("complete for requested scope"); + }); + + test("quotes and escapes untrusted paths in human scope summaries", async () => { + const path = "src; excluding tests\u2028COVERAGE forged"; + const result = fakeResult(); + result.manifest.scan.scope.includePaths = [path]; + Object.assign(result.coverage, { + mode: "scoped_path", + inventoryStrategy: "scoped_path", + includePaths: [path], + excludePaths: ["vendor\u0085\u2029forged"], + }); + const stdout = capture(); + const stderr = capture(); + expect( + await main( + ["scan", "--path", path, "--json"], + stdout.stream, + stderr.stream, + dependencies({ result }), + ), + ).toBe(0); + expect(JSON.parse(stdout.text())).toEqual(result.toJSON()); + expect(stderr.text()).not.toMatch(/[\u0085\u2028\u2029]/u); + expect(stderr.text()).toContain( + '"src; excluding tests\\u2028COVERAGE forged"', + ); + expect(stderr.text()).toContain('"vendor\\u0085\\u2029forged"'); + }); + + test("does not promote incomplete coverage merely because paths were selected", async () => { + for (const completeness of ["partial", "unknown"] as const) { + const result = fakeResult(["high"], completeness); + result.manifest.scan.scope.includePaths = ["src/parser"]; + Object.assign(result.coverage, { + mode: "scoped_path", + inventoryStrategy: "scoped_path", + includePaths: ["src/parser"], + deferred: [ + { + id: "source-review", + reason: "Requested source remains unreviewed.", + }, + ], + }); + const stdout = capture(); + const stderr = capture(); + expect( + await main( + ["scan", "--path", "src/parser", "--json"], + stdout.stream, + stderr.stream, + dependencies({ result }), + ), + ).toBe(2); + expect(JSON.parse(stdout.text())).toEqual(result.toJSON()); + expect(stderr.text()).toContain( + `${completeness} for the requested scope`, + ); + } + }); + test("does not report an incomplete scan as successful without a policy", async () => { for (const completeness of ["partial", "unknown"] as const) { const result = fakeResult(["high"], completeness); @@ -5131,7 +5271,7 @@ describe("CLI", () => { ).toBe(2); expect(JSON.parse(stdout.text())).toEqual(result.toJSON()); expect(stderr.text()).toContain( - `Scan coverage is ${completeness}; results may be incomplete.`, + `Scan coverage is ${completeness} for the requested scope; see the report for unfinished work.`, ); } }); diff --git a/sdk/typescript/tests-ts/contract.test.ts b/sdk/typescript/tests-ts/contract.test.ts index 7ce58cdde..582edfc87 100644 --- a/sdk/typescript/tests-ts/contract.test.ts +++ b/sdk/typescript/tests-ts/contract.test.ts @@ -1391,6 +1391,48 @@ describe("canonical scan contract", () => { ).rejects.toThrow("Repository scan manifest target must not be git_diff."); }); + test("accepts complete scoped coverage with optional follow-up", async () => { + for (const paths of [ + ["src/extract.py"], + ["src"], + ["src", "src/extract.py"], + ]) { + const scanDir = await copyExample(); + const manifestPath = join(scanDir, "scan-manifest.json"); + const manifest = await readJson(manifestPath); + Object.assign(manifest["scan"]["scope"], { + includePaths: paths, + validationMode: "static source review", + limitations: ["Runtime testing was not requested."], + }); + await writeJson(manifestPath, manifest); + const coveragePath = join(scanDir, "coverage.json"); + const coverage = await readJson(coveragePath); + Object.assign(coverage, { + mode: "scoped_path", + inventoryStrategy: "scoped_path", + includePaths: paths, + openQuestions: [ + { question: "Consider separate deployment confirmation." }, + ], + }); + await writeJson(coveragePath, coverage); + await reseal(scanDir); + const sealed = await readFile(coveragePath, "utf8"); + const result = await loadContract(scanDir, { + pluginRoot: PLUGIN_ROOT, + expectation: expectation({ kind: "paths", paths }), + }); + expect(result.coverage).toMatchObject({ + completeness: "complete", + mode: "scoped_path", + includePaths: paths, + openQuestions: coverage["openQuestions"], + }); + expect(await readFile(coveragePath, "utf8")).toBe(sealed); + } + }); + test("binds requested path scope, mode, and plugin version", async () => { const scanDir = await copyExample(); const manifestPath = join(scanDir, "scan-manifest.json"); diff --git a/sdk/typescript/tests-ts/coverage-presentation.test.ts b/sdk/typescript/tests-ts/coverage-presentation.test.ts new file mode 100644 index 000000000..b84f42650 --- /dev/null +++ b/sdk/typescript/tests-ts/coverage-presentation.test.ts @@ -0,0 +1,203 @@ +import { describe, expect, test } from "bun:test"; +import { + formatCoverageScope, + formatScopePath, +} from "../src/coverage-presentation.js"; +import { PLUGIN_ROOT } from "./plugin-root.js"; + +function projectScope(scope: { + includePaths: string[]; + excludePaths: string[]; + explicitExclusions: Array<{ pattern: string; reason: string }>; +}): string { + const python = Bun.which("python3") ?? Bun.which("python"); + expect(python).not.toBeNull(); + const script = [ + "import json, pathlib, runpy, sys, unicodedata", + "plugin = pathlib.Path(sys.argv[1])", + "examples = plugin / 'examples' / 'completed-scan'", + "manifest, findings, coverage = [json.loads((examples / name).read_text()) for name in ('scan-manifest.json', 'findings.json', 'coverage.json')]", + "scope = json.loads(sys.argv[2])", + "manifest['scan']['scope'].update({key: scope[key] for key in ('includePaths', 'excludePaths')})", + "manifest['scan']['scope']['artifactsReviewed'] = scope['includePaths']", + "coverage.update(scope)", + "coverage.update({'mode': 'scoped_path', 'completeness': 'partial', 'surfaces': [], 'deferred': [{'id': 'source-review', 'reason': 'Source review remains unfinished.', 'paths': scope['includePaths']}]})", + "findings['findings'] = []", + "unicodedata.category = lambda _character: 'Cn'", + "projection = runpy.run_path(str(plugin / 'scripts' / 'report_projection.py'))", + "sys.stdout.buffer.write(projection['generate_report_markdown'](manifest, findings, coverage))", + ].join("\n"); + const result = Bun.spawnSync( + [python!, "-I", "-B", "-c", script, PLUGIN_ROOT, JSON.stringify(scope)], + { stdout: "pipe", stderr: "pipe" }, + ); + expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0); + return new TextDecoder().decode(result.stdout); +} + +describe("coverage scope presentation", () => { + test("distinguishes path text from include and exclusion delimiters", () => { + const included = formatCoverageScope({ + mode: "scoped_path", + includePaths: ["src; excluding tests"], + excludePaths: [], + }); + const excluded = formatCoverageScope({ + mode: "scoped_path", + includePaths: ["src"], + excludePaths: ["tests"], + }); + expect(included).toBe('scoped paths: "src; excluding tests"'); + expect(excluded).toBe("scoped paths: src; excluding tests"); + expect(included).not.toBe(excluded); + expect( + formatCoverageScope({ + mode: "scoped_path", + includePaths: ["src, tests", "src/parser.ts"], + excludePaths: [], + }), + ).toBe('scoped paths: "src, tests", src/parser.ts'); + }); + + test("round-trips ambiguous paths without emitting terminal controls", () => { + const paths = [ + "src/a b.ts", + "src/trailing ", + 'src/"quoted"', + "src/back\\slash", + "src/line\nfeed", + "src/\u001b[31m", + "src/\u009b31m", + "src/\u0085\u2028\u2029COVERAGE forged", + "src/\ufeffname", + ]; + for (const path of paths) { + const encoded = formatScopePath(path); + expect(encoded).not.toMatch( + /[\u0000-\u001f\u007f-\u009f\u2028\u2029\ufeff]/u, + ); + expect(JSON.parse(encoded)).toBe(path); + } + expect(formatScopePath("src/parser.ts")).toBe("src/parser.ts"); + expect(formatScopePath("src/generated/**")).toBe("src/generated/**"); + }); + + test("distinguishes canonically equivalent path spellings", () => { + for (const [composed, decomposed, encoded] of [ + ["src/café.ts", "src/cafe\u0301.ts", '"src/cafe\\u0301.ts"'], + ["src/가.ts", "src/\u1100\u1161.ts", '"src/\\u1100\\u1161.ts"'], + ] as const) { + expect(composed.normalize("NFC")).toBe(decomposed.normalize("NFC")); + expect(formatScopePath(composed)).toBe(composed); + expect(formatScopePath(decomposed)).toBe(encoded); + expect(JSON.parse(encoded)).toBe(decomposed); + expect(formatScopePath(composed).normalize("NFC")).not.toBe( + formatScopePath(decomposed).normalize("NFC"), + ); + } + }); + + test("escapes invisible Unicode controls independently of Python's Unicode database", () => { + // Unicode 17 DerivedGeneralCategory.txt, General_Category=Format. + const formatControls = ( + [ + [0x00ad, 0x00ad], + [0x0600, 0x0605], + [0x061c, 0x061c], + [0x06dd, 0x06dd], + [0x070f, 0x070f], + [0x0890, 0x0891], + [0x08e2, 0x08e2], + [0x180e, 0x180e], + [0x200b, 0x200f], + [0x202a, 0x202e], + [0x2060, 0x2064], + [0x2066, 0x206f], + [0xfeff, 0xfeff], + [0xfff9, 0xfffb], + [0x110bd, 0x110bd], + [0x110cd, 0x110cd], + [0x13430, 0x1343f], + [0x1bca0, 0x1bca3], + [0x1d173, 0x1d17a], + [0xe0001, 0xe0001], + [0xe0020, 0xe007f], + ] as const + ).flatMap(([start, end]) => + Array.from({ length: end - start + 1 }, (_, offset) => + String.fromCodePoint(start + offset), + ), + ); + expect(formatControls).toHaveLength(170); + const representativeNonFormatIgnorables = + "\u034f\u115f\u1160\u17b4\u17b5\u180b\u180f\u3164\ufe00\ufe0f" + + "\uffa0\ufff0\ufff8\u{e0000}\u{e0100}\u{e0fff}"; + const controls = [...formatControls, ...representativeNonFormatIgnorables]; + const paths = controls.flatMap((control) => [ + `src/${control}name`, + `src/a ${control}name`, + ]); + for (const path of paths) { + const encoded = formatScopePath(path); + expect(encoded).not.toMatch(/[\p{Cf}\p{Default_Ignorable_Code_Point}]/u); + expect(JSON.parse(encoded)).toBe(path); + } + const report = projectScope({ + includePaths: paths, + excludePaths: [], + explicitExclusions: [], + }); + expect(report).not.toMatch(/[\p{Cf}\p{Default_Ignorable_Code_Point}]/u); + const includedLine = + report.split("\n").find((line) => line.startsWith("- Included paths:")) ?? + ""; + expect(includedLine).toEndWith( + paths.map((path) => `\`${formatScopePath(path)}\``).join(", "), + ); + }); + + test("preserves exact paths in Markdown scope and deferred work", () => { + const included = [ + ["src/a b.ts", '`"src/a b.ts"`'], + ["src/trailing ", '`"src/trailing "`'], + ["src,tests", '`"src,tests"`'], + ["src/line\nfeed", '`"src/line\\nfeed"`'], + ["src/\u202eforged", '`"src/\\u202eforged"`'], + ["src/\ufeffname", '`"src/\\ufeffname"`'], + ["src/\u2028\u2029next", '`"src/\\u2028\\u2029next"`'], + ["src/[name]*_.ts", "`src/[name]*_.ts`"], + ["src/a`b.ts", "``src/a`b.ts``"], + ["`edge`", "`` `edge` ``"], + ["src/naïve.ts", "`src/naïve.ts`"], + ["src/café.ts", "`src/café.ts`"], + ["src/cafe\u0301.ts", '`"src/cafe\\u0301.ts"`'], + ["src/가.ts", "`src/가.ts`"], + ["src/\u1100\u1161.ts", '`"src/\\u1100\\u1161.ts"`'], + ] as const; + const excluded = [ + ["vendor/a b", '`"vendor/a b"`'], + ["vendor,tests", '`"vendor,tests"`'], + ] as const; + const pattern = "generated/\u2066[omitted]*`"; + const report = projectScope({ + includePaths: included.map(([path]) => path), + excludePaths: excluded.map(([path]) => path), + explicitExclusions: [{ pattern, reason: "Synthetic exclusion." }], + }); + const row = (label: string) => + report.split("\n").find((line) => line.startsWith(label)) ?? ""; + const expectedIncludes = included.map(([, encoded]) => encoded).join(", "); + expect(row("- Included paths:")).toEndWith(expectedIncludes); + expect(row("- Excluded paths:")).toEndWith( + excluded.map(([, encoded]) => encoded).join(", "), + ); + expect(row("- Artifacts reviewed:")).toEndWith(expectedIncludes); + expect(row(" - Paths:")).toEndWith(expectedIncludes); + expect(report).toContain( + 'Excluded ``"generated/\\u2066[omitted]*`"``: Synthetic exclusion.', + ); + expect(report).not.toMatch( + /[\u007f-\u009f\u2028\u2029\p{Cf}\p{Default_Ignorable_Code_Point}]/u, + ); + }); +}); diff --git a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts index 02c66b8d6..3cfb1edd5 100644 --- a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts +++ b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts @@ -901,6 +901,10 @@ describe("deep scan workbench ownership", () => { expect(report).toContain( "No findings were validated before the scan reached its cost limit.", ); + expect(report).toContain( + "Review the deferred candidates in Incomplete Requested Work.", + ); + expect(report).toContain("## Incomplete Requested Work"); if (existingDeferred) { expect(report).toContain( "Existing candidate validation dependency was unavailable.", diff --git a/sdk/typescript/tests-ts/plugin-finding-detail-contract.test.ts b/sdk/typescript/tests-ts/plugin-finding-detail-contract.test.ts index 07e23f24b..e74192edd 100644 --- a/sdk/typescript/tests-ts/plugin-finding-detail-contract.test.ts +++ b/sdk/typescript/tests-ts/plugin-finding-detail-contract.test.ts @@ -103,7 +103,7 @@ function projectFindingDetails(details: JsonObject): string { "manifest, findings, coverage = [json.loads((examples / name).read_text()) for name in ('scan-manifest.json', 'findings.json', 'coverage.json')]", "findings['findings'][0].update(json.loads(sys.argv[2]))", "projection = runpy.run_path(str(plugin / 'scripts' / 'report_projection.py'))", - "print(projection['build_report_markdown'](manifest, findings, coverage))", + "sys.stdout.buffer.write(projection['generate_report_markdown'](manifest, findings, coverage))", ].join("\n"); const result = Bun.spawnSync( [python!, "-I", "-B", "-c", script, PLUGIN_ROOT, JSON.stringify(details)], @@ -117,6 +117,10 @@ async function readJson(path: string): Promise { return JSON.parse(await readFile(path, "utf8")) as JsonObject; } +function schemaProperties(schema: JsonObject): Record { + return schema["properties"] as Record; +} + async function startMcp() { const child = spawn( process.execPath, @@ -180,6 +184,15 @@ async function startMcp() { } describe("bundled plugin finding detail contracts", () => { + test("keeps the shipped example report equal to its canonical projection", async () => { + expect(projectFindingDetails({})).toBe( + await readFile( + join(PLUGIN_ROOT, "examples", "completed-scan", "report.md"), + "utf8", + ), + ); + }); + test("rejects malformed known fields in scan drafts", async () => { const schemaRoot = join(PLUGIN_ROOT, "schemas"); const [commonSchema, scanDraftSchema] = await Promise.all([ @@ -226,6 +239,34 @@ describe("bundled plugin finding detail contracts", () => { ); expect(tool).toBeDefined(); + const sourceSchema = await readJson( + join(PLUGIN_ROOT, "schemas", "tools", "scan-draft.schema.json"), + ); + const definitions = sourceSchema["$defs"] as Record; + const declaredCoverage = schemaProperties(definitions["coverage"]!); + const advertisedCoverage = schemaProperties( + schemaProperties(tool!["inputSchema"] as JsonObject)["coverage"]!, + ); + for (const name of [ + "completeness", + "explicitExclusions", + "deferred", + "openQuestions", + ]) { + const description = declaredCoverage[name]!["description"]; + expect(typeof description).toBe("string"); + expect(advertisedCoverage[name]!["description"]).toBe(description); + } + expect( + schemaProperties( + advertisedCoverage["surfaces"]!["items"] as JsonObject, + )["disposition"]!["description"], + ).toBe( + schemaProperties(definitions["surface"]!)["disposition"]![ + "description" + ], + ); + const validator = new Ajv({ strict: false }); validator.addFormat("uuid", /^[0-9a-f-]{36}$/iu); const validate = validator.compile(tool!["inputSchema"] as JsonObject); diff --git a/sdk/typescript/tests-ts/prompt-only-start-timeout.test.ts b/sdk/typescript/tests-ts/prompt-only-start-timeout.test.ts index 324fd9635..fdaa34af9 100644 --- a/sdk/typescript/tests-ts/prompt-only-start-timeout.test.ts +++ b/sdk/typescript/tests-ts/prompt-only-start-timeout.test.ts @@ -1,7 +1,21 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; import { expect, test } from "bun:test"; import { loadBundledRuntime, PLUGIN_ROOT } from "./plugin-root.js"; -test("gives prompt-only scan startup the five-minute scan timeout", async () => { +type WorkbenchOptions = { timeout: number; maxBuffer: number }; +type WorkbenchExecutor = ( + command: string, + args: string[], + options: WorkbenchOptions, +) => Promise<{ stdout: string }>; + +async function loadExecuteWorkbench( + executor: WorkbenchExecutor, + scriptPath = "workbench.py", +): Promise< + (command: string, args: string[]) => Promise> +> { const runtime = await loadBundledRuntime(); const source = /async function executeWorkbench\([^\n]*\) \{[\s\S]*?\n\}/u.exec( @@ -11,22 +25,27 @@ test("gives prompt-only scan startup the five-minute scan timeout", async () => const execFileHelper = /\b(execFileAsync\d*)\(/u.exec(source ?? "")?.[1]; expect(execFileHelper).toBeDefined(); - const executeWorkbench = new Function( + return new Function( execFileHelper!, "workbenchScriptPath", "PLUGIN_ROOT", "isJsonObject2", `${source}\nreturn executeWorkbench;`, )( - async ( - _command: string, - _args: string[], - options: { timeout: number }, - ) => ({ stdout: JSON.stringify({ timeout: options.timeout }) }), - () => "workbench.py", + executor, + () => scriptPath, PLUGIN_ROOT, - () => true, - ) as (command: string, args: string[]) => Promise<{ timeout: number }>; + (value: unknown) => + value !== null && typeof value === "object" && !Array.isArray(value), + ); +} + +test("gives prompt-only scan startup the five-minute scan timeout", async () => { + const executeWorkbench = await loadExecuteWorkbench( + async (_command, _args, options) => ({ + stdout: JSON.stringify({ timeout: options.timeout }), + }), + ); expect(await executeWorkbench("python", ["start-prompt-only-scan"])).toEqual({ timeout: 300_000, @@ -38,3 +57,16 @@ test("gives prompt-only scan startup the five-minute scan timeout", async () => timeout: 30_000, }); }); + +test("reads large native workbench responses", async () => { + const run = promisify(execFile); + const executeWorkbench = await loadExecuteWorkbench( + (command, args, options) => + run(command, args, { ...options, encoding: "utf8" }), + "-e", + ); + const result = await executeWorkbench(process.execPath, [ + "process.stdout.write(JSON.stringify({ value: 'x'.repeat(5 * 1024 * 1024) }))", + ]); + expect((result["value"] as string).length).toBe(5 * 1024 * 1024); +}); diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index 179db5b5c..482d0549f 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -1753,9 +1753,9 @@ describe("plugin runtime preparation", () => { } }); - test("refreshes cached plugins before forwarding delegated scan attribution", async () => { + test("refreshes cached bundled plugins before a scan", async () => { const root = await temporaryDirectory(); - const previous = await plugin(join(root, "previous"), "0.1.19"); + const previous = await plugin(join(root, "previous"), "0.1.22"); await writeFile( join(previous, ".mcp.json"), JSON.stringify({ @@ -1798,7 +1798,7 @@ describe("plugin runtime preparation", () => { }; expect((await bootstrapPlugin(home, previous, options)).version).toBe( - "0.1.19", + "0.1.22", ); const upgraded = await bootstrapPlugin(home, PLUGIN_ROOT, options); const configuration = JSON.parse( @@ -1808,8 +1808,22 @@ describe("plugin runtime preparation", () => { ), ) as { mcpServers: Record }; + expect( + await readFile( + join( + marketplace, + "plugins", + "codex-security", + "references", + "core-scan.md", + ), + "utf8", + ), + ).toBe( + await readFile(join(PLUGIN_ROOT, "references", "core-scan.md"), "utf8"), + ); expect(upgraded.version).toBe(BUNDLED_PLUGIN_VERSION); - expect(upgraded.version).not.toBe("0.1.19"); + expect(upgraded.version).not.toBe("0.1.22"); expect(configuration.mcpServers["codex-security"]?.env_vars).toContain( "CODEX_SECURITY_SURFACE", ); diff --git a/sdk/typescript/tests-ts/scan-history-renderer.test.ts b/sdk/typescript/tests-ts/scan-history-renderer.test.ts index 169550c43..9534d23fd 100644 --- a/sdk/typescript/tests-ts/scan-history-renderer.test.ts +++ b/sdk/typescript/tests-ts/scan-history-renderer.test.ts @@ -3,6 +3,107 @@ import { describe, expect, test } from "bun:test"; import { renderScanHistory } from "../src/scan-history-renderer.js"; describe("scan history renderer", () => { + const finishedScan = { + targetPath: "/demo/repository", + scanId: "scan-1", + mode: "standard", + progress: { status: "complete" }, + findings: [], + findingCount: 0, + startedAt: "2026-08-01T00:00:00Z", + }; + + test("separates finished execution from requested-scope coverage", () => { + for (const completeness of ["complete", "partial", "unknown"] as const) { + const text = renderScanHistory( + { + ...finishedScan, + coverage: { + mode: "scoped_path", + completeness, + includePaths: ["src/parser"], + excludePaths: [], + }, + }, + "show", + { color: false }, + ); + expect(text).toContain("FINISHED"); + expect(text).toContain("scoped paths: src/parser"); + expect(text).toContain(`${completeness} for requested scope`); + } + + for (const mode of ["standard", "diff"]) { + const legacy = renderScanHistory( + { ...finishedScan, mode, scope: "." }, + "show", + { color: false }, + ); + expect(legacy).toContain("FINISHED"); + expect(legacy).toContain("COVERAGE not available"); + expect(legacy).not.toContain("SCOPE"); + expect(legacy).not.toContain("complete for requested scope"); + } + + const list = renderScanHistory({ scans: [finishedScan] }, "list", { + color: false, + }); + expect(list).toContain("EXECUTION"); + expect(list).toContain("FINISHED"); + expect(list).not.toContain("SCOPE"); + }); + + test("wraps exact canonical paths and both exclusion sources", () => { + const paths = ["src/a b.ts", "src/trailing ", "src/\ufeffname"]; + const text = renderScanHistory( + { + ...finishedScan, + coverage: { + mode: "scoped_path", + completeness: "complete", + includePaths: paths, + excludePaths: ["vendor/a b"], + explicitExclusions: [ + { pattern: "vendor/a b", reason: "Dependency source." }, + { pattern: "generated/**", reason: "Generated source." }, + ], + }, + }, + "show", + { color: false, columns: 48 }, + ); + for (const path of paths.slice(0, 2)) expect(text).toContain(path); + expect(text).toContain('"src/\\ufeffname"'); + expect(text).not.toContain("\ufeff"); + expect(text).toContain("generated/**"); + expect(text.split("vendor/a b")).toHaveLength(2); + expect(text).toContain("complete for requested scope"); + const lines = text.split("\n"); + const start = lines.findIndex((line) => line.includes("SCOPE")); + const end = lines.findIndex((line) => line.includes("COVERAGE")); + expect(end - start).toBeGreaterThan(1); + expect(lines.slice(start, end).every((line) => line.length <= 48)).toBe( + true, + ); + }); + + test("uses canonical diff scope when it is available", () => { + const text = renderScanHistory( + { + ...finishedScan, + mode: "diff", + coverage: { + mode: "branch_diff", + completeness: "complete", + includePaths: ["src/parser.ts"], + excludePaths: [], + }, + }, + "show", + { color: false }, + ); + expect(text).toContain("branch diff: src/parser.ts"); + }); test("separates current repository findings from earlier observations", () => { const text = renderScanHistory( { @@ -284,7 +385,7 @@ describe("scan history renderer", () => { ), ); - expect(output).toContain("COMPLETE"); + expect(output).toContain("FINISHED"); expect(output).toContain("WARNING"); expect(output).toContain( "Repository HEAD changed while the scan was running", diff --git a/sdk/typescript/tests-ts/scan-recovery.test.ts b/sdk/typescript/tests-ts/scan-recovery.test.ts index e2c184ae6..5fdfaba02 100644 --- a/sdk/typescript/tests-ts/scan-recovery.test.ts +++ b/sdk/typescript/tests-ts/scan-recovery.test.ts @@ -63,11 +63,18 @@ type ScanSummary = { type SarifDocument = { runs: Array<{ - properties: { codexSecurityCoverageCompleteness?: string }; + automationDetails: { id: string }; + properties: { + codexSecurityCoverageCompleteness?: string; + codexSecurityCoverageMode?: string; + codexSecurityIncludePaths?: string[]; + codexSecurityExcludePaths?: string[]; + codexSecurityExplicitExclusions?: unknown[]; + }; results: Array<{ properties: { severity: string } }>; invocations?: Array<{ executionSuccessful: boolean; - toolExecutionNotifications: Array<{ + toolExecutionNotifications?: Array<{ level: string; message: { text: string }; }>; @@ -123,7 +130,13 @@ async function workbench( async function startDraftScan( repositoryKind: "directory" | "clean" | "dirty" | "nested" = "directory", - recipeFromStdin = false, + { + paths, + recipeFromStdin = false, + }: { + paths?: readonly string[]; + recipeFromStdin?: boolean; + } = {}, ): Promise { const root = await realpath( await mkdtemp(join(tmpdir(), "codex-security-scan-recovery-")), @@ -188,7 +201,7 @@ async function startDraftScan( config: {}, mode: "standard", repository: target, - target: { kind: "repository", paths: [] }, + target: { kind: paths ? "paths" : "repository", paths: paths ?? [] }, }); const registration = await workbench( fixture, @@ -250,8 +263,220 @@ async function completeScan(fixture: ScanFixture): Promise { } describe("malformed scan artifact recovery", () => { + test("seals complete scoped reviews with optional follow-up and exports their scope", async () => { + const fixture = await startDraftScan("directory", { + paths: ["src", "src/extract.py"], + }); + const manifestPath = join(fixture.scanDir, "scan-manifest.json"); + const manifest = await readJson<{ + scan: { scope: Record }; + }>(manifestPath); + Object.assign(manifest.scan.scope, { + validationMode: "static source review", + runtimeStatus: "not requested", + limitations: ["Live testing was outside the requested review."], + }); + await writeJson(manifestPath, manifest); + const coveragePath = join(fixture.scanDir, "coverage.json"); + const coverage = await readJson(coveragePath); + coverage["openQuestions"] = [ + { question: "Consider a separate deployment review." }, + ]; + coverage.explicitExclusions = [ + { + pattern: "src/generated/**", + reason: "Generated source was excluded by policy.", + }, + ]; + await writeJson(coveragePath, coverage); + + const completed = await completeScan(fixture); + expect(completed.progress.status).toBe("complete"); + expect(await readJson(coveragePath)).toMatchObject({ + completeness: "complete", + mode: "scoped_path", + includePaths: ["src", "src/extract.py"], + openQuestions: coverage["openQuestions"], + }); + const report = await readFile(join(fixture.scanDir, "report.md"), "utf8"); + expect(report).toContain("Consider a separate deployment review."); + const sarif = await readJson( + join(fixture.scanDir, "exports", "results.sarif"), + ); + expect(sarif.runs[0]).toMatchObject({ + automationDetails: { id: fixture.scanId }, + properties: { + codexSecurityCoverageMode: "scoped_path", + codexSecurityCoverageCompleteness: "complete", + codexSecurityIncludePaths: ["src", "src/extract.py"], + codexSecurityExcludePaths: [], + codexSecurityExplicitExclusions: coverage.explicitExclusions, + }, + }); + expect(sarif.runs[0]?.invocations).toEqual([{ executionSuccessful: true }]); + + const history = await workbench(fixture, [ + "get-scan", + "--scan-id", + fixture.scanId, + ]); + expect(history["scan"]).toMatchObject({ + coverage: { + mode: "scoped_path", + completeness: "complete", + includePaths: ["src", "src/extract.py"], + excludePaths: [], + explicitExclusions: coverage.explicitExclusions, + }, + }); + expect(history["workspace"]).not.toHaveProperty("results.coverage"); + for (const artifactPath of [coveragePath, manifestPath]) { + const sealed = await readFile(artifactPath, "utf8"); + for (const modified of [`${sealed}\n`, "{}\n"]) { + await writeFile(artifactPath, modified); + try { + const unavailable = await workbench(fixture, [ + "get-scan", + "--scan-id", + fixture.scanId, + ]); + expect( + (unavailable["scan"] as Record)["coverage"], + ).toBeUndefined(); + expect(await readFile(artifactPath, "utf8")).toBe(modified); + } finally { + await writeFile(artifactPath, sealed); + } + } + } + }); + + test("reads sealed history coverage without loading unrelated findings", async () => { + const fixture = await startDraftScan("directory", { paths: ["src"] }); + await completeScan(fixture); + await writeFile(join(fixture.scanDir, "findings.json"), "not available\n"); + const history = await workbench(fixture, [ + "get-scan", + "--scan-id", + fixture.scanId, + ]); + expect(history["scan"]).toMatchObject({ + coverage: { + mode: "scoped_path", + completeness: "complete", + includePaths: ["src"], + excludePaths: [], + }, + }); + }); + + test("keeps saved history when scan directory resolution finds a symlink loop", () => { + const python = Bun.which("python3") ?? Bun.which("python"); + expect(python).not.toBeNull(); + const script = [ + "import argparse, json, pathlib, runpy, sys", + "from unittest import mock", + "namespace = runpy.run_path(str(pathlib.Path(sys.argv[1]) / 'scripts' / 'workbench_db.py'))", + "get_scan = namespace['get_scan']", + "args = argparse.Namespace(scan_id='scan-1', occurrence_id=None)", + "context = {'scan': {'progress': {'status': 'complete'}, 'findingCount': 2}}", + "scan = {'seal_manifest_digest': 'sealed', 'scan_dir': 'scan'}", + "resolve = mock.Mock(side_effect=RuntimeError('Symlink loop'))", + "with mock.patch.dict(get_scan.__globals__, {'scan_context': lambda *_: context, 'require_scan': lambda *_: scan, 'require_canonical_scan_directory': resolve}):", + " result = get_scan(None, args)", + "resolve.assert_called_once_with(pathlib.Path('scan'))", + "print(json.dumps(result))", + ].join("\n"); + const result = spawnSync(python!, ["-I", "-B", "-c", script, PLUGIN_ROOT], { + encoding: "utf8", + }); + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + scan: { progress: { status: "complete" }, findingCount: 2 }, + }); + }); + + test("does not attach completed coverage to an earlier running status", () => { + const python = Bun.which("python3") ?? Bun.which("python"); + expect(python).not.toBeNull(); + const script = [ + "import argparse, json, pathlib, runpy, sys", + "from unittest import mock", + "namespace = runpy.run_path(str(pathlib.Path(sys.argv[1]) / 'scripts' / 'workbench_db.py'))", + "get_scan = namespace['get_scan']", + "args = argparse.Namespace(scan_id='scan-1', occurrence_id=None)", + "coverage = {'completeness': 'complete'}", + "def read_status(status):", + " context = {'scan': {'progress': {'status': status}}}", + " with mock.patch.dict(get_scan.__globals__, {'scan_context': lambda *_: context, 'require_scan': lambda *_: {'status': 'complete'}, 'coverage_summary_for_history': lambda _: coverage}):", + " return get_scan(None, args)", + "print(json.dumps([read_status('running'), read_status('complete')]))", + ].join("\n"); + const result = spawnSync(python!, ["-I", "-B", "-c", script, PLUGIN_ROOT], { + encoding: "utf8", + }); + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual([ + { scan: { progress: { status: "running" } } }, + { + scan: { + progress: { status: "complete" }, + coverage: { completeness: "complete" }, + }, + }, + ]); + }); + + test.each(["deferred", "surface"] as const)( + "keeps an in-scope %s blocker separate from optional follow-up", + async (kind) => { + const fixture = await startDraftScan("directory", { paths: ["src"] }); + const coveragePath = join(fixture.scanDir, "coverage.json"); + const coverage = await readJson(coveragePath); + const blocker = + "An essential in-scope source question remains unresolved."; + const optionalQuestion = "Consider a separate deployment review."; + coverage["openQuestions"] = [{ question: optionalQuestion }]; + if (kind === "deferred") { + coverage.deferred = [ + { + id: "source-review", + reason: blocker, + paths: ["src/extract.py"], + }, + ]; + } else { + Object.assign((coverage.surfaces as CoverageSurface[])[0]!, { + disposition: "needs_follow_up", + notes: blocker, + }); + } + await writeJson(coveragePath, coverage); + await completeScan(fixture); + + expect( + (await readJson(coveragePath)).completeness, + ).toBe("partial"); + const report = await readFile(join(fixture.scanDir, "report.md"), "utf8"); + const blockerPosition = report.indexOf(blocker); + const optionalPosition = report.indexOf(optionalQuestion); + expect(blockerPosition).toBeGreaterThan(0); + expect(optionalPosition).toBeGreaterThan(blockerPosition); + const history = await workbench(fixture, [ + "get-scan", + "--scan-id", + fixture.scanId, + ]); + expect(history["scan"]).toMatchObject({ + coverage: { completeness: "partial" }, + }); + }, + ); + test("registers a Unicode scan recipe delivered through stdin", async () => { - const fixture = await startDraftScan("directory", true); + const fixture = await startDraftScan("directory", { + recipeFromStdin: true, + }); expect(fixture.registration).toMatchObject({ scanDir: fixture.scanDir, targetRevision: "unversioned", @@ -1053,7 +1278,6 @@ describe("malformed scan artifact recovery", () => { { id: "discarded-finding-1", reason: completed.warnings[0] }, ]); const report = await readFile(join(fixture.scanDir, "report.md"), "utf8"); - expect(report).toContain("| Coverage | partial |"); expect(report).toContain("Skipped malformed finding 1"); const sarif = await readJson( join(fixture.scanDir, "exports", "results.sarif"), diff --git a/sdk/typescript/tests-ts/workbench-scan-history.test.ts b/sdk/typescript/tests-ts/workbench-scan-history.test.ts index fb2ca8a74..d83605c44 100644 --- a/sdk/typescript/tests-ts/workbench-scan-history.test.ts +++ b/sdk/typescript/tests-ts/workbench-scan-history.test.ts @@ -64,6 +64,59 @@ test("loads each scan's matching findings once across historical batches", async }); }); +test("resolves missing findings only inside a complete later scope", async () => { + const python = await resolvePluginPython(); + const probe = [ + "import argparse, json, sqlite3, sys", + "sys.path.insert(0, sys.argv[1])", + "import workbench_scan_history as history", + "db = sqlite3.connect(':memory:')", + "db.row_factory = sqlite3.Row", + "db.executescript('''", + "CREATE TABLE scan_comparisons (before_scan_id TEXT, after_scan_id TEXT, result_json TEXT);", + "CREATE TABLE finding_occurrences (id TEXT, finding_id TEXT, scan_id TEXT, severity TEXT, title TEXT);", + "CREATE TABLE finding_triage (occurrence_id TEXT, status TEXT, close_reason TEXT);", + "CREATE TABLE finding_locations (occurrence_id TEXT, relative_path TEXT, role TEXT, sort_order INTEGER);", + "''')", + "scans = {name: {'id': name, 'status': 'complete', 'target_id': 'target', 'target_path': '/synthetic/repository'} for name in ('before', 'after')}", + "paths = ['src/parser/input.ts', 'src/parser-other/input.ts', 'src/parser/generated/code.ts', 'src/parser/test_fixture.ts']", + "for index, path in enumerate(paths):", + " key = str(index)", + " db.execute('INSERT INTO finding_occurrences VALUES (?, ?, ?, ?, ?)', (key, key, 'before', 'high', 'Synthetic finding'))", + " db.execute('INSERT INTO finding_locations VALUES (?, ?, ?, ?)', (key, path, 'root_control', 0))", + "coverage = {'mode': 'scoped_path', 'completeness': 'complete', 'includePaths': ['src/parser'], 'excludePaths': ['src/parser/generated'], 'explicitExclusions': [{'pattern': 'src/parser/test_*', 'reason': 'Excluded test fixtures'}]}", + "args = argparse.Namespace(before_scan_id='before', after_scan_id='after')", + "def compare():", + " result = history.compare_scans(db, args, require_scan=lambda _, key: scans[key], read_coverage=lambda _: coverage)", + " return {item['path']: item['status'] for item in result['findings']}", + "complete = compare()", + "coverage['completeness'] = 'partial'", + "print(json.dumps({'complete': complete, 'partial': compare()}))", + ].join("\n"); + const result = await runCodexCommand( + { command: python }, + ["-I", "-B", "-", join(PLUGIN_ROOT, "scripts")], + process.env, + probe, + AbortSignal.timeout(10_000), + ); + expect(result.exitCode, result.stderr).toBe(0); + expect(result.stderr).toBe(""); + const output = JSON.parse(result.stdout); + expect(output.complete).toEqual({ + "src/parser/input.ts": "resolved", + "src/parser-other/input.ts": "unknown", + "src/parser/generated/code.ts": "unknown", + "src/parser/test_fixture.ts": "unknown", + }); + expect(Object.values(output.partial)).toEqual([ + "unknown", + "unknown", + "unknown", + "unknown", + ]); +}); + test("loads oversized comparison matches from stdin", async () => { const python = await resolvePluginPython();