diff --git a/README.md b/README.md index cb7cc9f..922cfc1 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,8 @@ Open http://localhost:5000 in your browser. - [Plugin system](docs/plugins.md) - [Architecture](docs/architecture.md) - [Security model](docs/security.md) -- [Roadmap](docs/roadmap.md) +- [Soroban workflow guide](docs/soroban-workflow.md) +- [Roadmap](https://github.com/AyinkxLab/ai-code-assistant/issues) ## License diff --git a/app/reviews/routes.py b/app/reviews/routes.py index 2002612..1c2a98e 100644 --- a/app/reviews/routes.py +++ b/app/reviews/routes.py @@ -321,14 +321,39 @@ def _run_project_review(data: dict): def _save_result(review: Review, result: dict, config: dict) -> None: - """Persist a completed (or failed) review together with its findings.""" - review.summary = json.dumps(result.get("summary") or {}) - review.config = json.dumps(config) - findings = [ - ReviewFinding(review_id=review.id, **finding) for finding in result.get("findings") or [] - ] - review.findings_count = len(findings) - db.session.add_all(findings) + """Persist a completed (or failed) review together with its findings. + + Defense in depth: ``result`` is re-validated before anything is written, + so even a caller-supplied dict cannot invent files, findings, coverage + numbers, or metrics. Unvalidated entries are rejected, not coerced. + """ + if not isinstance(result, dict): + result = {} + summary = result.get("summary") + if not isinstance(summary, dict): + summary = {} + cap = None + if isinstance(config, dict): + cap = config.get("max_findings") + validated = reviews_service.validate_review_result( + { + "summary": summary, + "findings": result.get("findings"), + "known_files": result.get("known_files"), + }, + kind=review.kind, + known_files=result.get("known_files"), + max_findings=cap, + ) + findings = validated["findings"] + review.summary = json.dumps(validated["summary"]) + try: + review.config = json.dumps(config) + except (TypeError, ValueError): + review.config = "{}" + rows = [ReviewFinding(review_id=review.id, **finding) for finding in findings] + review.findings_count = len(rows) + db.session.add_all(rows) error = result.get("error") if error: review.status = STATUS_FAILED diff --git a/app/services/reviews.py b/app/services/reviews.py index 6c20c81..2a58da8 100644 --- a/app/services/reviews.py +++ b/app/services/reviews.py @@ -11,8 +11,11 @@ All prompts treat repository content as untrusted data and request a strict JSON payload so findings can be persisted as structured rows. The parser is -defensive: when the model does not return valid JSON (e.g. the offline mock -provider), it falls back to a text summary and no fabricated findings. +strict: parsed model output that cannot be validated is rejected, not coerced. +Unknown severities, categories, confidences, malformed lines, non-string +fields, invented file paths, and fabricated coverage percentages/metrics are +dropped rather than defaulted, so the service never invents files, findings, +coverage numbers, or metrics. """ from __future__ import annotations @@ -78,6 +81,14 @@ } ] } +Validation rules (unvalidated output is rejected, not defaulted): +- severity must be exactly one of critical|high|medium|low|informational +- category must match the review kind's vocabulary; unknown values rejected +- confidence must be exactly confirmed|potential|suggestion +- file must name a file from the shown context; unknown paths rejected +- line must be an integer >= 1 or null; strings/floats rejected +- explanation is required; missing or non-string explanations rejected +- never report coverage percentages, metrics, or files not shown Use empty arrays for sections with no content. findings may be empty. """ @@ -206,54 +217,235 @@ def _extract_json_object(text: str) -> dict | None: return None +def _looks_like_coverage_claim(text: str) -> bool: + """Return ``True`` when ``text`` looks like an invented coverage number. + + The review service has no coverage runner, so any coverage percentage or + coverage-with-a-number claim in model output is unvalidated and must be + rejected, never persisted. Bare uses of "cover" without a number (e.g. + "cover edge cases") are allowed. + """ + if not isinstance(text, str) or not text: + return False + lowered = text.lower() + if "coverage" not in lowered and "cover" not in lowered: + return False + if re.search(r"\d+\s*%|\bpercent\b", lowered): + return True + return bool(re.search(r"\bcoverage\b\s*[:=]?\s*\d", lowered)) + + +def _valid_file_syntax(value) -> str | None: + """Validate a model-supplied file path's syntax (no known-file check). + + Returns the normalized path or ``None`` when the value cannot be + validated. Rejects non-strings, empties, traversal (``..``), backslashes, + null bytes, and control characters instead of coercing them. + """ + if not isinstance(value, str): + return None + cleaned = value.strip().lstrip("/")[:2000] + if not cleaned: + return None + if "\x00" in cleaned or "\\" in cleaned: + return None + if any(ord(char) < 32 for char in cleaned): + return None + parts = cleaned.split("/") + if any(part in ("", ".", "..") for part in parts): + return None + return cleaned + + +def _normalize_known_files(known_files) -> set[str] | None: + """Normalize an allow-list of real file paths, or ``None`` when absent. + + ``None`` (no allow-list supplied) disables the known-file check. Any + supplied iterable — including an empty one, meaning "no files were in + context" — enables it, so invented paths are rejected. + """ + if known_files is None: + return None + normalized: set[str] = set() + if isinstance(known_files, (str, bytes)): + return normalized + try: + iterator = iter(known_files) # type: ignore[arg-type] + except TypeError: + return None + for entry in iterator: + path = _valid_file_syntax(entry) + if path is not None: + normalized.add(path) + return normalized + + +def _accepted_paths(paths, allowed_files: set[str] | None) -> list[str]: + """Return syntactically valid paths, optionally restricted to ``allowed_files``. + + Invented or malformed paths are dropped; nothing is rewritten to a nearby + known file. + """ + accepted: list[str] = [] + seen: set[str] = set() + for path in paths or []: + if not isinstance(path, str): + continue + cleaned = _valid_file_syntax(path) + if cleaned is None: + continue + if allowed_files is not None and cleaned not in allowed_files: + continue + if cleaned in seen: + continue + seen.add(cleaned) + accepted.append(cleaned) + return accepted + + +def _max_findings_cap(max_findings) -> int: + """Resolve the findings cap, defaulting to ``REVIEW_MAX_FINDINGS``.""" + if max_findings is None: + try: + from flask import current_app, has_app_context + + if has_app_context(): + max_findings = int(current_app.config.get("REVIEW_MAX_FINDINGS", 100)) + else: + max_findings = 100 + except Exception: + max_findings = 100 + try: + return max(0, int(max_findings)) + except (TypeError, ValueError): + return 100 + + def _as_string_list(value) -> list[str]: + """Return validated strings from ``value`` without coercing types. + + Only ``str`` items are accepted; numbers, dicts, and other types are + rejected (dropped) rather than stringified so the service never invents + summary content from unvalidated model output. A single string is split + into lines for convenience. Items that look like fabricated coverage + claims are dropped. + """ + items: list[str] = [] if isinstance(value, str): - return [item.strip() for item in value.splitlines() if item.strip()] - if isinstance(value, list): - return [str(item).strip() for item in value if str(item).strip()] - return [] + candidates = value.splitlines() + elif isinstance(value, list): + candidates = value + else: + return [] + for item in candidates: + if not isinstance(item, str): + continue + text = item.strip()[:2000] + if not text: + continue + if _looks_like_coverage_claim(text): + continue + items.append(text) + return items -def _normalize_summary(data: dict | None, raw: str) -> dict: +def _normalize_summary(data, raw: str) -> dict: + """Validate a model summary without coercing unvalidated values. + + Only ``str`` overall assessments and ``str`` list items are accepted. + Non-string values, fabricated coverage claims, and unknown keys are + rejected (dropped), never coerced into persisted summary content. + """ summary: dict = {} + source = data if isinstance(data, dict) else {} + overall = source.get("overall_assessment") + overall_text = overall.strip()[:6000] if isinstance(overall, str) else "" + overall_text = overall_text.strip() + if overall_text and not _looks_like_coverage_claim(overall_text): + summary["overall_assessment"] = overall_text + else: + summary["overall_assessment"] = "" for key in SUMMARY_KEYS: - if key in ("overall_assessment",): - summary[key] = str((data or {}).get(key) or "").strip() - else: - summary[key] = _as_string_list((data or {}).get(key)) + if key == "overall_assessment": + continue + summary[key] = _as_string_list(source.get(key)) summary["raw"] = raw return summary -def _coerce_finding(raw, kind: str, threshold_rank: int | None) -> dict | None: - """Validate and normalize a single finding, or return ``None`` to skip.""" +def _validate_finding( + raw, + kind: str, + threshold_rank: int | None, + known_files: set[str] | None = None, +) -> dict | None: + """Validate a single finding, or return ``None`` to reject it. + + Rejects (never coerces): non-dict entries, missing/non-string + explanations, unknown severity/category/confidence values, non-integer or + out-of-range lines, invalid file paths, files outside ``known_files`` when + provided, non-string recommendations, and fabricated coverage claims. + """ if not isinstance(raw, dict): return None - explanation = str(raw.get("explanation") or "").strip() + explanation = raw.get("explanation") + if not isinstance(explanation, str): + return None + explanation = explanation.strip()[:6000] if not explanation: return None + if _looks_like_coverage_claim(explanation): + return None - severity = str(raw.get("severity") or "medium").strip().lower() + severity = raw.get("severity") + if not isinstance(severity, str): + return None + severity = severity.strip().lower() if severity not in SEVERITIES: - severity = "medium" + return None if threshold_rank is not None and _SEVERITY_RANK[severity] > threshold_rank: return None - category = str(raw.get("category") or "other").strip().lower() + category = raw.get("category") + if not isinstance(category, str): + return None + category = category.strip().lower() allowed = CATEGORIES_BY_KIND.get(kind) or ("other",) if category not in allowed: - category = "other" + return None - confidence = str(raw.get("confidence") or "suggestion").strip().lower() + confidence = raw.get("confidence") + if not isinstance(confidence, str): + return None + confidence = confidence.strip().lower() if confidence not in CONFIDENCES: - confidence = "suggestion" + return None + + file_path: str | None = None + if raw.get("file") is not None: + file_path = _valid_file_syntax(raw.get("file")) + if file_path is None: + return None + if known_files is not None and file_path not in known_files: + return None - file_path = str(raw.get("file") or "").strip().lstrip("/")[:2000] or None line = raw.get("line") - try: - line = int(line) if line is not None else None - except (TypeError, ValueError): - line = None + if line is not None: + if isinstance(line, bool) or not isinstance(line, int): + return None + if line < 1 or line > 10_000_000: + return None + + recommendation: str | None = None + if raw.get("recommendation") is not None: + candidate = raw.get("recommendation") + if not isinstance(candidate, str): + return None + candidate = candidate.strip()[:6000] + if candidate: + if _looks_like_coverage_claim(candidate): + return None + recommendation = candidate return { "file": file_path, @@ -261,11 +453,18 @@ def _coerce_finding(raw, kind: str, threshold_rank: int | None) -> dict | None: "severity": severity, "category": category, "explanation": explanation, - "recommendation": str(raw.get("recommendation") or "").strip() or None, + "recommendation": recommendation, "confidence": confidence, } +# Backwards-compatible alias: the old coercing helper is gone; any external +# import of ``_coerce_finding`` now gets strict validation. +def _coerce_finding(raw, kind: str, threshold_rank: int | None, known_files=None) -> dict | None: + """Strict alias for :func:`_validate_finding` (reject, don't coerce).""" + return _validate_finding(raw, kind, threshold_rank, known_files) + + def _rank_threshold(threshold: str | None) -> int | None: threshold = (threshold or "").strip().lower() if threshold in _SEVERITY_RANK: @@ -273,23 +472,69 @@ def _rank_threshold(threshold: str | None) -> int | None: return None -def parse_review_response(text: str, *, kind: str, threshold: str | None = None) -> dict: - """Parse a provider response into ``{"summary", "findings", "raw"}``.""" - raw = (text or "").strip() +def parse_review_response( + text: str, + *, + kind: str, + threshold: str | None = None, + known_files=None, + max_findings: int | None = None, +) -> dict: + """Parse and validate a provider response into summary + findings. + + Guardrails (reject, never coerce): + + * The payload must be a JSON object; ``findings`` must be a list and + ``summary`` a dict — anything else yields no findings and an empty + validated summary (never prose-derived findings or metrics). + * Each finding is validated by :func:`_validate_finding`: unknown + severity/category/confidence, non-integer lines, invalid paths, files + outside ``known_files`` (when supplied), and coverage fabrications are + rejected. + * ``files_affected`` entries are validated against ``known_files`` when + supplied; invented paths are dropped. + * Coverage-looking summary items are dropped; the service has no coverage + runner and never reports measured percentages. + * Findings are capped at ``max_findings`` (or ``REVIEW_MAX_FINDINGS`` + outside tests) so oversized model output cannot inflate metrics. + """ + raw = text.strip()[:20000] if isinstance(text, str) else "" threshold_rank = _rank_threshold(threshold) + allowed_files = _normalize_known_files(known_files) + max_findings = _max_findings_cap(max_findings) payload = _extract_json_object(raw) + if max_findings == 0: + return { + "summary": ( + _normalize_summary(payload.get("summary"), raw) + if payload is not None + else _normalize_summary(None, raw) + ), + "findings": [], + "raw": raw, + "error": None, + "known_files": sorted(allowed_files) if allowed_files is not None else None, + } if payload is not None: + raw_findings = payload.get("findings") + if not isinstance(raw_findings, list): + raw_findings = [] findings = [] - for item in payload.get("findings") or []: - finding = _coerce_finding(item, kind, threshold_rank) + for item in raw_findings: + finding = _validate_finding(item, kind, threshold_rank, allowed_files) if finding is not None: findings.append(finding) + if len(findings) >= max_findings: + break + summary = _normalize_summary(payload.get("summary"), raw) + summary["files_affected"] = _accepted_paths(summary.get("files_affected"), allowed_files) return { - "summary": _normalize_summary(payload.get("summary") or {}, raw), + "summary": summary, "findings": findings, "raw": raw, "error": None, + "known_files": sorted(allowed_files) if allowed_files is not None else None, } return { @@ -297,16 +542,61 @@ def parse_review_response(text: str, *, kind: str, threshold: str | None = None) "findings": [], "raw": raw, "error": None, + "known_files": sorted(allowed_files) if allowed_files is not None else None, } +def validate_review_result(result, *, kind: str, known_files=None, max_findings=None) -> dict: + """Re-validate an already-parsed review result before persistence. + + Used as defense in depth by the route layer: ``result`` may come from the + parser or from any caller-supplied dict (tests, retries). Only validated + summary keys plus findings that pass :func:`_validate_finding` survive; + everything else — invented files, coverage numbers/metrics, unknown + vocabularies, malformed rows — is rejected, never coerced. Findings are + capped at ``max_findings`` so oversized output cannot inflate metrics. + """ + if not isinstance(result, dict): + result = {} + if known_files is None and "known_files" in result: + known_files = result.get("known_files") + allowed_files = _normalize_known_files(known_files) + max_findings = _max_findings_cap(max_findings) + raw_findings = result.get("findings") + if not isinstance(raw_findings, list): + raw_findings = [] + findings = [] + for item in raw_findings: + if len(findings) >= max_findings: + break + finding = _validate_finding(item, kind, None, allowed_files) + if finding is not None: + findings.append(finding) + summary = _normalize_summary(result.get("summary"), "") + summary.pop("raw", None) + summary["files_affected"] = _accepted_paths(summary.get("files_affected"), allowed_files) + return {"summary": summary, "findings": findings} + + # -------------------------------------------------------------------------- # Provider calls # -------------------------------------------------------------------------- -def _run_json(prompt: str, *, kind: str, threshold: str | None = None) -> dict: - """Run a completion and parse the structured response.""" +def _run_json( + prompt: str, + *, + kind: str, + threshold: str | None = None, + known_files=None, + max_findings: int | None = None, +) -> dict: + """Run a completion and parse the structured response. + + ``known_files`` (when supplied) is the allow-list of real file paths from + the bounded review context; findings or ``files_affected`` entries outside + it are rejected so the model cannot invent files. + """ try: provider = get_provider() text = provider.complete( @@ -321,8 +611,11 @@ def _run_json(prompt: str, *, kind: str, threshold: str | None = None) -> dict: "findings": [], "raw": "", "error": str(exc), + "known_files": list(known_files) if known_files is not None else None, } - parsed = parse_review_response(text, kind=kind, threshold=threshold) + parsed = parse_review_response( + text, kind=kind, threshold=threshold, known_files=known_files, max_findings=max_findings + ) parsed["error"] = None return parsed @@ -347,23 +640,48 @@ def build_pr_context(pr: dict, files: list[dict], config: dict) -> dict: budget = max(2000, int(config.get("max_context_chars") or 2000)) selected = [] - for file in files: + for file in files or []: if len(selected) >= max_files: break - if languages and not _matches_languages(file.get("filename") or "", languages): + if not isinstance(file, dict): + continue + filename = file.get("filename") + if not isinstance(filename, str) or not filename.strip(): + continue + if languages and not _matches_languages(filename or "", languages): continue selected.append(file) changed = [] per_file = max(budget // max(len(selected), 1), 2000) for file in selected: - patch = _clip(file.get("patch") or "", per_file) - changed.append( - f"- {file.get('filename')} ({file.get('status')}, " - f"+{file.get('additions')}/-{file.get('deletions')})\n{patch}" - ) - - test_files = [f.get("filename") for f in selected if is_test_path(f.get("filename") or "")] + filename = file.get("filename") + if not isinstance(filename, str) or not _valid_file_syntax(filename): + continue + patch = file.get("patch") + patch = patch if isinstance(patch, str) else "" + patch = _clip(patch, per_file) + status = file.get("status") if isinstance(file.get("status"), str) else "" + try: + additions = int(file.get("additions") or 0) + except (TypeError, ValueError): + additions = 0 + try: + deletions = int(file.get("deletions") or 0) + except (TypeError, ValueError): + deletions = 0 + changed.append(f"- {filename} ({status}, " f"+{additions}/-{deletions})\n{patch}") + + test_files = [ + f.get("filename") + for f in selected + if isinstance(f.get("filename"), str) and is_test_path(f.get("filename") or "") + ] + known_files = [ + f.get("filename") + for f in selected + if isinstance(f.get("filename"), str) and _valid_file_syntax(f.get("filename")) is not None + ] note = "" if files and len(selected) < len(files): note = ( @@ -377,14 +695,27 @@ def build_pr_context(pr: dict, files: list[dict], config: dict) -> dict: "files_text": body + note, "test_files": test_files, "selected_count": len(selected), - "total_count": len(files), + "total_count": len(files) if isinstance(files, list) else 0, + "known_files": known_files, } def review_pull_request(pr: dict, files: list[dict], config: dict) -> dict: """Review a pull request and return a structured summary + findings.""" - context = build_pr_context(pr, files, config) - description = _clip(pr.get("body") or "(no description provided)", 8000) + context = build_pr_context(pr if isinstance(pr, dict) else {}, files, config) + if not isinstance(pr, dict): + pr = {} + if not isinstance(config, dict): + config = {} + number = pr.get("number") if isinstance(pr.get("number"), int) else None + title = pr.get("title") if isinstance(pr.get("title"), str) else "" + state = pr.get("state") if isinstance(pr.get("state"), str) else "" + merged = pr.get("merged") if isinstance(pr.get("merged"), bool) else False + author = pr.get("author") if isinstance(pr.get("author"), str) else "" + base = pr.get("base") if isinstance(pr.get("base"), str) else "" + head = pr.get("head") if isinstance(pr.get("head"), str) else "" + body = pr.get("body") if isinstance(pr.get("body"), str) else "(no description provided)" + description = _clip(body, 8000) tests_note = ( ", ".join(context["test_files"]) if context["test_files"] @@ -398,10 +729,10 @@ def review_pull_request(pr: dict, files: list[dict], config: dict) -> dict: focus_note = " and ".join(focus) or "general" prompt = ( - f"Pull request #{pr.get('number')}: {pr.get('title')}\n" - f"State: {pr.get('state')} (merged: {pr.get('merged')})\n" - f"Author: {pr.get('author')}\n" - f"Base: {pr.get('base')} -> Head: {pr.get('head')}\n\n" + f"Pull request #{number}: {title}\n" + f"State: {state} (merged: {merged})\n" + f"Author: {author}\n" + f"Base: {base} -> Head: {head}\n\n" f"Description:\n{description}\n\n" f"Changed files:\n{context['files_text']}\n\n" f"Test files in this change:\n{tests_note}\n\n" @@ -412,7 +743,13 @@ def review_pull_request(pr: dict, files: list[dict], config: dict) -> dict: "proves the issue, otherwise 'potential' or 'suggestion'. Missing tests " "are best captured as a finding with category 'tests'.\n" + _JSON_SCHEMA ) - return _run_json(prompt, kind="pr", threshold=config.get("severity_threshold")) + return _run_json( + prompt, + kind="pr", + threshold=config.get("severity_threshold"), + known_files=context.get("known_files"), + max_findings=config.get("max_findings"), + ) # -------------------------------------------------------------------------- @@ -422,9 +759,17 @@ def review_pull_request(pr: dict, files: list[dict], config: dict) -> dict: def _project_context(project, config: dict, kind: str) -> dict: """Return bounded source/test file context for a project review.""" + if not isinstance(config, dict): + config = {} languages = config.get("languages") - budget = max(2000, int(config.get("max_context_chars") or 2000)) - max_files = max(1, int(config.get("max_files") or 1)) + try: + budget = max(2000, int(config.get("max_context_chars") or 2000)) + except (TypeError, ValueError): + budget = 2000 + try: + max_files = max(1, int(config.get("max_files") or 1)) + except (TypeError, ValueError): + max_files = 1 files = [f for f in _text_files(project) if _matches_languages(f.path, languages)] source_files = files[:max_files] @@ -438,12 +783,18 @@ def _project_context(project, config: dict, kind: str) -> dict: budget=budget // 2, ) + known_files = [f.path for f in source_files if _valid_file_syntax(f.path) is not None] + if kind == "tests": + for row in [f for f in files if f.path in test_files][: max_files // 2]: + if _valid_file_syntax(row.path) is not None and row.path not in known_files: + known_files.append(row.path) return { "blocks": blocks, "test_files": test_files[:200], "test_blocks": test_blocks, "structure": None, "count": len(source_files), + "known_files": known_files, } @@ -491,7 +842,15 @@ def analyze_code_quality(project, config: dict) -> dict: "Set confidence 'confirmed' only when the shown files prove the issue; " "otherwise use 'potential' or 'suggestion'.\n" + _JSON_SCHEMA ) - return _run_json(prompt, kind="quality", threshold=config.get("severity_threshold")) + if not isinstance(config, dict): + config = {} + return _run_json( + prompt, + kind="quality", + threshold=config.get("severity_threshold"), + known_files=context.get("known_files"), + max_findings=config.get("max_findings"), + ) _TEST_CATEGORY_HINT = ( @@ -540,7 +899,15 @@ def analyze_tests(project, config: dict) -> dict: "Set confidence 'confirmed' only when the shown files prove the issue; " "otherwise use 'potential' or 'suggestion'.\n" + _JSON_SCHEMA ) - return _run_json(prompt, kind="tests", threshold=config.get("severity_threshold")) + if not isinstance(config, dict): + config = {} + return _run_json( + prompt, + kind="tests", + threshold=config.get("severity_threshold"), + known_files=context.get("known_files"), + max_findings=config.get("max_findings"), + ) _SECURITY_CATEGORY_HINT = ( @@ -582,7 +949,15 @@ def review_project(project, kind: str, config: dict) -> dict: "Set confidence 'confirmed' only when the shown files prove the issue; " "otherwise use 'potential' or 'suggestion'.\n" + _JSON_SCHEMA ) - return _run_json(prompt, kind="security", threshold=config.get("severity_threshold")) + if not isinstance(config, dict): + config = {} + return _run_json( + prompt, + kind="security", + threshold=config.get("severity_threshold"), + known_files=context.get("known_files"), + max_findings=config.get("max_findings"), + ) # -------------------------------------------------------------------------- diff --git a/docs/reviews.md b/docs/reviews.md index b5086a0..8eccc82 100644 --- a/docs/reviews.md +++ b/docs/reviews.md @@ -56,6 +56,25 @@ Findings use a normalized vocabulary (see `app/models/review_finding.py`): A finding carries a file path and line only — never raw repository content. +### Output validation (reject, never coerce) + +Parsed model output that cannot be validated is rejected, never coerced: + +- Unknown severity, category, or confidence values are dropped (no defaulting + to `medium`/`other`/`suggestion`). +- Findings need a string explanation; non-integer, boolean, zero, negative, + or oversized lines are dropped. +- File paths must be syntactically valid (no traversal, backslashes, or + control characters) and, when the review context supplies a known-file + allow-list, must name a file from that context — invented paths and + `files_affected` entries are dropped. +- Coverage percentages and coverage-with-a-number claims are dropped from + summaries, findings, and recommendations: the service has no coverage + runner and never reports measured percentages. Metrics are computed only + from persisted `Review`/`ReviewFinding` rows, never from model text. +- Non-object payloads, non-list `findings`, non-dict `summary`, and non-string + summary items are dropped; findings are capped at `REVIEW_MAX_FINDINGS`. + ## Environment settings All review behaviour is environment-driven and documented in diff --git a/tests/test_reviews_api.py b/tests/test_reviews_api.py index 1354df6..eb27380 100644 --- a/tests/test_reviews_api.py +++ b/tests/test_reviews_api.py @@ -21,7 +21,7 @@ "file": "app/main.py", "line": 2, "severity": "high", - "category": "bug", + "category": "complexity", "explanation": "Unchecked input.", "recommendation": "Validate it.", "confidence": "confirmed", @@ -94,7 +94,7 @@ def _pr_dict(): def _pr_files(): return [ { - "filename": "app.py", + "filename": "app/main.py", "patch": "@@\n+def f()", "status": "modified", "additions": 2, @@ -136,6 +136,69 @@ def test_run_project_review(self, client, make_user, login, monkeypatch): assert review.summary_dict["overall_assessment"] == "Looks fine." assert ReviewFinding.query.filter_by(review_id=review.id).count() == 1 + def test_invented_files_and_coverage_are_rejected_not_persisted( + self, client, make_user, login, monkeypatch + ): + user = make_user() + login() + project = _make_project(user) + invented = json.dumps( + { + "summary": { + "overall_assessment": "Coverage is 99%.", + "files_affected": ["app/main.py", "not/real.py"], + "testing_recommendations": ["Raise coverage to 90 percent."], + }, + "findings": [ + { + "file": "not/real.py", + "line": 1, + "severity": "high", + "category": "complexity", + "explanation": "Invented file.", + "confidence": "confirmed", + }, + { + "file": "app/main.py", + "line": 2, + "severity": "high", + "category": "complexity", + "explanation": "Coverage dropped to 12%.", + "confidence": "confirmed", + }, + { + "file": "app/main.py", + "line": 3, + "severity": "medium", + "category": "complexity", + "explanation": "Nested conditionals.", + "confidence": "confirmed", + }, + ], + } + ) + monkeypatch.setattr("app.services.reviews.get_provider", lambda: FakeProvider(invented)) + response = client.post( + "/reviews/api/reviews", + json={"source": "project", "project_id": project.id, "kind": "quality"}, + ) + assert response.status_code == 201 + payload = response.get_json() + assert payload["findings_count"] == 1 + review = db.session.get(Review, payload["id"]) + assert review.summary_dict["overall_assessment"] == "" + assert review.summary_dict["files_affected"] == ["app/main.py"] + assert review.summary_dict["testing_recommendations"] == [] + rows = ReviewFinding.query.filter_by(review_id=review.id).all() + assert len(rows) == 1 + assert rows[0].file == "app/main.py" + assert rows[0].line == 3 + assert "12%" not in (rows[0].explanation or "") + + metrics = client.get("/reviews/api/metrics").get_json() + assert metrics["findings"]["total"] == 1 + assert "not/real.py" not in metrics["findings"]["by_file"] + def test_run_project_review_not_ready(self, client, make_user, login): user = make_user() login() @@ -645,7 +708,7 @@ def _make_review(self, user, with_finding=True): file="app/main.py", line=2, severity="high", - category="bug", + category="readability", explanation="Unchecked input.", recommendation="Validate it.", confidence="confirmed", @@ -696,7 +759,7 @@ def test_export_markdown(self, client, make_user, login): assert "## Summary" in text assert "Looks fine." in text assert "## Findings" in text - assert "[HIGH] bug - app/main.py:2" in text + assert "[HIGH] readability - app/main.py:2" in text assert "[CONFIRMED]" in text assert "**Recommendation:** Validate it." in text @@ -762,7 +825,20 @@ def boom(*args, **kwargs): def fake_review_project(proj, kind, config): captured["config"] = config - return {"summary": {"overall_assessment": "retried"}, "findings": [], "raw": ""} + return { + "summary": {"overall_assessment": "retried"}, + "findings": [ + { + "file": "app/main.py", + "line": 1, + "severity": "low", + "category": "maintainability", + "explanation": "Retry note.", + "confidence": "suggestion", + } + ], + "raw": "", + } monkeypatch.setattr( "app.reviews.routes.reviews_service.review_project", fake_review_project diff --git a/tests/test_reviews_delete.py b/tests/test_reviews_delete.py index 4399cc2..699f65a 100644 --- a/tests/test_reviews_delete.py +++ b/tests/test_reviews_delete.py @@ -18,7 +18,7 @@ { "file": "app/main.py", "severity": "high", - "category": "bug", + "category": "maintainability", "explanation": "e", "confidence": "confirmed", } diff --git a/tests/test_reviews_service.py b/tests/test_reviews_service.py index 71ceef1..85c8c84 100644 --- a/tests/test_reviews_service.py +++ b/tests/test_reviews_service.py @@ -63,7 +63,7 @@ def complete(self, messages, *, stream=False): "file": "app/main.py", "line": 4, "severity": "critical", - "category": "bug", + "category": "complexity", "explanation": "Null dereference when data is missing.", "recommendation": "Guard against None.", "confidence": "confirmed", @@ -71,8 +71,8 @@ def complete(self, messages, *, stream=False): { "file": "app/main.py", "line": 9, - "severity": "informational", - "category": "other", + "severity": "low", + "category": "maintainability", "explanation": "Style note only.", "recommendation": "", "confidence": "suggestion", @@ -104,10 +104,11 @@ def complete(self, messages, *, stream=False): "confidence": "potential", }, { - "file": "app/dup.py", + "file": "app/legacy.py", + "line": 5, "severity": "medium", "category": "duplication", - "explanation": "Duplicated parsing logic.", + "explanation": "Duplicated parsing logic in the legacy module.", "recommendation": "Reuse a single function.", "confidence": "confirmed", }, @@ -138,7 +139,8 @@ def complete(self, messages, *, stream=False): "confidence": "confirmed", }, { - "file": "tests/test_worker.py", + "file": "tests/test_app.py", + "line": 9, "severity": "medium", "category": "flaky-test", "explanation": "The test depends on a real timer sleep.", @@ -152,7 +154,7 @@ def complete(self, messages, *, stream=False): class TestParseReviewResponse: def test_parses_json_findings(self, app): - result = reviews.parse_review_response(JSON_REPLY, kind="pr") + result = reviews.parse_review_response(JSON_REPLY, kind="quality") assert result["error"] is None assert result["summary"]["overall_assessment"] == "Solid overall." assert len(result["findings"]) == 2 @@ -163,13 +165,13 @@ def test_parses_json_findings(self, app): assert finding["confidence"] == "confirmed" def test_threshold_drops_low_severity(self, app): - result = reviews.parse_review_response(JSON_REPLY, kind="pr", threshold="high") + result = reviews.parse_review_response(JSON_REPLY, kind="quality", threshold="high") severities = [f["severity"] for f in result["findings"]] assert severities == ["critical"] def test_markdown_fenced_json(self, app): fenced = "```json\n" + JSON_REPLY + "\n```" - result = reviews.parse_review_response(fenced, kind="pr") + result = reviews.parse_review_response(fenced, kind="quality") assert len(result["findings"]) == 2 def test_unstructured_text_falls_back(self, app): @@ -177,7 +179,7 @@ def test_unstructured_text_falls_back(self, app): assert result["findings"] == [] assert "raw" in result["summary"] - def test_invalid_finding_entries_skipped(self, app): + def test_invalid_finding_entries_rejected_not_coerced(self, app): payload = { "summary": {"overall_assessment": "x"}, "findings": [ @@ -185,44 +187,103 @@ def test_invalid_finding_entries_skipped(self, app): {"severity": "high"}, {"explanation": " "}, {"file": "a.py", "line": "nope", "explanation": "ok", "category": "bogus"}, + {"file": "a.py", "line": 3, "explanation": "ok", "category": "bogus"}, + { + "file": "a.py", + "line": 3, + "severity": "bogus", + "explanation": "ok", + "category": "bug", + "confidence": "confirmed", + }, + { + "file": "a.py", + "line": 3, + "severity": "high", + "explanation": "kept", + "category": "bug", + "confidence": "confirmed", + }, ], } result = reviews.parse_review_response(json.dumps(payload), kind="pr") assert len(result["findings"]) == 1 finding = result["findings"][0] assert finding["file"] == "a.py" - assert finding["line"] is None - assert finding["category"] == "other" - assert finding["severity"] == "medium" + assert finding["line"] == 3 + assert finding["category"] == "bug" + assert finding["severity"] == "high" def test_categories_scoped_by_kind(self, app): payload = { - "findings": [{"explanation": "x", "category": "missing-tests"}], + "findings": [ + { + "explanation": "x", + "category": "missing-tests", + "severity": "low", + "confidence": "suggestion", + } + ], } pr_result = reviews.parse_review_response(json.dumps(payload), kind="pr") tests_result = reviews.parse_review_response(json.dumps(payload), kind="tests") - assert pr_result["findings"][0]["category"] == "other" + assert pr_result["findings"] == [] assert tests_result["findings"][0]["category"] == "missing-tests" def test_quality_categories_include_readability_and_dead_code(self, app): payload = { "findings": [ - {"explanation": "x", "category": "readability"}, - {"explanation": "y", "category": "dead-code"}, - {"explanation": "z", "category": "bug"}, + { + "explanation": "x", + "category": "readability", + "severity": "low", + "confidence": "suggestion", + }, + { + "explanation": "y", + "category": "dead-code", + "severity": "low", + "confidence": "suggestion", + }, + { + "explanation": "z", + "category": "bug", + "severity": "low", + "confidence": "suggestion", + }, ] } result = reviews.parse_review_response(json.dumps(payload), kind="quality") categories = [f["category"] for f in result["findings"]] - assert categories == ["readability", "dead-code", "other"] + assert categories == ["readability", "dead-code"] def test_tests_categories_include_new_test_concerns(self, app): payload = { "findings": [ - {"explanation": "x", "category": "coverage-gap"}, - {"explanation": "y", "category": "missing-assertion"}, - {"explanation": "z", "category": "flaky-test"}, - {"explanation": "w", "category": "readability"}, + { + "explanation": "x", + "category": "coverage-gap", + "severity": "low", + "confidence": "suggestion", + }, + { + "explanation": "y", + "category": "missing-assertion", + "severity": "low", + "confidence": "suggestion", + }, + { + "explanation": "z", + "category": "flaky-test", + "severity": "low", + "confidence": "suggestion", + }, + { + "explanation": "w", + "category": "readability", + "severity": "low", + "confidence": "suggestion", + }, ] } result = reviews.parse_review_response(json.dumps(payload), kind="tests") @@ -230,7 +291,6 @@ def test_tests_categories_include_new_test_concerns(self, app): "coverage-gap", "missing-assertion", "flaky-test", - "other", ] @@ -264,6 +324,7 @@ def test_language_filter(self, app): context = reviews.build_pr_context(pr, files, config) assert "app.py" in context["files_text"] assert "README.md" not in context["files_text"] + assert context["known_files"] == ["app.py"] def test_max_files_bounds_context(self, app): config = {"languages": None, "max_files": 2, "max_context_chars": 50000} @@ -324,7 +385,9 @@ def test_project_context_max_files(self, app): class TestAnalyzeCodeQuality: def test_structured_findings_cover_named_concerns(self, app, monkeypatch): - project = _ready_project([("app/main.py", "def f():\n pass\n")]) + project = _ready_project( + [("app/main.py", "def f():\n pass\n"), ("app/legacy.py", "def g():\n pass\n")] + ) monkeypatch.setattr(reviews, "get_provider", lambda: FakeProvider(QUALITY_JSON_REPLY)) config = { "languages": None, @@ -335,7 +398,7 @@ def test_structured_findings_cover_named_concerns(self, app, monkeypatch): result = reviews.analyze_code_quality(project, config) assert result["error"] is None categories = {f["category"] for f in result["findings"]} - assert {"readability", "dead-code", "duplication"} <= categories + assert categories == {"readability", "dead-code", "duplication"} def test_review_project_quality_delegates_to_analyze_code_quality(self, app, monkeypatch): project = _ready_project([("app/main.py", "x")]) @@ -395,14 +458,27 @@ def test_review_project_structured(self, app, monkeypatch): } result = reviews.review_project(project, "quality", config) assert result["error"] is None - # The critical finding survives; the informational one is below the - # "low" severity threshold and is correctly dropped. - assert len(result["findings"]) == 1 - assert result["findings"][0]["severity"] == "critical" + # Threshold "low" keeps critical through low; only informational is dropped. + assert [f["severity"] for f in result["findings"]] == ["critical", "low"] def test_review_project_unknown_kind_defaults_to_quality(self, app, monkeypatch): project = _ready_project([("app/main.py", "x")]) - monkeypatch.setattr(reviews, "get_provider", lambda: FakeProvider(JSON_REPLY)) + quality_reply = json.dumps( + { + "summary": {"overall_assessment": "ok"}, + "findings": [ + { + "file": "app/main.py", + "line": 1, + "severity": "low", + "category": "maintainability", + "explanation": "Note.", + "confidence": "suggestion", + } + ], + } + ) + monkeypatch.setattr(reviews, "get_provider", lambda: FakeProvider(quality_reply)) config = { "languages": None, "max_files": 40, @@ -413,7 +489,22 @@ def test_review_project_unknown_kind_defaults_to_quality(self, app, monkeypatch) assert result["findings"] def test_review_pull_request(self, app, monkeypatch): - monkeypatch.setattr(reviews, "get_provider", lambda: FakeProvider(JSON_REPLY)) + pr_reply = json.dumps( + { + "summary": {"overall_assessment": "ok"}, + "findings": [ + { + "file": "app/main.py", + "line": 1, + "severity": "high", + "category": "bug", + "explanation": "Unchecked input.", + "confidence": "confirmed", + } + ], + } + ) + monkeypatch.setattr(reviews, "get_provider", lambda: FakeProvider(pr_reply)) pr = { "number": 3, "title": "Add feature", @@ -432,6 +523,45 @@ def test_review_pull_request(self, app, monkeypatch): } result = reviews.review_pull_request(pr, files, config) assert result["findings"][0]["file"] == "app/main.py" + assert result["known_files"] == ["app/main.py"] + + def test_review_pull_request_rejects_invented_files(self, app, monkeypatch): + pr_reply = json.dumps( + { + "summary": {"files_affected": ["app/main.py", "secret.py"]}, + "findings": [ + { + "file": "secret.py", + "line": 1, + "severity": "high", + "category": "bug", + "explanation": "Invented.", + "confidence": "confirmed", + }, + { + "file": "app/main.py", + "line": 1, + "severity": "high", + "category": "bug", + "explanation": "Unchecked input.", + "confidence": "confirmed", + }, + ], + } + ) + monkeypatch.setattr(reviews, "get_provider", lambda: FakeProvider(pr_reply)) + result = reviews.review_pull_request( + {"number": 3, "title": "t"}, + [_pr_file("app/main.py")], + { + "languages": None, + "max_files": 40, + "max_context_chars": 40000, + "severity_threshold": "low", + }, + ) + assert [f["file"] for f in result["findings"]] == ["app/main.py"] + assert result["summary"]["files_affected"] == ["app/main.py"] def test_provider_error_reported(self, app, monkeypatch): from app.services.llm import LLMProviderError @@ -568,6 +698,86 @@ def test_every_kind_returns_structured_location_and_vocabulary(self, app): assert finding["file"] == "a.py" assert finding["line"] == 3 + def test_unknown_vocabularies_are_rejected_not_defaulted(self, app): + base = { + "file": "a.py", + "line": 3, + "severity": "high", + "category": "bug", + "explanation": "real issue", + "confidence": "confirmed", + } + for field, bad in ( + ("severity", "critical-ish"), + ("category", "vibes"), + ("confidence", "certain"), + ): + payload = {"findings": [dict(base, **{field: bad})]} + assert reviews.parse_review_response(json.dumps(payload), kind="pr")["findings"] == [] + + def test_non_integer_and_out_of_range_lines_rejected(self, app): + for bad_line in ("3", 3.0, True, 0, -1, 10_000_001): + payload = { + "findings": [ + { + "file": "a.py", + "line": bad_line, + "severity": "high", + "category": "bug", + "explanation": "real issue", + "confidence": "confirmed", + } + ] + } + assert reviews.parse_review_response(json.dumps(payload), kind="pr")["findings"] == [] + + def test_unknown_files_rejected_when_allow_list_supplied(self, app): + payload = { + "summary": {"overall_assessment": "ok", "files_affected": ["app/main.py", "evil.py"]}, + "findings": [ + { + "file": "evil.py", + "line": 1, + "severity": "high", + "category": "bug", + "explanation": "invented", + "confidence": "confirmed", + }, + { + "file": "app/main.py", + "line": 1, + "severity": "high", + "category": "bug", + "explanation": "real", + "confidence": "confirmed", + }, + ], + } + result = reviews.parse_review_response( + json.dumps(payload), kind="pr", known_files=["app/main.py"] + ) + assert [f["file"] for f in result["findings"]] == ["app/main.py"] + assert result["summary"]["files_affected"] == ["app/main.py"] + assert result["known_files"] == ["app/main.py"] + + def test_empty_allow_list_rejects_every_path(self, app): + payload = { + "summary": {"files_affected": ["app/main.py"]}, + "findings": [ + { + "file": "app/main.py", + "line": 1, + "severity": "high", + "category": "bug", + "explanation": "invented without context", + "confidence": "confirmed", + } + ], + } + result = reviews.parse_review_response(json.dumps(payload), kind="pr", known_files=[]) + assert result["findings"] == [] + assert result["summary"]["files_affected"] == [] + def test_labels_instructed_consistently_for_pr_and_project(self, app, monkeypatch): project = _ready_project([("app/main.py", "x")]) provider = CapturingProvider(JSON_REPLY) @@ -606,6 +816,122 @@ def test_finding_serialization_has_no_raw_content(self, app): assert "raw_content" not in serialized assert {"file", "line", "severity", "category", "confidence"} <= set(serialized) + def test_traversal_and_empty_paths_rejected(self, app): + for bad in ("../secret.py", "", "/", "a//b.py"): + payload = { + "findings": [ + { + "file": bad, + "line": 1, + "severity": "high", + "category": "bug", + "explanation": "x", + "confidence": "confirmed", + } + ] + } + assert reviews.parse_review_response(json.dumps(payload), kind="pr")["findings"] == [] + + def test_coverage_numbers_and_metrics_rejected(self, app): + payload = { + "summary": { + "overall_assessment": "Coverage is 87% and looks great.", + "testing_recommendations": ["Coverage is now 92 percent."], + "files_affected": ["app/main.py"], + }, + "findings": [ + { + "file": "app/main.py", + "line": 1, + "severity": "medium", + "category": "coverage-gap", + "explanation": "Coverage dropped to 45%.", + "recommendation": "Raise coverage to 90%.", + "confidence": "potential", + } + ], + } + result = reviews.parse_review_response(json.dumps(payload), kind="tests") + assert result["findings"] == [] + assert result["summary"]["overall_assessment"] == "" + assert result["summary"]["testing_recommendations"] == [] + assert result["summary"]["files_affected"] == ["app/main.py"] + + allowed = reviews.parse_review_response( + json.dumps( + { + "summary": {"testing_recommendations": ["Cover edge cases."]}, + "findings": [], + } + ), + kind="tests", + ) + assert allowed["summary"]["testing_recommendations"] == ["Cover edge cases."] + + def test_non_string_fields_and_unknown_keys_rejected(self, app): + payload = { + "summary": {"overall_assessment": 42, "bogus": ["x"], "important_findings": [1, None]}, + "findings": [ + { + "file": "a.py", + "line": 1, + "severity": "high", + "category": "bug", + "explanation": "real", + "confidence": "confirmed", + "recommendation": 7, + } + ], + } + result = reviews.parse_review_response(json.dumps(payload), kind="pr") + assert result["findings"] == [] + assert result["summary"]["overall_assessment"] == "" + assert result["summary"]["important_findings"] == [] + assert "bogus" not in result["summary"] + + def test_non_list_findings_rejected(self, app): + payload = {"summary": {}, "findings": {"file": "a.py"}} + result = reviews.parse_review_response(json.dumps(payload), kind="pr") + assert result["findings"] == [] + + def test_findings_capped_and_validate_helper_rejects(self, app): + payload = { + "findings": [ + { + "file": f"a{i}.py", + "line": 1, + "severity": "low", + "category": "other", + "explanation": f"issue {i}", + "confidence": "suggestion", + } + for i in range(5) + ] + } + result = reviews.parse_review_response(json.dumps(payload), kind="pr", max_findings=2) + assert len(result["findings"]) == 2 + validated = reviews.validate_review_result( + {"summary": {"overall_assessment": 1}, "findings": payload["findings"]}, + kind="pr", + max_findings=2, + ) + assert len(validated["findings"]) == 2 + assert validated["summary"]["overall_assessment"] == "" + assert ( + reviews.validate_review_result( + {"summary": {}, "findings": payload["findings"]}, kind="pr", max_findings=0 + )["findings"] + == [] + ) + assert reviews.validate_review_result( + { + "summary": {"files_affected": ["evil.py", "a0.py"]}, + "findings": payload["findings"], + "known_files": ["a0.py"], + }, + kind="pr", + )["summary"]["files_affected"] == ["a0.py"] + #: A repository/PR payload that tries to hijack the reviewer's instructions. INJECTION_ATTEMPT = (