From 6a90d664dc2d992bb041081a94b383a526ad3628 Mon Sep 17 00:00:00 2001 From: CodeGovindz Date: Tue, 27 Oct 2026 00:47:38 +0530 Subject: [PATCH] P2: web app with live progress; ignore infrastructure CI failures Web: - POST /api/verify streams pipeline events as NDJSON; per-IP rate limit (10 runs/hour, in memory); maxDuration 300. - Home page with PR input and examples, live progress log, and the report rendered as a printed receipt (problems first, evidence chips, copy-as-PR-comment and share link). ?pr= auto-runs a shared link. Precision: - Failed check runs carry their first failure annotation; HTTP 5xx, runner shutdowns, "Authorization required" deploys and cancelled runs count as ignored, not failed. Cap#3412's Format (Biome) died on an HTTP 504 and was producing a false CONTRADICTED. - A CONTRADICTED verdict resting only on CI checks must cite a check that really failed. - Extraction: ticked checkboxes with checkable facts are claims; claim text no longer starts with "After this PR". Verifier reasoning is short plain English without internal field names. - Markdown report shows the self-contained claim text, not the quote. - env.ts: turbopackIgnore hints so the bundler doesn't trace the whole project into the serverless function. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/PLAN.md | 3 +- fixtures/expectations.json | 6 + fixtures/real-failing-ci.json | 95 ++++++++---- src/app/api/verify/route.ts | 70 +++++++++ src/app/globals.css | 63 ++++++-- src/app/layout.tsx | 5 +- src/app/page.tsx | 123 ++++++++-------- src/components/ProgressLog.tsx | 74 ++++++++++ src/components/ReceiptsApp.tsx | 184 +++++++++++++++++++++++ src/components/ReportView.tsx | 217 ++++++++++++++++++++++++++++ src/core/__tests__/guard.test.ts | 55 +++++++ src/core/__tests__/signals.test.ts | 63 +++++++- src/core/env.ts | 6 +- src/core/github.ts | 39 ++++- src/core/guard.ts | 15 ++ src/core/prompts.ts | 19 ++- src/core/report.ts | 5 +- src/core/signals.ts | 70 +++++++-- src/core/types.ts | 4 + src/lib/__tests__/ratelimit.test.ts | 46 ++++++ src/lib/ratelimit.ts | 40 +++++ 21 files changed, 1067 insertions(+), 135 deletions(-) create mode 100644 src/app/api/verify/route.ts create mode 100644 src/components/ProgressLog.tsx create mode 100644 src/components/ReceiptsApp.tsx create mode 100644 src/components/ReportView.tsx create mode 100644 src/lib/__tests__/ratelimit.test.ts create mode 100644 src/lib/ratelimit.ts diff --git a/docs/PLAN.md b/docs/PLAN.md index 34ddf3d..a5c81a6 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -130,7 +130,8 @@ Fixtures pinned under `fixtures/real-*.json` (bundle at head SHA). Highlights: " 3. Skipped/neutral/stale checks are ignored instead of making CI look "pending" forever. 4. `sanitizePrBody()` strips bot sections (e.g. Greptile's `` block), HTML comments and tags before extraction; the quote guard runs on the sanitized text. 5. CONTRADICTED below confidence 0.75 is downgraded to UNVERIFIED. -6. `npm run eval` runs every `fixtures/real-*.json` through the live models against `fixtures/expectations.json`. After the fixes: docs#46488 PASS, Cap#3412 MIXED (was a false FAIL), dispenso#129 MIXED, no CONTRADICTED anywhere. Extraction is not deterministic on Bedrock even at temperature 0, so expectations on single claims are "if extracted", not "required". +6. A failed CI check is not always a code failure. Cap's `Format (Biome)` died on `Unexpected HTTP response: 504` before running Biome, and its Vercel status failed on "Authorization required to deploy". The fetch now reads each failed run's first failure annotation (and legacy status descriptions); infrastructure failures and cancelled runs count as "ignored", and a CONTRADICTED verdict resting only on CI checks must cite one that really failed. +7. `npm run eval` runs every `fixtures/real-*.json` through the live models against `fixtures/expectations.json`. After the fixes: docs#46488 PASS, Cap#3412 MIXED (was a false FAIL, twice), dispenso#129 MIXED, no CONTRADICTED anywhere. Extraction is not deterministic on Bedrock even at temperature 0, so expectations on single claims are "if extracted", not "required". - CLI quirk (Windows PowerShell): `npm run verify -- --fresh --dump-fixture x` drops the unknown flags in npm's argument forwarding (the URL still works, and the flag VALUES become positionals!). `npx tsx scripts/verify.ts --json --fresh --dump-fixture x` works everywhere — document it in the README (P4). ## 6. Signals (deterministic, computed once per PR, no LLM) diff --git a/fixtures/expectations.json b/fixtures/expectations.json index 7a7496d..d90df14 100644 --- a/fixtures/expectations.json +++ b/fixtures/expectations.json @@ -30,6 +30,12 @@ "verdict": "UNVERIFIED", "required": true }, + { + "$comment": "'Format (Biome)' failed on 'Unexpected HTTP response: 504' while setting up (it never ran Biome), and 'Lint (Biome)' passed. A Biome claim must not be CONTRADICTED by an infrastructure failure.", + "type": "noClaimVerdict", + "textIncludes": "biome", + "verdict": "CONTRADICTED" + }, { "$comment": "A tests_pass contradiction is legitimate only if it cites a test-kind check.", "type": "contradictedMustCiteKind", diff --git a/fixtures/real-failing-ci.json b/fixtures/real-failing-ci.json index 642d494..b16baec 100644 --- a/fixtures/real-failing-ci.json +++ b/fixtures/real-failing-ci.json @@ -66,175 +66,204 @@ "name": "Rust tests (Linux)", "status": "completed", "conclusion": "skipped", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121753574971" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121753574971", + "failureMessage": null }, { "name": "GPUI patch tests (${{ matrix.settings.name }})", "status": "completed", "conclusion": "skipped", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121753574389" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121753574389", + "failureMessage": null }, { "name": "Verify Tauri plugin versions", "status": "completed", "conclusion": "skipped", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121753574201" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121753574201", + "failureMessage": null }, { "name": "Clippy", "status": "completed", "conclusion": "skipped", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121753573877" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121753573877", + "failureMessage": null }, { "name": "Build GPUI Desktop", "status": "completed", "conclusion": "skipped", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121753573825" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121753573825", + "failureMessage": null }, { "name": "matrix.platform", "status": "completed", "conclusion": "skipped", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374015733/job/121752900251" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374015733/job/121752900251", + "failureMessage": null }, { "name": "Deep review label", "status": "completed", "conclusion": "skipped", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374015733/job/121752899954" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374015733/job/121752899954", + "failureMessage": null }, { "name": "Engine verify", "status": "completed", "conclusion": "success", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374015733/job/121752897420" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374015733/job/121752897420", + "failureMessage": null }, { "name": "Perf budget", "status": "completed", "conclusion": "success", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014999/job/121752729176" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014999/job/121752729176", + "failureMessage": null }, { "name": "measure", "status": "completed", "conclusion": "skipped", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014999/job/121752715981" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014999/job/121752715981", + "failureMessage": null }, { "name": "Socket Security: Pull Request Alerts", "status": "completed", "conclusion": "success", - "htmlUrl": "https://github.com/CapSoftware/Cap/runs/121752164629" + "htmlUrl": "https://github.com/CapSoftware/Cap/runs/121752164629", + "failureMessage": null }, { "name": "real-capture", "status": "completed", "conclusion": "skipped", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014928/job/121752163261" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014928/job/121752163261", + "failureMessage": null }, { "name": "real-capture (windows-2022)", "status": "completed", "conclusion": "skipped", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014928/job/121752163083" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014928/job/121752163083", + "failureMessage": null }, { "name": "Verify recording", "status": "completed", "conclusion": "success", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014928/job/121752161138" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014928/job/121752161138", + "failureMessage": null }, { "name": "Vercel Agent Review", "status": "completed", "conclusion": "neutral", - "htmlUrl": "https://github.com/CapSoftware/Cap/runs/121752116939" + "htmlUrl": "https://github.com/CapSoftware/Cap/runs/121752116939", + "failureMessage": null }, { "name": "Superagent Security Scan", "status": "completed", "conclusion": "success", - "htmlUrl": "https://github.com/CapSoftware/Cap/runs/121752093068" + "htmlUrl": "https://github.com/CapSoftware/Cap/runs/121752093068", + "failureMessage": null }, { "name": "Contributor trust", "status": "completed", "conclusion": "neutral", - "htmlUrl": "https://github.com/CapSoftware/Cap/runs/121752086725" + "htmlUrl": "https://github.com/CapSoftware/Cap/runs/121752086725", + "failureMessage": null }, { "name": "Socket Security: Project Report", "status": "completed", "conclusion": "success", - "htmlUrl": "https://github.com/CapSoftware/Cap/runs/121752040618" + "htmlUrl": "https://github.com/CapSoftware/Cap/runs/121752040618", + "failureMessage": null }, { "name": "Vercel Preview Comments", "status": "completed", "conclusion": "success", - "htmlUrl": "https://github.com/CapSoftware/Cap/runs/121752040311" + "htmlUrl": "https://github.com/CapSoftware/Cap/runs/121752040311", + "failureMessage": null }, { "name": "Greptile Review", "status": "completed", "conclusion": "success", - "htmlUrl": "https://github.com/CapSoftware/Cap/runs/121752028683" + "htmlUrl": "https://github.com/CapSoftware/Cap/runs/121752028683", + "failureMessage": null }, { "name": "Plan", "status": "completed", "conclusion": "success", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374015733/job/121752017555" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374015733/job/121752017555", + "failureMessage": null }, { "name": "Typecheck", "status": "completed", "conclusion": "success", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121752017000" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121752017000", + "failureMessage": null }, { "name": "Format (Cargo)", "status": "completed", "conclusion": "success", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121752016774" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121752016774", + "failureMessage": null }, { "name": "Detect Changes", "status": "completed", "conclusion": "success", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121752016655" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121752016655", + "failureMessage": null }, { "name": "Lint (Biome)", "status": "completed", "conclusion": "success", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121752016636" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121752016636", + "failureMessage": null }, { "name": "Format (Biome)", "status": "completed", "conclusion": "failure", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121752016536" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121752016536", + "failureMessage": "Unexpected HTTP response: 504" }, { "name": "Mobile", "status": "completed", "conclusion": "success", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121752016219" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014764/job/121752016219", + "failureMessage": null }, { "name": "Scope", "status": "completed", "conclusion": "success", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014928/job/121752012623" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014928/job/121752012623", + "failureMessage": null }, { "name": "scope", "status": "completed", "conclusion": "success", - "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014999/job/121752012479" + "htmlUrl": "https://github.com/CapSoftware/Cap/actions/runs/40374014999/job/121752012479", + "failureMessage": null } ], "combinedStatus": { @@ -243,16 +272,18 @@ { "context": "Vercel – cap-status", "state": "success", - "targetUrl": "https://vercel.com/mc-ilroy/cap-status/7JEpFXmFbEWxyxt7bPaZK6VaxZFD" + "targetUrl": "https://vercel.com/mc-ilroy/cap-status/7JEpFXmFbEWxyxt7bPaZK6VaxZFD", + "description": "Skipped - Not affected" }, { "context": "Vercel – cap-web", "state": "failure", - "targetUrl": "https://vercel.com/git/authorize?team=Cap%20Software%20Inc&slug=mc-ilroy&teamId=team_vbZRU7UW78rpKKIj4c9PfFAC&type=github&job=%7B%22headInfo%22%3A%7B%22sha%22%3A%2292cc9220a77f97d784a6a711facda0ee8c53e3e4%22%7D%2C%22id%22%3A%22QmYD5yvL6jWwdRGE7p9Wa6hhuKz4FvTWPPRBebrgkY5UtK%22%2C%22org%22%3A%22CapSoftware%22%2C%22prId%22%3A3412%2C%22repo%22%3A%22Cap%22%7D" + "targetUrl": "https://vercel.com/git/authorize?team=Cap%20Software%20Inc&slug=mc-ilroy&teamId=team_vbZRU7UW78rpKKIj4c9PfFAC&type=github&job=%7B%22headInfo%22%3A%7B%22sha%22%3A%2292cc9220a77f97d784a6a711facda0ee8c53e3e4%22%7D%2C%22id%22%3A%22QmYD5yvL6jWwdRGE7p9Wa6hhuKz4FvTWPPRBebrgkY5UtK%22%2C%22org%22%3A%22CapSoftware%22%2C%22prId%22%3A3412%2C%22repo%22%3A%22Cap%22%7D", + "description": "Authorization required to deploy." } ] }, - "fetchedAt": "2026-10-26T18:30:46.321Z", + "fetchedAt": "2026-10-26T19:08:19.974Z", "filesTruncated": false, "commitsTruncated": false } diff --git a/src/app/api/verify/route.ts b/src/app/api/verify/route.ts new file mode 100644 index 0000000..9e827cd --- /dev/null +++ b/src/app/api/verify/route.ts @@ -0,0 +1,70 @@ +/** + * POST /api/verify { prUrl, fresh? } -> NDJSON stream of pipeline events + * (one JSON object per line). The final line is either a `done` event + * carrying the full Report or an `error` event. + */ +import { z } from "zod"; +import { parsePrUrl } from "@/core/github"; +import { runReceipts, type PipelineEvent } from "@/core/pipeline"; +import { clientKey, takeRun } from "@/lib/ratelimit"; + +// Long PRs with Kimi K3 reasoning can take ~45 s; Fluid compute allows 300 s. +export const maxDuration = 300; + +const RequestSchema = z.object({ + prUrl: z.string().trim().min(1).max(500), + fresh: z.boolean().optional(), +}); + +export async function POST(request: Request): Promise { + let body: z.infer; + try { + body = RequestSchema.parse(await request.json()); + } catch { + return Response.json({ error: "Send JSON like {\"prUrl\": \"https://github.com/o/r/pull/1\"}." }, { status: 400 }); + } + try { + parsePrUrl(body.prUrl); + } catch (error) { + return Response.json({ error: error instanceof Error ? error.message : "Not a GitHub PR URL." }, { status: 400 }); + } + + const limit = takeRun(clientKey(request.headers)); + if (!limit.ok) { + const minutes = Math.ceil(limit.retryAfterSec / 60); + return Response.json( + { error: `Rate limit reached (10 runs per hour). Try again in ${minutes} min, or self-host Receipts.` }, + { status: 429, headers: { "Retry-After": String(limit.retryAfterSec) } }, + ); + } + + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + async start(controller) { + let finished = false; + const send = (event: PipelineEvent): void => { + if (event.type === "done" || event.type === "error") finished = true; + controller.enqueue(encoder.encode(`${JSON.stringify(event)}\n`)); + }; + try { + await runReceipts(body.prUrl, { onEvent: send, fresh: body.fresh }); + } catch (error) { + // runReceipts emits its own error event, except for config errors + // thrown before the pipeline starts. + if (!finished) { + send({ type: "error", error: error instanceof Error ? error.message : String(error) }); + } + } finally { + controller.close(); + } + }, + }); + + return new Response(stream, { + headers: { + "Content-Type": "application/x-ndjson; charset=utf-8", + "Cache-Control": "no-store", + "X-Accel-Buffering": "no", + }, + }); +} diff --git a/src/app/globals.css b/src/app/globals.css index a2dc41e..70b51b8 100644 --- a/src/app/globals.css +++ b/src/app/globals.css @@ -1,26 +1,69 @@ @import "tailwindcss"; :root { - --background: #ffffff; - --foreground: #171717; + color-scheme: dark; + --background: #0c0c0d; + --foreground: #ececec; + --muted: #9a9a9f; + --line: #26262a; + --panel: #141416; + --paper: #f5f2ea; + --ink: #1d1b18; + --ink-muted: #6d675d; + --paper-line: #d9d3c5; + --ok: #15803d; + --bad: #c2261d; + --warn: #b45309; + --accent: #f2c14e; } @theme inline { --color-background: var(--background); --color-foreground: var(--foreground); + --color-muted: var(--muted); + --color-line: var(--line); + --color-panel: var(--panel); + --color-paper: var(--paper); + --color-ink: var(--ink); + --color-ink-muted: var(--ink-muted); + --color-paper-line: var(--paper-line); + --color-ok: var(--ok); + --color-bad: var(--bad); + --color-warn: var(--warn); + --color-accent: var(--accent); --font-sans: var(--font-geist-sans); --font-mono: var(--font-geist-mono); } -@media (prefers-color-scheme: dark) { - :root { - --background: #0a0a0a; - --foreground: #ededed; - } -} - body { background: var(--background); color: var(--foreground); - font-family: Arial, Helvetica, sans-serif; + font-family: var(--font-geist-sans), system-ui, sans-serif; +} + +/* Torn-paper bottom edge for the receipt card. */ +.receipt-edge { + --tooth: 10px; + mask: + conic-gradient(from -45deg at bottom, #0000, #000 1deg 89deg, #0000 90deg) 50% / calc(var(--tooth) * 2) 100%; + padding-bottom: calc(var(--tooth) + 1.5rem); +} + +@keyframes receipt-print { + from { + clip-path: inset(0 0 100% 0); + } + to { + clip-path: inset(0 0 0 0); + } +} + +.receipt-print { + animation: receipt-print 700ms ease-out both; +} + +@media (prefers-reduced-motion: reduce) { + .receipt-print { + animation: none; + } } diff --git a/src/app/layout.tsx b/src/app/layout.tsx index 9852c15..8892051 100644 --- a/src/app/layout.tsx +++ b/src/app/layout.tsx @@ -13,8 +13,9 @@ const geistMono = Geist_Mono({ }); export const metadata: Metadata = { - title: "Create Next App", - description: "Generated by create next app", + title: "Receipts", + description: + "Every agent claim, with a receipt. Receipts checks the claims in a GitHub pull request against the real diff and CI, using open models on Amazon Bedrock.", }; export default function RootLayout({ children }: LayoutProps<"/">) { diff --git a/src/app/page.tsx b/src/app/page.tsx index c887311..a325d5e 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -1,69 +1,64 @@ -import Image from "next/image"; +import { Suspense } from "react"; +import { ReceiptsApp } from "@/components/ReceiptsApp"; + +const REPO_URL = "https://github.com/CodeGovindz/receipts"; + +const STEPS = [ + { + title: "Extract", + body: "An open model reads the PR title, description and commits and pulls out every checkable claim: “added tests”, “all tests pass”, “no breaking changes”.", + }, + { + title: "Check", + body: "Kimi K3 checks each claim against the real diff, the changed test files and every CI check, which are first classified as test, lint, build or deploy without AI.", + }, + { + title: "Guard", + body: "Every verdict must cite a file, line or CI check that really exists in the PR. A verdict with made-up evidence becomes unverified, so a PR description can't talk its way to a pass.", + }, +]; export default function Home() { return ( -
-
- Next.js logo -
-

- To get started, edit the{" "} - - page.tsx - {" "} - file. -

-

- Looking for a starting point or more instructions? Head over to{" "} - - Templates - {" "} - or the{" "} - - Learning - {" "} - center. -

-
- -
-
+
+
+

Receipts

+

+ Every agent claim, with a receipt. +

+

+ Coding agents write confident pull requests. Paste one here and Receipts checks every claim + in it against the actual diff and CI: verified,{" "} + contradicted or unverified, each + with proof. +

+
+ +
+ + + +
+ +
+ {STEPS.map((step, index) => ( +
+

0{index + 1}

+

{step.title}

+

{step.body}

+
+ ))} +
+ + +
); } diff --git a/src/components/ProgressLog.tsx b/src/components/ProgressLog.tsx new file mode 100644 index 0000000..eeeb412 --- /dev/null +++ b/src/components/ProgressLog.tsx @@ -0,0 +1,74 @@ +"use client"; + +import { friendlyModelName } from "@/core/report"; +import type { PipelineEvent } from "@/core/pipeline"; + +function describe(event: PipelineEvent): string | null { + switch (event.type) { + case "fetching": + return "Fetching the PR, diff and CI checks from GitHub"; + case "fetched": + return `Fetched “${event.title}” · ${event.filesChanged} files · ${event.checkRuns} CI checks`; + case "extracting": + return "Extracting claims from the PR text"; + case "extracted": + if (event.error) return `Claim extraction failed: ${event.error}`; + return ( + `Found ${event.total} checkable claim${event.total === 1 ? "" : "s"}` + + (event.dropped > 0 ? ` (${event.dropped} dropped: quote not found in PR)` : "") + ); + case "verifying": + return `Checking ${event.index}/${event.total} · ${event.claim}`; + case "fallback": + return `${friendlyModelName(event.from)} → ${friendlyModelName(event.to)} for claim ${event.index} (${event.reason})`; + case "done": + return "Receipt printed"; + case "error": + return event.error; + } +} + +export function ProgressLog({ + events, + running, + elapsedMs, +}: { + events: PipelineEvent[]; + running: boolean; + elapsedMs: number; +}) { + const lines = events + .map((event) => ({ event, text: describe(event) })) + .filter((line): line is { event: PipelineEvent; text: string } => line.text !== null); + + return ( +
+
+ {running ? "Running" : "Log"} + {(elapsedMs / 1000).toFixed(1)}s +
+
    + {lines.map(({ event, text }, index) => { + const isLast = index === lines.length - 1; + const isError = event.type === "error"; + const marker = isError ? "!" : isLast && running ? "›" : "✓"; + return ( +
  1. + + {marker} + + + {text} + +
    2. + ); + })} +
+
+ ); +} diff --git a/src/components/ReceiptsApp.tsx b/src/components/ReceiptsApp.tsx new file mode 100644 index 0000000..aa7c951 --- /dev/null +++ b/src/components/ReceiptsApp.tsx @@ -0,0 +1,184 @@ +"use client"; + +import { useCallback, useEffect, useRef, useState } from "react"; +import { useSearchParams } from "next/navigation"; +import type { PipelineEvent } from "@/core/pipeline"; +import type { Report } from "@/core/types"; +import { ProgressLog } from "./ProgressLog"; +import { ReportView } from "./ReportView"; + +const EXAMPLES = [ + { + url: "https://github.com/CapSoftware/Cap/pull/3412", + label: "Cap #3412", + hint: "“all 41 tests pass”, but CI skipped the tests", + }, + { + url: "https://github.com/facebookincubator/dispenso/pull/129", + label: "dispenso #129", + hint: "long test plan, no new tests", + }, + { + url: "https://github.com/github/docs/pull/46488", + label: "github/docs #46488", + hint: "docs-only fix", + }, +]; + +type Status = "idle" | "running" | "done" | "error"; + +export function ReceiptsApp() { + const searchParams = useSearchParams(); + const [input, setInput] = useState(""); + const [status, setStatus] = useState("idle"); + const [events, setEvents] = useState([]); + const [report, setReport] = useState(null); + const [title, setTitle] = useState(null); + const [error, setError] = useState(null); + const [startedAt, setStartedAt] = useState(null); + const [now, setNow] = useState(0); + const autoRan = useRef(false); + + useEffect(() => { + if (status !== "running") return; + const timer = setInterval(() => setNow(Date.now()), 100); + return () => clearInterval(timer); + }, [status]); + + const run = useCallback(async (prUrl: string, fresh = false) => { + const url = prUrl.trim(); + if (url === "") return; + setInput(url); + setStatus("running"); + setEvents([]); + setReport(null); + setTitle(null); + setError(null); + setStartedAt(Date.now()); + setNow(Date.now()); + window.history.replaceState(null, "", `?pr=${encodeURIComponent(url)}`); + + const fail = (message: string) => { + setError(message); + setStatus("error"); + setNow(Date.now()); + }; + + let response: Response; + try { + response = await fetch("/api/verify", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ prUrl: url, fresh }), + }); + } catch { + fail("Could not reach the Receipts server."); + return; + } + if (!response.ok || !response.body) { + const body = (await response.json().catch(() => null)) as { error?: string } | null; + fail(body?.error ?? `Request failed (${response.status}).`); + return; + } + + const reader = response.body.pipeThrough(new TextDecoderStream()).getReader(); + let buffer = ""; + let finished = false; + for (;;) { + const { value, done } = await reader.read().catch(() => ({ value: undefined, done: true })); + if (done) break; + buffer += value; + let newline = buffer.indexOf("\n"); + while (newline >= 0) { + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + newline = buffer.indexOf("\n"); + if (line === "") continue; + const event = JSON.parse(line) as PipelineEvent; + setEvents((previous) => [...previous, event]); + if (event.type === "fetched") setTitle(event.title); + if (event.type === "done") { + finished = true; + setReport(event.report); + setStatus("done"); + setNow(Date.now()); + } + if (event.type === "error") { + finished = true; + fail(event.error); + } + } + } + if (!finished) fail("The connection closed before the receipt was printed (the server may have timed out)."); + }, []); + + useEffect(() => { + const pr = searchParams.get("pr"); + if (pr && !autoRan.current) { + autoRan.current = true; + void run(pr, searchParams.get("fresh") === "1"); + } + }, [searchParams, run]); + + const elapsedMs = startedAt === null ? 0 : now - startedAt; + + return ( +
+
{ + event.preventDefault(); + void run(input); + }} + > + + setInput(event.target.value)} + placeholder="https://github.com/owner/repo/pull/123" + className="min-w-0 flex-1 rounded-md border border-line bg-panel px-4 py-3 font-mono text-sm text-foreground placeholder:text-muted/60 focus:border-accent focus:outline-none" + /> + +
+ +
+ Try: + {EXAMPLES.map((example) => ( + + ))} +
+ + {events.length > 0 || status === "running" ? ( + + ) : null} + + {status === "error" && error && events.length === 0 && ( +

+ {error} +

+ )} + + {report && } +
+ ); +} diff --git a/src/components/ReportView.tsx b/src/components/ReportView.tsx new file mode 100644 index 0000000..b534e02 --- /dev/null +++ b/src/components/ReportView.tsx @@ -0,0 +1,217 @@ +"use client"; + +import { useState } from "react"; +import { modelsFooter, toMarkdown } from "@/core/report"; +import type { ClaimEvidence, ClaimResult, Grade, Report, Verdict } from "@/core/types"; + +const VERDICT_STYLE: Record = { + VERIFIED: { label: "Verified", mark: "✓", color: "var(--ok)" }, + CONTRADICTED: { label: "Contradicted", mark: "✕", color: "var(--bad)" }, + UNVERIFIED: { label: "Unverified", mark: "?", color: "var(--warn)" }, +}; + +const GRADE_COLOR: Record = { + PASS: "var(--ok)", + FAIL: "var(--bad)", + MIXED: "var(--warn)", +}; + +// Problems first: a reviewer should see contradictions before confirmations. +const VERDICT_ORDER: Record = { CONTRADICTED: 0, UNVERIFIED: 1, VERIFIED: 2 }; + +function prLabel(prUrl: string): string { + const match = prUrl.match(/github\.com\/([^/]+)\/([^/]+)\/pull\/(\d+)/); + return match ? `${match[1]}/${match[2]}#${match[3]}` : prUrl; +} + +function Divider() { + return
; +} + +function evidenceLabel(item: ClaimEvidence): string { + if (item.type === "file" && item.path) { + const file = item.path.split("/").pop() ?? item.path; + return item.line !== null ? `${file}:${item.line}` : file; + } + if (item.type === "check_run") return `CI: ${item.checkRunName ?? "check"}`; + return `signal: ${item.detail ?? "?"}`; +} + +function EvidenceChip({ item }: { item: ClaimEvidence }) { + const className = + "inline-flex max-w-full items-center truncate rounded border border-paper-line px-1.5 py-0.5 font-mono text-[11px] text-ink-muted"; + const title = item.path ?? item.checkRunName ?? item.detail ?? undefined; + if (item.url) { + return ( + + {evidenceLabel(item)} + + ); + } + return ( + + {evidenceLabel(item)} + + ); +} + +function ClaimRow({ result }: { result: ClaimResult }) { + const style = VERDICT_STYLE[result.verdict]; + return ( +
  • + + {style.mark} + +
    +

    + {style.label}: + {result.claim.text} +

    + {result.claim.quote !== "" && result.claim.quote !== result.claim.text && ( +

    + “{result.claim.quote}” · {result.claim.source.replace("_", " ")} +

    + )} +

    + + {style.label} + {" "} + — {result.reasoning} +

    + {result.evidence.length > 0 && ( +
    + {result.evidence.map((item, index) => ( + + ))} +
    + )} +
    +
    • + ); +} + +/** `text` is a function so browser-only values (window.location) are read on click, not during render. */ +function CopyButton({ text, label }: { text: () => string; label: string }) { + const [copied, setCopied] = useState(false); + return ( + + ); +} + +export function ReportView({ report, title }: { report: Report; title: string | null }) { + const counts = { VERIFIED: 0, CONTRADICTED: 0, UNVERIFIED: 0 }; + for (const result of report.claims) counts[result.verdict]++; + const sorted = [...report.claims].sort( + (a, b) => VERDICT_ORDER[a.verdict] - VERDICT_ORDER[b.verdict], + ); + const trust = report.trustScore === null ? "—" : `${Math.round(report.trustScore * 100)}%`; + + return ( +
    +
    +

    Receipts

    + + {prLabel(report.prUrl)} + + {title &&

    {title}

    } +
    + + + +
    +
    +

    Grade

    +

    + {report.grade} +

    +
    +
    +

    Trust

    +

    {trust}

    +
    +
    +

    + {counts.VERIFIED} verified · {counts.CONTRADICTED} contradicted · {counts.UNVERIFIED} unverified + {report.droppedClaims > 0 && ` · ${report.droppedClaims} dropped (quote not found in PR)`} +

    + + + + {sorted.length === 0 ? ( +

    No checkable claims found in the PR text.

    + ) : ( +
      + {sorted.map((result, index) => ( + + ))} +
    + )} + + {report.signals.length > 0 && ( + <> + +
    +

    Signals (no AI)

    +
      + {report.signals.map((signal) => ( +
    • + · + {signal} +
      • + ))} +
    +
    + + )} + + + +
    +

    {modelsFooter(report.modelsUsed)}

    +

    + {(report.timings.totalMs / 1000).toFixed(1)}s · head {report.headSha.slice(0, 7)} ·{" "} + {report.timings.llmCalls} model calls{report.cached ? " · from cache" : ""} +

    +
    + toMarkdown(report)} label="Copy as PR comment" /> + `${window.location.origin}/?pr=${encodeURIComponent(report.prUrl)}`} + label="Copy share link" + /> +
    +
    +
    + ); +} diff --git a/src/core/__tests__/guard.test.ts b/src/core/__tests__/guard.test.ts index 53f2d99..7933da9 100644 --- a/src/core/__tests__/guard.test.ts +++ b/src/core/__tests__/guard.test.ts @@ -658,6 +658,61 @@ describe("signal corner: removedExports vs signatureChanges (correction 4)", () }); }); +describe("CI-only contradictions", () => { + it("downgrades a contradiction whose only cited check failed for infrastructure reasons", () => { + const bundle = makeBundle({ + checkRuns: [ + { + name: "Format (Biome)", + status: "completed", + conclusion: "failure", + htmlUrl: "https://github.com/acme/widget/runs/504", + failureMessage: "Unexpected HTTP response: 504", + }, + ], + }); + const guarded = applyGuards( + claimOf({ text: "Biome checks pass", category: "other" }), + { + verdict: "CONTRADICTED", + confidence: 0.9, + reasoning: "Format (Biome) failed.", + evidence: [{ type: "check_run", checkRunName: "Format (Biome)" }], + }, + bundle, + computeSignals(bundle), + ); + expect(guarded.verdict).toBe("UNVERIFIED"); + expect(guarded.reasoning).toContain("None of the cited CI checks actually failed"); + }); + + it("keeps a contradiction backed by a real failure", () => { + const bundle = makeBundle({ + checkRuns: [ + { + name: "Format (Biome)", + status: "completed", + conclusion: "failure", + htmlUrl: "https://github.com/acme/widget/runs/1", + failureMessage: "File src/app.ts is not formatted.", + }, + ], + }); + const guarded = applyGuards( + claimOf({ text: "Biome checks pass", category: "other" }), + { + verdict: "CONTRADICTED", + confidence: 0.9, + reasoning: "Format (Biome) failed on src/app.ts.", + evidence: [{ type: "check_run", checkRunName: "Format (Biome)" }], + }, + bundle, + computeSignals(bundle), + ); + expect(guarded.verdict).toBe("CONTRADICTED"); + }); +}); + // Keep the ClaimResult import honest: the guard's output feeds claim results. describe("guard output shape feeds ClaimResult", () => { it("produces fields a ClaimResult can absorb", () => { diff --git a/src/core/__tests__/signals.test.ts b/src/core/__tests__/signals.test.ts index e38145e..c2a4500 100644 --- a/src/core/__tests__/signals.test.ts +++ b/src/core/__tests__/signals.test.ts @@ -4,13 +4,15 @@ import { ciKindOutcome, classifyCheckName, computeCiSignal, + computeSignals, effectiveCheckEffect, + isInfraFailure, } from "../signals"; import type { PrBundle } from "../types"; function bundleWith( - checkRuns: Array<{ name: string; conclusion: string | null; status?: string }>, - statuses: Array<{ context: string; state: string }> = [], + checkRuns: Array<{ name: string; conclusion: string | null; status?: string; failureMessage?: string }>, + statuses: Array<{ context: string; state: string; targetUrl?: string; description?: string }> = [], ): PrBundle { return { prUrl: "https://github.com/acme/widget/pull/1", @@ -28,6 +30,7 @@ function bundleWith( status: run.status ?? "completed", conclusion: run.conclusion, htmlUrl: "https://example.com/ci/1", + failureMessage: run.failureMessage ?? null, })), combinedStatus: statuses.length > 0 @@ -36,7 +39,8 @@ function bundleWith( statuses: statuses.map((status) => ({ context: status.context, state: status.state, - targetUrl: null, + targetUrl: status.targetUrl ?? null, + description: status.description ?? null, })), } : null, @@ -78,6 +82,7 @@ describe("effectiveCheckEffect (Bug 3: skipped/neutral/stale are ignorable)", () ["completed", "skipped", "ignored"], ["completed", "neutral", "ignored"], ["completed", "stale", "ignored"], + ["completed", "cancelled", "ignored"], // usually superseded by a newer push ["queued", null, "pending"], ["in_progress", null, "pending"], ["completed", null, "ignored"], @@ -191,3 +196,55 @@ describe("computeCiSignal", () => { expect(ciLine ?? "").toContain("Vercel – cap-web [deploy]"); }); }); + +describe("infrastructure failures are not code failures", () => { + it.each([ + ["Unexpected HTTP response: 504", true], + ["The runner has received a shutdown signal.", true], + ["Authorization required to deploy.", true], + ["Process completed with exit code 1.", false], + ["2 tests failed in login.test.ts", false], + ])("isInfraFailure(%s) -> %s", (message, expected) => { + expect(isInfraFailure(message)).toBe(expected); + }); + + it("ignores a check that failed on an HTTP 504 (the real Cap#3412 'Format (Biome)' run)", () => { + const ci = computeCiSignal( + bundleWith([ + { name: "Format (Biome)", conclusion: "failure", failureMessage: "Unexpected HTTP response: 504" }, + { name: "Lint (Biome)", conclusion: "success" }, + ]), + ); + const format = ci.checkRuns.find((run) => run.name === "Format (Biome)"); + expect(format?.effective).toBe("ignored"); + expect(format?.infraFailure).toBe(true); + expect(ci.outcome).toBe("success"); + const summary = buildSignalsSummary( + computeSignals( + bundleWith([ + { name: "Format (Biome)", conclusion: "failure", failureMessage: "Unexpected HTTP response: 504" }, + ]), + ), + ); + expect(summary.join("\n")).toContain("infrastructure reasons"); + }); + + it("keeps a real failure as a failure", () => { + const ci = computeCiSignal( + bundleWith([{ name: "Unit tests", conclusion: "failure", failureMessage: "Process completed with exit code 1." }]), + ); + expect(ci.checkRuns[0].effective).toBe("failure"); + expect(ci.outcome).toBe("failure"); + }); + + it("ignores a Vercel status that failed on GitHub authorization", () => { + const ci = computeCiSignal( + bundleWith( + [{ name: "Unit tests", conclusion: "success" }], + [{ context: "Vercel – cap-web", state: "failure", targetUrl: "https://vercel.com/git/authorize?team=x" }], + ), + ); + expect(ci.checkRuns.find((run) => run.name === "Vercel – cap-web")?.effective).toBe("ignored"); + expect(ci.outcome).toBe("success"); + }); +}); diff --git a/src/core/env.ts b/src/core/env.ts index 1cf61a7..8d0b5d9 100644 --- a/src/core/env.ts +++ b/src/core/env.ts @@ -24,8 +24,10 @@ export class ConfigError extends Error { * chain. */ export function loadDotEnv(path = ".env"): void { - if (!fs.existsSync(path)) return; - const lines = fs.readFileSync(path, "utf8").split(/\r?\n/); + // turbopackIgnore: Next.js loads .env itself; without these hints the + // bundler traces the whole project into the serverless function. + if (!fs.existsSync(/*turbopackIgnore: true*/ path)) return; + const lines = fs.readFileSync(/*turbopackIgnore: true*/ path, "utf8").split(/\r?\n/); for (const line of lines) { const trimmed = line.trim(); if (trimmed === "" || trimmed.startsWith("#")) continue; diff --git a/src/core/github.ts b/src/core/github.ts index 7440ae0..e0aae93 100644 --- a/src/core/github.ts +++ b/src/core/github.ts @@ -20,6 +20,8 @@ export interface PrLocator { /** Hard caps so one huge PR can't blow up the run (docs/PLAN.md §16). */ export const MAX_FILES = 100; export const MAX_COMMITS = 50; +/** Failed check runs whose annotations are fetched to learn why they failed. */ +export const MAX_FAILURE_LOOKUPS = 6; /** Error with a user-facing message; `status` carries the GitHub status when known. */ export class GitHubError extends Error { @@ -152,6 +154,28 @@ export function sanitizePrBody(body: string | null): string | null { return text.trim(); } +/** First failure-level annotation of a check run (max 300 chars); null when absent or unreadable. */ +async function firstFailureAnnotation( + octokit: Octokit, + owner: string, + repo: string, + checkRunId: number, +): Promise { + try { + const response = await octokit.rest.checks.listAnnotations({ + owner, + repo, + check_run_id: checkRunId, + per_page: 20, + }); + const failure = response.data.find((annotation) => annotation.annotation_level === "failure"); + const message = failure?.message?.trim(); + return message ? message.slice(0, 300) : null; + } catch { + return null; // a missing reason must never fail the whole run + } +} + function createOctokit(githubToken: string | undefined): Octokit { return new Octokit({ ...(githubToken ? { auth: githubToken } : {}), @@ -247,11 +271,23 @@ export async function fetchPrBundle( ref: headSha, per_page: 100, }); - checkRuns = response.data.check_runs.map((run: RawCheckRun) => ({ + const runs = response.data.check_runs; + // Why did a run fail? Its first failure annotation tells real failures + // ("2 tests failed") apart from infrastructure ones ("HTTP 504"). + const failed = runs + .filter((run) => run.conclusion === "failure" || run.conclusion === "timed_out") + .slice(0, MAX_FAILURE_LOOKUPS); + const messages = new Map( + await Promise.all( + failed.map(async (run) => [run.id, await firstFailureAnnotation(octokit, owner, repo, run.id)] as const), + ), + ); + checkRuns = runs.map((run: RawCheckRun) => ({ name: run.name ?? "(unnamed check)", status: run.status ?? "completed", conclusion: run.conclusion ?? null, htmlUrl: run.html_url ?? prUrl, + failureMessage: messages.get(run.id) ?? null, })); } catch (error) { const status = isOctokitError(error) && typeof error.status === "number" ? error.status : undefined; @@ -276,6 +312,7 @@ export async function fetchPrBundle( context: status.context ?? "(unknown context)", state: status.state ?? "pending", targetUrl: status.target_url ?? null, + description: status.description ?? null, })), }; } catch (error) { diff --git a/src/core/guard.ts b/src/core/guard.ts index c31f5c0..a2b870b 100644 --- a/src/core/guard.ts +++ b/src/core/guard.ts @@ -150,6 +150,21 @@ export function applyGuards( reasoning = "Model gave no checkable evidence."; } + // A contradiction that rests only on CI checks must cite a check that + // really failed: skipped, cancelled and infrastructure-failed checks + // (effective "ignored") say nothing about the code. + if ( + verdict === "CONTRADICTED" && + kept.every((item) => item.type === "check_run") && + !kept.some( + (item) => + signals.ci.checkRuns.find((run) => run.name === item.checkRunName)?.effective === "failure", + ) + ) { + verdict = "UNVERIFIED"; + reasoning = `None of the cited CI checks actually failed (they were skipped, cancelled or broke for infrastructure reasons). ${reasoning}`; + } + // Bug 2 (deterministic backstop) — a tests_pass verdict must match the CI // evidence by kind, regardless of what the model decided this run: // VERIFIED only while every kind the claim talks about succeeded; diff --git a/src/core/prompts.ts b/src/core/prompts.ts index 33bc0cd..45eac48 100644 --- a/src/core/prompts.ts +++ b/src/core/prompts.ts @@ -23,14 +23,14 @@ const EXTRACTION_RULES = [ "Each claim `text` must be a complete, self-contained sentence understandable without the PR — never a fragment. Expand shorthand: not \"five fail before the fix.\" but \"Five of the six new tests fail without the fix.\"", "`quote` must be an exact verbatim substring of the source sentence that supports the claim; `source` is one of `title | pr_body | commit`; `category` is one of `tests_added | tests_pass | bug_fixed | feature_added | no_breaking_change | refactor_only | docs_updated | dependency_change | performance | security | other`.", "`scope` is one of `ci` (a statement about CI results or the PR's observable state), `local` (a statement about runs performed on the author's machine, e.g. \"all tests pass locally\", \"I ran ctest\"), `unspecified` (everything else).", - "Ignore plans, intentions, TODOs, questions, quoted text, PR-template boilerplate, and unfilled checkbox sections — they make no claim about this PR. A filled-in answer inside a template section IS a claim if it states a checkable fact.", + "Ignore plans, intentions, TODOs, questions, quoted text, PR-template boilerplate, and unticked checkboxes — they make no claim about this PR. A filled-in answer inside a template section, or a TICKED checkbox, IS a claim if it states a checkable fact (\"[x] I have added tests\").", "Dedupe near-identical claims. Maximum 12 claims, most important first.", ].join("\n"); const EXTRACTION_EXAMPLES = [ "Examples:", - "1. The body says: \"Under load, the SDK stays in its recording phase and keeps the camera active. This PR fixes that.\" The first sentence is the pre-existing bug — do NOT extract it. Emit ONE claim: {\"text\":\"After this PR, an encoder error no longer leaves the SDK stuck in its recording phase\",\"quote\":\"This PR fixes that.\",\"source\":\"pr_body\",\"category\":\"bug_fixed\",\"scope\":\"unspecified\"}.", - "2. The body says: \"## Checklist\\n- [x] I have read the CONTRIBUTING document\\n- [ ] I have added tests\" — both are template boilerplate, extract nothing. But a filled section \"## Testing done\\nAll 141 tests pass locally.\" IS a claim: {\"text\":\"All 141 tests pass locally\",\"quote\":\"All 141 tests pass locally.\",\"source\":\"pr_body\",\"category\":\"tests_pass\",\"scope\":\"local\"}.", + "1. The body says: \"Under load, the SDK stays in its recording phase and keeps the camera active. This PR fixes that.\" The first sentence is the pre-existing bug — do NOT extract it. Emit ONE claim: {\"text\":\"Under load, the SDK no longer gets stuck in its recording phase with the camera active\",\"quote\":\"This PR fixes that.\",\"source\":\"pr_body\",\"category\":\"bug_fixed\",\"scope\":\"unspecified\"}. Write claim text as a plain statement of fact; never start it with \"After this PR\".", + "2. The body says: \"## Checklist\\n- [x] I have read the CONTRIBUTING document\\n- [ ] I have updated the docs\\n- [x] I have added tests that prove my fix is effective\". Unticked boxes claim nothing, and \"read the CONTRIBUTING document\" can't be checked, so skip both. The TICKED, checkable box IS a claim: {\"text\":\"The PR adds tests that prove the fix works\",\"quote\":\"I have added tests that prove my fix is effective\",\"source\":\"pr_body\",\"category\":\"tests_added\",\"scope\":\"unspecified\"}. A filled section \"## Testing done\\nAll 141 tests pass locally.\" is also a claim: {\"text\":\"All 141 tests pass on the author's machine\",\"quote\":\"All 141 tests pass locally.\",\"source\":\"pr_body\",\"category\":\"tests_pass\",\"scope\":\"local\"}.", ].join("\n"); /** @@ -86,6 +86,11 @@ const CATEGORY_RULES = "if removedExports shows a public API removal. `bug_fixed` / `feature_added`: VERIFIED " + "requires a cited hunk that plausibly implements the fix/feature; the reasoning must " + "state whether a test covers it.\n" + + "CI rules: only a check with `effective: \"failure\"` counts as failed. A check with " + + "`effective: \"ignored\"` produced no result (skipped, cancelled, or it failed for " + + "infrastructure reasons — see its failureMessage, e.g. an HTTP 504 during setup) and can " + + "never support or contradict a claim. A claim that a check passed locally (\"scoped Biome " + + "checks pass\") is not contradicted by a CI check that ran something broader.\n" + "Scope rules: claims with `scope: \"local\"` describe runs on the author's machine — " + "they cannot be confirmed by CI, so they are UNVERIFIED unless the diff itself proves " + "the claim. If a claim describes how the code behaved BEFORE this PR (rather than the " + @@ -93,8 +98,11 @@ const CATEGORY_RULES = const RESPONSE_SHAPE = 'Return `{"verdict":"VERIFIED|CONTRADICTED|UNVERIFIED","confidence":0-1,' + - '"reasoning":"≤2 sentences","evidence":[{"type":"file|check_run|signal","path":"...",' + - '"line":123,"checkRunName":"...","detail":"..."}]}`.'; + '"reasoning":"1-2 short sentences","evidence":[{"type":"file|check_run|signal","path":"...",' + + '"line":123,"checkRunName":"...","detail":"..."}]}`.\n' + + "The reasoning is read by a human code reviewer: at most 2 short sentences (under 50 words) " + + "in plain English. Name files and CI checks, but never mention internal field names such " + + "as effective, kind, scope, category or signal keys."; /** The signal keys relevant to each claim category (always includes CI state). */ function relevantSignalKeys(category: Claim["category"]): SignalKey[] { @@ -139,6 +147,7 @@ function ciForPrompt(ci: Signals["ci"]): string { status: run.status, conclusion: run.conclusion, effective: run.effective, + ...(run.failureMessage ? { failureMessage: run.failureMessage } : {}), url: run.htmlUrl, })), ); diff --git a/src/core/report.ts b/src/core/report.ts index 2bf392d..691a43e 100644 --- a/src/core/report.ts +++ b/src/core/report.ts @@ -155,10 +155,11 @@ export function toMarkdown(report: Report): string { lines.push("| Verdict | Claim | Reasoning | Evidence |"); lines.push("|---|---|---|---|"); for (const result of report.claims) { - const claimText = result.claim.quote !== "" ? result.claim.quote : result.claim.text; + // The self-contained claim text, not the raw quote: quotes can be + // fragments ("five fail before the fix.") that mean nothing out of context. lines.push( `| ${VERDICT_ICON[result.verdict]} ${result.verdict} | ` + - `${tableCell(claimText, 160)} | ${tableCell(result.reasoning)} | ` + + `${tableCell(result.claim.text, 160)} | ${tableCell(result.reasoning)} | ` + `${evidenceMarkdown(result.evidence) || "—"} |`, ); } diff --git a/src/core/signals.ts b/src/core/signals.ts index 17f3ed1..20734ec 100644 --- a/src/core/signals.ts +++ b/src/core/signals.ts @@ -87,15 +87,33 @@ export function claimCheckKindHints(text: string): CheckKind[] { export type CheckEffect = "success" | "failure" | "pending" | "ignored"; /** Conclusion/status combinations that neither pass nor fail (Bug 3 treatment). */ -const IGNORABLE_CONCLUSIONS = new Set(["skipped", "neutral", "stale"]); -const FAILURE_CONCLUSIONS = new Set(["failure", "timed_out", "cancelled", "startup_failure", "action_required"]); +// "cancelled" is ignorable: runs are usually cancelled because a newer push +// superseded them, which says nothing about the code. +const IGNORABLE_CONCLUSIONS = new Set(["skipped", "neutral", "stale", "cancelled"]); +const FAILURE_CONCLUSIONS = new Set(["failure", "timed_out", "startup_failure", "action_required"]); const PENDING_STATUSES = new Set(["queued", "in_progress"]); -export function effectiveCheckEffect(status: string, conclusion: string | null): CheckEffect { +/** + * Failure messages that mean the CI machinery broke, not the code: a check + * that died on an HTTP 504 while setting up never ran its tool, so it can + * neither support nor contradict a claim. + */ +const INFRA_FAILURE = + /unexpected http response: 5\d\d|\b(502|503|504) (bad gateway|service unavailable|gateway time-?out)|internal server error|runner has received a shutdown signal|lost communication with the server|ECONNRESET|ETIMEDOUT|no space left on device|authorization required|exceeded .*(minutes|quota|spending limit)|billing|rate limit/i; + +export function isInfraFailure(message: string | null | undefined): boolean { + return typeof message === "string" && INFRA_FAILURE.test(message); +} + +export function effectiveCheckEffect( + status: string, + conclusion: string | null, + failureMessage: string | null = null, +): CheckEffect { if (conclusion !== null) { if (IGNORABLE_CONCLUSIONS.has(conclusion)) return "ignored"; if (conclusion === "success") return "success"; - if (FAILURE_CONCLUSIONS.has(conclusion)) return "failure"; + if (FAILURE_CONCLUSIONS.has(conclusion)) return isInfraFailure(failureMessage) ? "ignored" : "failure"; } if (PENDING_STATUSES.has(status)) return "pending"; if (conclusion === null && (status === "completed" || status === "")) return "ignored"; @@ -111,6 +129,10 @@ export interface CiObservation { htmlUrl: string; kind: CheckKind; effective: CheckEffect; + /** Why it failed (first failure annotation or status description), when known. */ + failureMessage: string | null; + /** Failed for infrastructure reasons, so treated as "ignored". */ + infraFailure: boolean; } export interface CiSignal { @@ -459,15 +481,27 @@ function computeDependencyChanges(bundle: PrBundle): DependencyChange[] { * covers both "no CI at all" and "nothing actually executed". */ export function computeCiSignal(bundle: PrBundle): CiSignal { - const observations: CiObservation[] = bundle.checkRuns.map((run) => ({ - name: run.name, - status: run.status, - conclusion: run.conclusion ?? null, - htmlUrl: run.htmlUrl, - kind: classifyCheckName(run.name), - effective: effectiveCheckEffect(run.status, run.conclusion ?? null), - })); + const observations: CiObservation[] = bundle.checkRuns.map((run) => { + const failureMessage = run.failureMessage ?? null; + const failed = run.conclusion !== null && FAILURE_CONCLUSIONS.has(run.conclusion ?? ""); + return { + name: run.name, + status: run.status, + conclusion: run.conclusion ?? null, + htmlUrl: run.htmlUrl, + kind: classifyCheckName(run.name), + effective: effectiveCheckEffect(run.status, run.conclusion ?? null, failureMessage), + failureMessage, + infraFailure: failed && isInfraFailure(failureMessage), + }; + }); for (const status of bundle.combinedStatus?.statuses ?? []) { + const description = status.description ?? null; + const failed = status.state === "failure" || status.state === "error"; + // Vercel reports failed GitHub authorization as a failed status whose + // target URL is the authorize page. + const infraFailure = + failed && (isInfraFailure(description) || /\/git\/authorize\b/.test(status.targetUrl ?? "")); observations.push({ name: status.context, // Legacy statuses have no separate status/conclusion pair; classify @@ -476,7 +510,9 @@ export function computeCiSignal(bundle: PrBundle): CiSignal { conclusion: status.state, htmlUrl: status.targetUrl ?? bundle.prUrl, kind: classifyCheckName(status.context), - effective: legacyStateEffect(status.state), + effective: infraFailure ? "ignored" : legacyStateEffect(status.state), + failureMessage: failed ? description : null, + infraFailure, }); } @@ -599,6 +635,14 @@ export function buildSignalsSummary(signals: Signals): string[] { break; } } + const infra = ci.checkRuns.filter((run) => run.infraFailure); + if (infra.length > 0) { + const listed = infra + .slice(0, MAX_LIST) + .map((run) => (run.failureMessage ? `${run.name} (“${run.failureMessage}”)` : run.name)) + .join(", "); + lines.push(`Ignored ${infra.length} check(s) that failed for infrastructure reasons, not code: ${listed}`); + } const testKindOutcome = ciKindOutcome(ci, "test"); if (testKindOutcome === "none" && !ci.none) { lines.push("No CI check runs tests for head SHA"); diff --git a/src/core/types.ts b/src/core/types.ts index e4494cc..87f175c 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -85,6 +85,8 @@ export const CheckRunSchema = z.object({ conclusion: z.string().nullable(), /** Link to the check run detail page (evidence URL). */ htmlUrl: z.string(), + /** First failure annotation of a failed run, e.g. "Unexpected HTTP response: 504". */ + failureMessage: z.string().nullable().optional(), }); export type CheckRun = z.infer; @@ -92,6 +94,8 @@ export const StatusEntrySchema = z.object({ context: z.string(), state: z.string(), targetUrl: z.string().nullable(), + /** Status description, e.g. Vercel's "Authorization required to deploy." */ + description: z.string().nullable().optional(), }); export type StatusEntry = z.infer; diff --git a/src/lib/__tests__/ratelimit.test.ts b/src/lib/__tests__/ratelimit.test.ts new file mode 100644 index 0000000..4a454ab --- /dev/null +++ b/src/lib/__tests__/ratelimit.test.ts @@ -0,0 +1,46 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { + clearRateLimit, + clientKey, + RATE_LIMIT_RUNS, + RATE_LIMIT_WINDOW_MS, + takeRun, +} from "../ratelimit"; + +describe("takeRun", () => { + beforeEach(() => clearRateLimit()); + + it("allows RATE_LIMIT_RUNS runs per window, then refuses with a retry time", () => { + const start = 1_000_000; + for (let i = 0; i < RATE_LIMIT_RUNS; i++) { + expect(takeRun("1.2.3.4", start + i).ok).toBe(true); + } + // The oldest run (at `start`) leaves the window 3600 s after it happened. + const refused = takeRun("1.2.3.4", start + RATE_LIMIT_RUNS); + expect(refused).toEqual({ ok: false, retryAfterSec: 3600 }); + }); + + it("frees a slot once the oldest run leaves the window", () => { + const start = 1_000_000; + for (let i = 0; i < RATE_LIMIT_RUNS; i++) takeRun("ip", start); + expect(takeRun("ip", start + RATE_LIMIT_WINDOW_MS - 1).ok).toBe(false); + expect(takeRun("ip", start + RATE_LIMIT_WINDOW_MS).ok).toBe(true); + }); + + it("counts each client separately", () => { + for (let i = 0; i < RATE_LIMIT_RUNS; i++) takeRun("a", 0); + expect(takeRun("a", 1).ok).toBe(false); + expect(takeRun("b", 1).ok).toBe(true); + }); +}); + +describe("clientKey", () => { + it("uses the first x-forwarded-for address", () => { + expect(clientKey(new Headers({ "x-forwarded-for": "9.9.9.9, 10.0.0.1" }))).toBe("9.9.9.9"); + }); + + it("falls back to x-real-ip, then 'unknown'", () => { + expect(clientKey(new Headers({ "x-real-ip": "8.8.8.8" }))).toBe("8.8.8.8"); + expect(clientKey(new Headers())).toBe("unknown"); + }); +}); diff --git a/src/lib/ratelimit.ts b/src/lib/ratelimit.ts new file mode 100644 index 0000000..300af9e --- /dev/null +++ b/src/lib/ratelimit.ts @@ -0,0 +1,40 @@ +/** + * Best-effort per-IP rate limit for the public web app (plan §10): a sliding + * one-hour window kept in memory. On Vercel each function instance has its + * own memory, so this bounds the AWS bill per instance, not globally — the + * README documents the limitation (Upstash is the post-hackathon upgrade). + */ + +export const RATE_LIMIT_RUNS = 10; +export const RATE_LIMIT_WINDOW_MS = 60 * 60 * 1000; + +export type RateLimitResult = + | { ok: true; remaining: number } + | { ok: false; retryAfterSec: number }; + +const hits = new Map(); + +/** Records one run for `key` if it is under the limit. */ +export function takeRun(key: string, now: number = Date.now()): RateLimitResult { + const recent = (hits.get(key) ?? []).filter((at) => now - at < RATE_LIMIT_WINDOW_MS); + if (recent.length >= RATE_LIMIT_RUNS) { + hits.set(key, recent); + const retryAfterMs = recent[0] + RATE_LIMIT_WINDOW_MS - now; + return { ok: false, retryAfterSec: Math.max(1, Math.ceil(retryAfterMs / 1000)) }; + } + recent.push(now); + hits.set(key, recent); + return { ok: true, remaining: RATE_LIMIT_RUNS - recent.length }; +} + +/** Client IP from the proxy headers Vercel sets; "unknown" when absent. */ +export function clientKey(headers: Headers): string { + const forwarded = headers.get("x-forwarded-for"); + if (forwarded) return forwarded.split(",")[0].trim(); + return headers.get("x-real-ip")?.trim() || "unknown"; +} + +/** Test support: forgets every recorded run. */ +export function clearRateLimit(): void { + hits.clear(); +}