From 141e23c58f3d12ef4445b0f6ea1cc2112ba780fb Mon Sep 17 00:00:00 2001 From: Hyo Date: Wed, 18 Nov 2026 22:23:50 +0900 Subject: [PATCH 1/2] feat: check a refutation again before dropping a finding --- convex/pullRequests/workflow.ts | 134 ++++++++++++++++-------- e2e/convex/prRecall.ts | 8 +- test/convex/pullRequests/review.test.ts | 24 ++++- 3 files changed, 119 insertions(+), 47 deletions(-) diff --git a/convex/pullRequests/workflow.ts b/convex/pullRequests/workflow.ts index 41be0cd..3859fee 100644 --- a/convex/pullRequests/workflow.ts +++ b/convex/pullRequests/workflow.ts @@ -780,24 +780,23 @@ export async function reviewWithContext(args: { ); // A second model pass challenges candidates; deterministic quote/line checks // remain mandatory. This is self-critique, not an independent review. - const verify = async ( - findings: Finding[], + type VerificationCandidate = { + id: string; + finding: Finding; + refutation?: string; + }; + // One verification request: which candidates hold, and which are refuted by + // a quote of the shown source. + const askVerification = async ( + instruction: string, + candidates: VerificationCandidate[], files: ChangedFile[], sources: ReadEvidence[], billingLabel: string, - ): Promise => { - if (findings.length === 0) return findings; - await args.guard(); - const candidates = findings.map((finding, index) => ({ - id: `F${index + 1}`, - finding, - })); + ) => { const critique = await chat( [ - { - role: "system", - content: `${REVIEW_PROMPT}\nThis is the verification pass. Check each supplied candidate against the source. Return retainedFindingIds for the candidates that hold. A candidate may be dropped only when the source shows it is incorrect, pre-existing or already handled: for each such candidate add a refutation with its id and an exact quote of 12-1500 characters from the SOURCE blocks that shows it. A candidate without a quoted refutation is kept; uncertainty alone is not a reason to drop it. Do not rewrite findings, change anchors or confidence, add suggestions, or introduce new findings.`, - }, + { role: "system", content: `${REVIEW_PROMPT}\n${instruction}` }, reviewContextMessage( { candidates, @@ -884,42 +883,87 @@ export async function reviewWithContext(args: { ) ); }; - const refuted = new Set( - ("refutations" in selection && Array.isArray(selection.refutations) - ? selection.refutations - : [] + const refuted = new Map(); + for (const r of "refutations" in selection && + Array.isArray(selection.refutations) + ? selection.refutations + : []) + if ( + r !== null && + typeof r === "object" && + "id" in r && + typeof r.id === "string" && + "quote" in r && + typeof r.quote === "string" && + r.quote.trim().length >= 12 && + r.quote.length <= 1500 && + !refuted.has(r.id) && + refutes(r.id, r.quote) ) - .filter( - (r: unknown): r is { id: string; quote: string } => - r !== null && - typeof r === "object" && - "id" in r && - typeof r.id === "string" && - "quote" in r && - typeof r.quote === "string" && - r.quote.trim().length >= 12 && - r.quote.length <= 1500 && - refutes(r.id, r.quote), - ) - .map((r: { id: string }) => r.id), + refuted.set(r.id, r.quote); + const retained = new Set(selection.retainedFindingIds as string[]); + return candidates + .filter((c) => !retained.has(c.id) && refuted.has(c.id)) + .map((c) => ({ ...c, refutation: refuted.get(c.id) })); + }; + // A second model pass challenges candidates; deterministic quote/line checks + // remain mandatory. This is self-critique, not an independent review. + const verify = async ( + findings: Finding[], + files: ChangedFile[], + sources: ReadEvidence[], + billingLabel: string, + ): Promise => { + if (findings.length === 0) return findings; + await args.guard(); + const candidates = findings.map((finding, index) => ({ + id: `F${index + 1}`, + finding, + })); + let dropped = await askVerification( + "This is the verification pass. Check each supplied candidate against the source. Return retainedFindingIds for the candidates that hold. A candidate may be dropped only when the source shows it is incorrect, pre-existing or already handled: for each such candidate add a refutation with its id and an exact quote of 12-1500 characters from the SOURCE blocks that shows it. A candidate without a quoted refutation is kept; uncertainty alone is not a reason to drop it. Do not rewrite findings, change anchors or confidence, add suggestions, or introduce new findings.", + candidates, + files, + sources, + billingLabel, ); + const trace = (step: string, checked: VerificationCandidate[]) => + args.trace?.({ + stage: "verified", + step, + kept: checked + .filter((c) => !dropped.some((d) => d.id === c.id)) + .map((c) => c.finding.title), + discarded: dropped.map((c) => c.finding.title), + }); + trace(billingLabel, candidates); + // A refuted candidate gets a second check of its refutation, so one + // misread of the source cannot drop a real defect. It is dropped only when + // that check refutes it again; if the check cannot run, the first + // decision stands. + if ( + dropped.length > 0 && + Date.now() - actionStarted <= COVERAGE_REQUEST_CUTOFF_MS + ) { + const appealed = dropped; + try { + await args.guard(); + dropped = await askVerification( + "This is a second check of refutations. Each supplied candidate was dropped by an earlier check, which quoted the source in its refutation field. Re-check each refutation independently against the source: it holds only if the quoted code, read with the code around it, shows the candidate is incorrect, pre-existing or already handled on every path the candidate describes. A refutation that misreads the code, covers only some of those paths, or quotes code the defect does not depend on does not hold. Return retainedFindingIds for the candidates whose refutation does not hold, and a refutation with an exact quote of 12-1500 characters from the SOURCE blocks for each candidate that is still refuted. Do not rewrite findings or introduce new ones.", + appealed, + files, + sources, + `${billingLabel}/appeal`, + ); + trace(`${billingLabel}/appeal`, appealed); + } catch (error) { + if (isLifecycleChange(error)) throw error; + dropped = appealed; + } + } const retainedFindingIds = candidates - .filter( - (c) => - (selection.retainedFindingIds as unknown[]).includes(c.id) || - !refuted.has(c.id), - ) + .filter((c) => !dropped.some((d) => d.id === c.id)) .map((c) => c.id); - args.trace?.({ - stage: "verified", - step: billingLabel, - kept: candidates - .filter((c) => retainedFindingIds.includes(c.id)) - .map((c) => c.finding.title), - discarded: candidates - .filter((c) => !retainedFindingIds.includes(c.id)) - .map((c) => c.finding.title), - }); // Selection cannot mutate a previously validated candidate. Validate its // original evidence again rather than trusting generated replacement text. return validateFindings( diff --git a/e2e/convex/prRecall.ts b/e2e/convex/prRecall.ts index b080ba0..00c9b87 100644 --- a/e2e/convex/prRecall.ts +++ b/e2e/convex/prRecall.ts @@ -8,7 +8,10 @@ import { createMeteredProvider } from "../../convex/_ai/chatBilling"; import { getDefaultAiModel } from "../../convex/_ai/model"; import { CreditFallbackProvider } from "../../convex/_ai/providers"; import { reviewWithContext } from "../../convex/pullRequests/workflow"; -import { ReviewContextLimitError } from "../../convex/pullRequests/errors"; +import { + ReviewContextLimitError, + reviewFailureDetails, +} from "../../convex/pullRequests/errors"; import { hasReviewablePatch, windowChangedSource, @@ -160,6 +163,9 @@ export const replayReview = internalAction({ ) : readPublic(args.repository, path, args.baseSha), guard: async () => guard(), + }).catch((error) => { + // The same sanitized details a production review records. + throw new Error(reviewFailureDetails(error, "Replay")); }); return { seconds: Math.round((Date.now() - started) / 1000), diff --git a/test/convex/pullRequests/review.test.ts b/test/convex/pullRequests/review.test.ts index 53cadae..048e650 100644 --- a/test/convex/pullRequests/review.test.ts +++ b/test/convex/pullRequests/review.test.ts @@ -1509,13 +1509,35 @@ describe("bounded review workflow", () => { ).rejects.toThrow("did not inspect"); }); it("does not publish speculative candidates removed by the critique pass", async () => { + const refuted = verified([], [{ id: "F1", quote: "function find(key) {" }]); + // The refutation is checked again, and holds. + const p = provider([use, stop([finding]), refuted, refuted]); + expect((await run(p)).findings).toEqual([]); + expect(p.chat).toHaveBeenCalledTimes(4); + const appeal = vi.mocked(p.chat).mock.calls[3]; + expect(appeal[1]?.billingLabel).toBe("pr-review/critique/appeal"); + expect( + JSON.stringify(appeal[0][1].content).includes("function find(key) {"), + ).toBe(true); + }); + it("keeps a candidate whose refutation does not hold on a second check", async () => { + const p = provider([ + use, + stop([finding]), + verified([], [{ id: "F1", quote: "function find(key) {" }]), + verified(["F1"]), + ]); + expect((await run(p)).findings).toEqual([finding]); + }); + it("keeps the first decision when the second check cannot run", async () => { + // The scripted provider has no answer for the second check. const p = provider([ use, stop([finding]), verified([], [{ id: "F1", quote: "function find(key) {" }]), ]); expect((await run(p)).findings).toEqual([]); - expect(p.chat).toHaveBeenCalledTimes(3); + expect(p.chat).toHaveBeenCalledTimes(4); }); it("keeps verified defects while rebuilding source state", async () => { expect( From 4508e553dc69f0de2104ce06823341ce98c7a798 Mon Sep 17 00:00:00 2001 From: Hyo Date: Thu, 19 Nov 2026 09:12:02 +0900 Subject: [PATCH 2/2] fix: keep the first verification decision when the second check has no refutation list --- convex/pullRequests/workflow.ts | 8 +++++++- test/convex/pullRequests/review.test.ts | 12 ++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/convex/pullRequests/workflow.ts b/convex/pullRequests/workflow.ts index f1a7416..c570636 100644 --- a/convex/pullRequests/workflow.ts +++ b/convex/pullRequests/workflow.ts @@ -836,6 +836,9 @@ export async function reviewWithContext(args: { sources: ReadEvidence[], billingLabel: string, timeoutMs?: number, + // A second check must say what it refutes; without that list it has no + // decision, and the first one stands. + requireRefutations = false, ) => { const critique = await chat( [ @@ -906,7 +909,9 @@ export async function reviewWithContext(args: { selection.retainedFindingIds.length || selection.retainedFindingIds.some( (id: unknown) => !candidates.some((candidate) => candidate.id === id), - ) + ) || + (requireRefutations && + !("refutations" in selection && Array.isArray(selection.refutations))) ) throw new Error("Verification introduced an unreviewed finding"); // A candidate is dropped only on a refutation quoting the shown source; @@ -1002,6 +1007,7 @@ export async function reviewWithContext(args: { sources, `${billingLabel}/appeal`, timeoutMs, + true, ); trace(`${billingLabel}/appeal`, appealed); } catch (error) { diff --git a/test/convex/pullRequests/review.test.ts b/test/convex/pullRequests/review.test.ts index 048e650..f624c3d 100644 --- a/test/convex/pullRequests/review.test.ts +++ b/test/convex/pullRequests/review.test.ts @@ -1529,6 +1529,18 @@ describe("bounded review workflow", () => { ]); expect((await run(p)).findings).toEqual([finding]); }); + it("keeps the first decision when the second check returns no refutation list", async () => { + const p = provider([ + use, + stop([finding]), + verified([], [{ id: "F1", quote: "function find(key) {" }]), + { + content: JSON.stringify({ retainedFindingIds: [], refutations: null }), + finishReason: "stop", + }, + ]); + expect((await run(p)).findings).toEqual([]); + }); it("keeps the first decision when the second check cannot run", async () => { // The scripted provider has no answer for the second check. const p = provider([