Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 99 additions & 46 deletions convex/pullRequests/workflow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -822,25 +822,27 @@ export async function reviewWithContext(args: {
);
// A second model pass challenges candidates; deterministic quote/line checks
// remain mandatory. This is self-critique, not an independent review.
const verify = async (
findings: Finding[],
type VerificationCandidate = {
id: string;
finding: Finding;
refutation?: string;
};
// One verification request: which candidates hold, and which are refuted by
// a quote of the shown source.
const askVerification = async (
instruction: string,
candidates: VerificationCandidate[],
files: ChangedFile[],
sources: ReadEvidence[],
billingLabel: string,
timeoutMs?: number,
): Promise<Finding[]> => {
if (findings.length === 0) return findings;
await args.guard();
const candidates = findings.map((finding, index) => ({
id: `F${index + 1}`,
finding,
}));
// A second check must say what it refutes; without that list it has no
Comment thread
hyochan marked this conversation as resolved.
Resolved
// decision, and the first one stands.
requireRefutations = false,
) => {
const critique = await chat(
[
{
role: "system",
content: `${REVIEW_PROMPT}\nThis is the verification pass. Check each supplied candidate against the source. Return retainedFindingIds for the candidates that hold. A candidate may be dropped only when the source shows it is incorrect, pre-existing or already handled: for each such candidate add a refutation with its id and an exact quote of 12-1500 characters from the SOURCE blocks that shows it. A candidate without a quoted refutation is kept; uncertainty alone is not a reason to drop it. Do not rewrite findings, change anchors or confidence, add suggestions, or introduce new findings.`,
},
{ role: "system", content: `${REVIEW_PROMPT}\n${instruction}` },
reviewContextMessage(
{
candidates,
Expand Down Expand Up @@ -907,7 +909,9 @@ export async function reviewWithContext(args: {
selection.retainedFindingIds.length ||
selection.retainedFindingIds.some(
(id: unknown) => !candidates.some((candidate) => candidate.id === id),
)
) ||
(requireRefutations &&
!("refutations" in selection && Array.isArray(selection.refutations)))
)
throw new Error("Verification introduced an unreviewed finding");
// A candidate is dropped only on a refutation quoting the shown source;
Expand All @@ -929,42 +933,91 @@ export async function reviewWithContext(args: {
)
);
};
const refuted = new Set(
("refutations" in selection && Array.isArray(selection.refutations)
? selection.refutations
: []
const refuted = new Map<string, string>();
for (const r of "refutations" in selection &&
Array.isArray(selection.refutations)
? selection.refutations
: [])
if (
r !== null &&
typeof r === "object" &&
"id" in r &&
typeof r.id === "string" &&
"quote" in r &&
typeof r.quote === "string" &&
r.quote.trim().length >= 12 &&
r.quote.length <= 1500 &&
!refuted.has(r.id) &&
refutes(r.id, r.quote)
)
.filter(
(r: unknown): r is { id: string; quote: string } =>
r !== null &&
typeof r === "object" &&
"id" in r &&
typeof r.id === "string" &&
"quote" in r &&
typeof r.quote === "string" &&
r.quote.trim().length >= 12 &&
r.quote.length <= 1500 &&
refutes(r.id, r.quote),
)
.map((r: { id: string }) => r.id),
refuted.set(r.id, r.quote);
const retained = new Set(selection.retainedFindingIds as string[]);
return candidates
.filter((c) => !retained.has(c.id) && refuted.has(c.id))
.map((c) => ({ ...c, refutation: refuted.get(c.id) }));
};
// A second model pass challenges candidates; deterministic quote/line checks
// remain mandatory. This is self-critique, not an independent review.
const verify = async (
findings: Finding[],
files: ChangedFile[],
sources: ReadEvidence[],
billingLabel: string,
timeoutMs?: number,
): Promise<Finding[]> => {
if (findings.length === 0) return findings;
await args.guard();
const candidates = findings.map((finding, index) => ({
id: `F${index + 1}`,
finding,
}));
let dropped = await askVerification(
"This is the verification pass. Check each supplied candidate against the source. Return retainedFindingIds for the candidates that hold. A candidate may be dropped only when the source shows it is incorrect, pre-existing or already handled: for each such candidate add a refutation with its id and an exact quote of 12-1500 characters from the SOURCE blocks that shows it. A candidate without a quoted refutation is kept; uncertainty alone is not a reason to drop it. Do not rewrite findings, change anchors or confidence, add suggestions, or introduce new findings.",
candidates,
files,
sources,
billingLabel,
timeoutMs,
);
const trace = (step: string, checked: VerificationCandidate[]) =>
args.trace?.({
stage: "verified",
step,
kept: checked
.filter((c) => !dropped.some((d) => d.id === c.id))
.map((c) => c.finding.title),
discarded: dropped.map((c) => c.finding.title),
});
trace(billingLabel, candidates);
// A refuted candidate gets a second check of its refutation, so one
// misread of the source cannot drop a real defect. It is dropped only when
// that check refutes it again; if the check cannot run, the first
// decision stands.
if (
dropped.length > 0 &&
Date.now() - actionStarted <= COVERAGE_REQUEST_CUTOFF_MS
) {
const appealed = dropped;
try {
await args.guard();
dropped = await askVerification(
"This is a second check of refutations. Each supplied candidate was dropped by an earlier check, which quoted the source in its refutation field. Re-check each refutation independently against the source: it holds only if the quoted code, read with the code around it, shows the candidate is incorrect, pre-existing or already handled on every path the candidate describes. A refutation that misreads the code, covers only some of those paths, or quotes code the defect does not depend on does not hold. Return retainedFindingIds for the candidates whose refutation does not hold, and a refutation with an exact quote of 12-1500 characters from the SOURCE blocks for each candidate that is still refuted. Do not rewrite findings or introduce new ones.",
appealed,
files,
sources,
`${billingLabel}/appeal`,
timeoutMs,
true,
);
trace(`${billingLabel}/appeal`, appealed);
} catch (error) {
if (isLifecycleChange(error)) throw error;
dropped = appealed;
}
}
const retainedFindingIds = candidates
.filter(
(c) =>
(selection.retainedFindingIds as unknown[]).includes(c.id) ||
!refuted.has(c.id),
)
.filter((c) => !dropped.some((d) => d.id === c.id))
.map((c) => c.id);
args.trace?.({
stage: "verified",
step: billingLabel,
kept: candidates
.filter((c) => retainedFindingIds.includes(c.id))
.map((c) => c.finding.title),
discarded: candidates
.filter((c) => !retainedFindingIds.includes(c.id))
.map((c) => c.finding.title),
});
// Selection cannot mutate a previously validated candidate. Validate its
// original evidence again rather than trusting generated replacement text.
return validateFindings(
Expand Down
8 changes: 7 additions & 1 deletion e2e/convex/prRecall.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,10 @@ import { createMeteredProvider } from "../../convex/_ai/chatBilling";
import { getDefaultAiModel } from "../../convex/_ai/model";
import { CreditFallbackProvider } from "../../convex/_ai/providers";
import { reviewWithContext } from "../../convex/pullRequests/workflow";
import { ReviewContextLimitError } from "../../convex/pullRequests/errors";
import {
ReviewContextLimitError,
reviewFailureDetails,
} from "../../convex/pullRequests/errors";
import {
hasReviewablePatch,
windowChangedSource,
Expand Down Expand Up @@ -160,6 +163,9 @@ export const replayReview = internalAction({
)
: readPublic(args.repository, path, args.baseSha),
guard: async () => guard(),
}).catch((error) => {
// The same sanitized details a production review records.
throw new Error(reviewFailureDetails(error, "Replay"));
});
return {
seconds: Math.round((Date.now() - started) / 1000),
Expand Down
36 changes: 35 additions & 1 deletion test/convex/pullRequests/review.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1509,13 +1509,47 @@ describe("bounded review workflow", () => {
).rejects.toThrow("did not inspect");
});
it("does not publish speculative candidates removed by the critique pass", async () => {
const refuted = verified([], [{ id: "F1", quote: "function find(key) {" }]);
// The refutation is checked again, and holds.
const p = provider([use, stop([finding]), refuted, refuted]);
expect((await run(p)).findings).toEqual([]);
expect(p.chat).toHaveBeenCalledTimes(4);
const appeal = vi.mocked(p.chat).mock.calls[3];
expect(appeal[1]?.billingLabel).toBe("pr-review/critique/appeal");
expect(
JSON.stringify(appeal[0][1].content).includes("function find(key) {"),
).toBe(true);
});
it("keeps a candidate whose refutation does not hold on a second check", async () => {
const p = provider([
use,
stop([finding]),
verified([], [{ id: "F1", quote: "function find(key) {" }]),
verified(["F1"]),
]);
expect((await run(p)).findings).toEqual([finding]);
});
it("keeps the first decision when the second check returns no refutation list", async () => {
const p = provider([
use,
stop([finding]),
verified([], [{ id: "F1", quote: "function find(key) {" }]),
{
content: JSON.stringify({ retainedFindingIds: [], refutations: null }),
finishReason: "stop",
},
]);
expect((await run(p)).findings).toEqual([]);
expect(p.chat).toHaveBeenCalledTimes(3);
});
it("keeps the first decision when the second check cannot run", async () => {
// The scripted provider has no answer for the second check.
const p = provider([
use,
stop([finding]),
verified([], [{ id: "F1", quote: "function find(key) {" }]),
]);
expect((await run(p)).findings).toEqual([]);
expect(p.chat).toHaveBeenCalledTimes(4);
});
it("keeps verified defects while rebuilding source state", async () => {
expect(
Expand Down