Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions convex/_generated/api.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -96,12 +96,15 @@ import type * as providerApiKeys_query from "../providerApiKeys/query.js";
import type * as providerApiKeys_usability from "../providerApiKeys/usability.js";
import type * as providerApiKeys_validation from "../providerApiKeys/validation.js";
import type * as pullRequests_action from "../pullRequests/action.js";
import type * as pullRequests_catchUp from "../pullRequests/catchUp.js";
import type * as pullRequests_commands from "../pullRequests/commands.js";
import type * as pullRequests_context from "../pullRequests/context.js";
import type * as pullRequests_errors from "../pullRequests/errors.js";
import type * as pullRequests_github from "../pullRequests/github.js";
import type * as pullRequests_internalAction from "../pullRequests/internalAction.js";
import type * as pullRequests_internalMutation from "../pullRequests/internalMutation.js";
import type * as pullRequests_internalQuery from "../pullRequests/internalQuery.js";
import type * as pullRequests_records from "../pullRequests/records.js";
import type * as pullRequests_review from "../pullRequests/review.js";
import type * as pullRequests_webhook from "../pullRequests/webhook.js";
import type * as pullRequests_workflow from "../pullRequests/workflow.js";
Expand Down Expand Up @@ -265,12 +268,15 @@ declare const fullApi: ApiFromModules<{
"providerApiKeys/usability": typeof providerApiKeys_usability;
"providerApiKeys/validation": typeof providerApiKeys_validation;
"pullRequests/action": typeof pullRequests_action;
"pullRequests/catchUp": typeof pullRequests_catchUp;
"pullRequests/commands": typeof pullRequests_commands;
"pullRequests/context": typeof pullRequests_context;
"pullRequests/errors": typeof pullRequests_errors;
"pullRequests/github": typeof pullRequests_github;
"pullRequests/internalAction": typeof pullRequests_internalAction;
"pullRequests/internalMutation": typeof pullRequests_internalMutation;
"pullRequests/internalQuery": typeof pullRequests_internalQuery;
"pullRequests/records": typeof pullRequests_records;
"pullRequests/review": typeof pullRequests_review;
"pullRequests/webhook": typeof pullRequests_webhook;
"pullRequests/workflow": typeof pullRequests_workflow;
Expand Down
7 changes: 7 additions & 0 deletions convex/crons.ts
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,13 @@ crons.daily(
{},
);

crons.daily(
"trim-review-traces",
{ hourUTC: 3, minuteUTC: 40 },
internal.pullRequests.internalMutation.trimReviewTraces,
{},
);

// Push webhooks update immediately; daily synchronization also catches missed events.
crons.daily(
"daily-repository-sync",
Expand Down
71 changes: 68 additions & 3 deletions convex/pullRequests/internalAction.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
UserApiKeyError,
usesUserKey,
} from "../_ai/providerResolution";
import type { AiProvider } from "../_ai/model";
import {
BATCH_REVIEW_THRESHOLD,
changedSince,
Expand All @@ -35,7 +36,8 @@ import {
treePaths,
type PullRequest,
} from "./github";
import { reviewWithContext } from "./workflow";
import { reviewWithContext, type ReviewTraceEvent } from "./workflow";
import { summarizeReviewTrace } from "./records";
import { REVIEW_QUOTA_FAILURES } from "./internalMutation";
import {
isReviewGroupModelFailure,
Expand Down Expand Up @@ -329,6 +331,45 @@ export const generate = internalAction({
const group = args.group ?? 0;
let stage = "Source preflight";
let batched = args.page !== undefined;
// What this run did, recorded once a model was chosen to measure review
// quality. A record that cannot be written never fails the review.
const traceEvents: ReviewTraceEvent[] = [];
// Read when the record is written: a fallback provider reports the
// provider that answered last.
let runProvider: AiProvider | undefined;
let runOutcome: "proposed" | "refused" | "group_failed" = "proposed";
let run:
| {
kind: "full" | "scoped" | "verification";
keySource: "gitdog" | "user";
changedFiles: number;
inspectedFiles: number;
uncoveredFiles: number;
discussionEntries: number;
findings: number;
openRequests: number;
}
| undefined;
const recordRun = async (error?: unknown) => {
if (!run || !runProvider) return;
await ctx
.runMutation(internal.pullRequests.internalMutation.recordReview, {
actionId: args.actionId,
...(batched ? { page, group } : {}),
...(error === undefined
? { outcome: runOutcome }
: {
outcome: isLifecycleChange(error) ? "cancelled" : "failed",
failedStage: stage,
Comment thread
hyochan marked this conversation as resolved.
Resolved
}),
...run,
providerId: runProvider.providerId,
Comment thread
hyochan marked this conversation as resolved.
Resolved
modelId: runProvider.modelId,
...summarizeReviewTrace(traceEvents),
durationMs: Date.now() - startedAt,
})
.catch(() => console.warn("Review record was not written"));
};
try {
const initial = await ctx.runQuery(
internal.pullRequests.internalQuery.get,
Expand Down Expand Up @@ -487,6 +528,17 @@ export const generate = internalAction({
userId: initial.repository.userId,
ticketActionId: args.actionId,
});
runProvider = rawAiProvider;
run = {
kind: verificationOnly ? "verification" : scoped ? "scoped" : "full",
keySource: usesUserKey(rawAiProvider) ? "user" : "gitdog",
changedFiles: selected.filter((f) => f.status !== "removed").length,
inspectedFiles: 0,
uncoveredFiles: 0,
discussionEntries: 0,
findings: 0,
openRequests: 0,
};
if (batched && !previous && group === 0) {
// Refuse up front instead of spending tokens on early batches and
// stopping at the account limit with nothing published.
Expand Down Expand Up @@ -532,6 +584,8 @@ export const generate = internalAction({
: [];
stage = "AI generation and evidence validation";
if (batched && !hasReviewableFiles) {
// No model is called for this group, so there is no run to record.
Comment thread
hyochan marked this conversation as resolved.
Resolved
run = undefined;
for (const file of selected) excludedPaths.add(file.filename);
// A removed file has no added code to review.
uncoveredFiles += selected.filter((f) => f.status !== "removed").length;
Expand Down Expand Up @@ -565,9 +619,11 @@ export const generate = internalAction({
});
let result: Awaited<ReturnType<typeof reviewWithContext>>;
try {
if (discussion) run.discussionEntries = discussion.length;
result = await reviewWithContext({
startedAt,
discussion,
trace: (event) => traceEvents.push(event),
provider,
batchGroup: batched,
allowSuggestions:
Expand Down Expand Up @@ -604,6 +660,7 @@ export const generate = internalAction({
(refused && refusedGroups >= MAX_REFUSED_REVIEW_GROUPS)
)
throw error;
runOutcome = refused ? "refused" : "group_failed";
Comment thread
hyochan marked this conversation as resolved.
Resolved
if (refused) {
refusedGroups++;
for (const file of selected) refusedPaths.add(file.filename);
Expand All @@ -619,6 +676,8 @@ export const generate = internalAction({
};
}
openRequests = result.openRequests;
run.findings = result.findings.length;
run.openRequests = result.openRequests.length;
// Each review group decides windowing for its own reads, so the way
// each quoted file was seen travels with the evidence itself.
findings.push(
Expand All @@ -638,9 +697,13 @@ export const generate = internalAction({
for (const file of selected) {
if (file.status === "removed") continue;
// A diff dropped for size was not reviewed even if its source was read.
if (hasReviewablePatch(file) && inspected.has(file.filename))
if (hasReviewablePatch(file) && inspected.has(file.filename)) {
inspectedFiles++;
else uncoveredFiles++;
run.inspectedFiles++;
} else {
uncoveredFiles++;
run.uncoveredFiles++;
}
}
for (const path of result.excludedPaths) excludedPaths.add(path);
for (const read of result.reads)
Expand Down Expand Up @@ -732,7 +795,9 @@ export const generate = internalAction({
uncoveredFiles,
},
});
await recordRun();
} catch (error) {
await recordRun(error);
// Do not persist provider bodies, source snippets or credentials in errors.
if (batched)
stage = `${stage} (review batch ${page}${group ? `, part ${group + 1}` : ""})`;
Expand Down
82 changes: 82 additions & 0 deletions convex/pullRequests/internalMutation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import {
reviewSummary,
} from "./review";
import { isBotPullRequestAuthor, REVIEW_BATCH_TOKEN_ESTIMATE } from "./github";
import { REVIEW_TRACE_RETENTION_MS, reviewTraceStepValidator } from "./records";

const snapshotValidator = v.object({
headSha: v.string(),
Expand Down Expand Up @@ -1467,3 +1468,84 @@ export const recordStatusComment = internalMutation({
return true;
},
});

export const recordReview = internalMutation({
args: {
actionId: v.id("ticketActions"),
page: v.optional(v.number()),
group: v.optional(v.number()),
kind: v.union(
v.literal("full"),
v.literal("scoped"),
v.literal("verification"),
),
// "refused" and "group_failed": the model call failed and the review
// went on without this run's files.
outcome: v.union(
v.literal("proposed"),
v.literal("refused"),
v.literal("group_failed"),
v.literal("failed"),
v.literal("cancelled"),
),
// The fixed name of the review stage a failed run stopped in.
failedStage: v.optional(v.string()),
providerId: v.string(),
modelId: v.string(),
keySource: v.union(v.literal("gitdog"), v.literal("user")),
durationMs: v.number(),
changedFiles: v.number(),
inspectedFiles: v.number(),
uncoveredFiles: v.number(),
discussionEntries: v.number(),
steps: v.number(),
candidates: v.number(),
discarded: v.number(),
findings: v.number(),
openRequests: v.number(),
trace: v.array(reviewTraceStepValidator),
},
handler: async (ctx, args) => {
const action = await ctx.db.get(args.actionId);
if (!action || action.actionType !== "review") return;
const ticket = await ctx.db.get(action.ticketId);
if (!ticket?.repositoryId) return;
const now = Date.now();
const { trace, ...counts } = args;
await ctx.db.insert("reviewRecords", {
...counts,
repositoryId: ticket.repositoryId,
...(trace.length
? { trace, traceExpiresAt: now + REVIEW_TRACE_RETENTION_MS }
: {}),
createdAt: now,
});
},
});

const REVIEW_TRACE_TRIM_BATCH = 20;
Comment thread
hyochan marked this conversation as resolved.
Resolved

// Clears review traces past their retention window; the counts stay.
export const trimReviewTraces = internalMutation({
args: {},
handler: async (ctx) => {
const expired = await ctx.db
.query("reviewRecords")
.withIndex("by_trace_expiry", (q) =>
q.gt("traceExpiresAt", 0).lte("traceExpiresAt", Date.now()),
)
.take(REVIEW_TRACE_TRIM_BATCH);
for (const record of expired)
await ctx.db.patch(record._id, {
trace: undefined,
traceExpiresAt: undefined,
});
if (expired.length === REVIEW_TRACE_TRIM_BATCH)
await ctx.scheduler.runAfter(
0,
internal.pullRequests.internalMutation.trimReviewTraces,
{},
);
return expired.length;
},
});
84 changes: 84 additions & 0 deletions convex/pullRequests/records.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
import { v } from "convex/values";
import type { ReviewTraceEvent } from "./workflow";

// How long a review run's per-step trace is kept for quality checks.
export const REVIEW_TRACE_RETENTION_MS = 30 * 24 * 60 * 60 * 1_000;
// Bounds one record to a few hundred KB even for multibyte text, so a trim
// batch stays well inside a transaction's read limit.
const MAX_TRACE_STEPS = 16;
const MAX_STEP_ENTRIES = 15;
const MAX_TITLE_LENGTH = 160;
const MAX_PATH_LENGTH = 200;

export const reviewTraceStepValidator = v.object({
step: v.string(),
proposed: v.optional(
v.array(
v.object({
path: v.string(),
priority: v.string(),
confidence: v.number(),
title: v.string(),
}),
),
),
kept: v.optional(v.array(v.string())),
discarded: v.optional(v.array(v.string())),
});

export type ReviewTraceStep = typeof reviewTraceStepValidator.type;

// Cut by code point so a surrogate pair is never split.
Comment thread
hyochan marked this conversation as resolved.
Resolved
const text = (value: unknown, limit: number) =>
typeof value === "string" ? Array.from(value).slice(0, limit).join("") : "";

/** A bounded copy of what each review step proposed and kept. */
export function summarizeReviewTrace(events: ReviewTraceEvent[]): {
trace: ReviewTraceStep[];
steps: number;
candidates: number;
discarded: number;
} {
let candidates = 0;
let discarded = 0;
const trace: ReviewTraceStep[] = [];
for (const event of events) {
if (event.stage === "proposed") {
candidates += event.candidates.length;
trace.push({
step: text(event.step, MAX_PATH_LENGTH),
proposed: event.candidates.slice(0, MAX_STEP_ENTRIES).map((c) => ({
path: text(c.path, MAX_PATH_LENGTH),
priority: text(c.priority, 8),
confidence:
typeof c.confidence === "number" && Number.isFinite(c.confidence)
? c.confidence
: 0,
title: text(c.title, MAX_TITLE_LENGTH),
})),
});
} else {
discarded += event.discarded.length;
trace.push({
step: text(event.step, MAX_PATH_LENGTH),
kept: event.kept
.slice(0, MAX_STEP_ENTRIES)
.map((t) => text(t, MAX_TITLE_LENGTH)),
discarded: event.discarded
.slice(0, MAX_STEP_ENTRIES)
.map((t) => text(t, MAX_TITLE_LENGTH)),
});
}
}
const half = MAX_TRACE_STEPS / 2;
return {
// The first steps and the last, where the final verification is.
Comment thread
hyochan marked this conversation as resolved.
Resolved
trace:
trace.length > MAX_TRACE_STEPS
? [...trace.slice(0, half), ...trace.slice(-half)]
: trace,
steps: events.filter((e) => e.stage === "proposed").length,
candidates,
discarded,
};
}
Loading
Loading