diff --git a/README.md b/README.md index a64c002..ddb1f9a 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,9 @@ approval until the new commits are reviewed. ## Pull request commands Gitdog reviews a pull request once, when it is opened or marked ready for -review. After that, ask for more in a pull request comment or from the CLI: +review. When pull request reviews turn on, it also gives up to five open pull +requests from the last 30 days their first review, skipping drafts and bots. +After that, ask for more in a pull request comment or from the CLI: | Pull request comment | CLI | What it does | | ----------------------- | ------------------------------------------- | ------------------------------------------------ | diff --git a/convex/apiKeys/gateway.ts b/convex/apiKeys/gateway.ts index 3d092e8..ebcecf0 100644 --- a/convex/apiKeys/gateway.ts +++ b/convex/apiKeys/gateway.ts @@ -1,3 +1,7 @@ +import { + reviewOpenPullRequests, + turnsReviewsOn, +} from "../pullRequests/catchUp"; import { internalMutation } from "../_generated/server"; import type { MutationCtx } from "../_generated/server"; import type { Doc } from "../_generated/dataModel"; @@ -200,6 +204,8 @@ export const execute = internalMutation({ Object.assign(settings, { [key]: value }); } await ctx.db.patch(repo._id, { settings, updatedAt: Date.now() }); + if (repo.isActive && turnsReviewsOn(repo.settings, settings)) + await reviewOpenPullRequests(ctx, repo); return repo._id; } case "repoIndexing/query:getIndexingStatus": diff --git a/convex/pullRequests/catchUp.ts b/convex/pullRequests/catchUp.ts new file mode 100644 index 0000000..b7aa56b --- /dev/null +++ b/convex/pullRequests/catchUp.ts @@ -0,0 +1,40 @@ +import { internal } from "../_generated/api"; +import type { Doc } from "../_generated/dataModel"; +import type { MutationCtx } from "../_generated/server"; + +// Turning reviews off and on again cannot start this more than once an hour. +const CATCH_UP_COOLDOWN_MS = 60 * 60_000; + +/** + * Reviews start on pull request events, so pull requests opened before + * reviews were on never get one. Each path that turns reviews on for an + * active repository calls this; it schedules a few of them for review. + */ +export async function reviewOpenPullRequests( + ctx: MutationCtx, + repository: Doc<"repositories">, +): Promise { + const now = Date.now(); + if ( + repository.pullRequestCatchUpAt !== undefined && + now - repository.pullRequestCatchUpAt < CATCH_UP_COOLDOWN_MS + ) + return; + await ctx.db.patch(repository._id, { pullRequestCatchUpAt: now }); + await ctx.scheduler.runAfter( + 0, + internal.pullRequests.internalAction.reviewOpenPullRequests, + { repositoryId: repository._id }, + ); +} + +/** Whether a settings change turns pull request reviews on. */ +export function turnsReviewsOn( + before: Doc<"repositories">["settings"], + after: { pullRequestReviewsEnabled?: boolean }, +): boolean { + return ( + before.pullRequestReviewsEnabled !== true && + after.pullRequestReviewsEnabled === true + ); +} diff --git a/convex/pullRequests/errors.ts b/convex/pullRequests/errors.ts index f0660b4..2e0d3b2 100644 --- a/convex/pullRequests/errors.ts +++ b/convex/pullRequests/errors.ts @@ -213,6 +213,10 @@ const fixedFailureReasons = new Map([ "Verification introduced an unreviewed finding", "The AI verification response was not a valid selection of the reviewed findings.", ], + [ + "Open pull requests are invalid", + "GitHub returned an unexpected list of open pull requests.", + ], [ "Finding must anchor to an added line in the reviewed diff", "A finding did not point to a line added by the pull request.", diff --git a/convex/pullRequests/github.ts b/convex/pullRequests/github.ts index e8e4d5b..4e95a6a 100644 --- a/convex/pullRequests/github.ts +++ b/convex/pullRequests/github.ts @@ -81,6 +81,20 @@ export async function getPullRequest( throw new Error("Pull request identity is invalid"); return pr; } +/** Open pull requests, the most recently updated first. */ +export async function listOpenPullRequests( + access: VerifiedRepositoryAccess, +): Promise> { + const pulls = await readReviewJson< + Array + >( + access, + "/pulls?state=open&sort=updated&direction=desc&per_page=30", + "List open pull requests", + ); + if (!Array.isArray(pulls)) throw new Error("Open pull requests are invalid"); + return pulls; +} // Every group is a full model review, so bound the total before any spend. const MAX_REVIEW_FILES = 300; export const BATCH_REVIEW_THRESHOLD = 60; diff --git a/convex/pullRequests/internalAction.ts b/convex/pullRequests/internalAction.ts index 0018ef2..f91f4c6 100644 --- a/convex/pullRequests/internalAction.ts +++ b/convex/pullRequests/internalAction.ts @@ -18,6 +18,7 @@ import { BATCH_REVIEW_THRESHOLD, changedSince, getPullRequest, + listOpenPullRequests, hasUnresolvedOwnThreads, resolveOwnThreads, isBotPullRequestAuthor, @@ -208,6 +209,75 @@ export const observe = internalAction({ }, }); +// Reviews start on pull request events, so pull requests opened before reviews +// were on never get one. When reviews turn on, a few recently updated ones that +// were never reviewed get their first review, spaced further apart than one +// review action can run, because an account runs one review at a time. +const CATCH_UP_PULL_REQUESTS = 5; +const CATCH_UP_WINDOW_MS = 30 * 24 * 60 * 60_000; +const CATCH_UP_SPACING_MS = 12 * 60_000; + +export const reviewOpenPullRequests = internalAction({ + args: { repositoryId: v.id("repositories") }, + handler: async (ctx, args): Promise => { + const repo = await ctx.runQuery( + internal.repositories.internalQuery.getById, + { repositoryId: args.repositoryId }, + ); + if ( + !repo || + repo.isActive !== true || + repo.settings.pullRequestReviewsEnabled !== true + ) + return; + // Reviews the account cannot pay for would only post failure comments on + // old pull requests; the owner can still ask for one with a command. + const provider = await resolveGenerationProvider(ctx, { + userId: repo.userId, + }).catch(() => null); + if (!provider) return; + if ( + !usesUserKey(provider) && + !(await ctx.runQuery( + internal.pullRequests.internalQuery.hasReviewAllowance, + { userId: repo.userId }, + )) + ) + return; + const access = await resolveVerifiedRepositoryAccess( + ctx, + args.repositoryId, + ); + const now = Date.now(); + const candidates = (await listOpenPullRequests(access)).filter( + (pr) => + pr !== null && + typeof pr === "object" && + Number.isSafeInteger(pr.number) && + pr.draft !== true && + typeof pr.user?.login === "string" && + !isBotPullRequestAuthor(pr.user) && + now - Date.parse(pr.updated_at) <= CATCH_UP_WINDOW_MS, + ); + const reviewed = new Set( + await ctx.runQuery(internal.pullRequests.internalQuery.reviewedNumbers, { + repositoryId: args.repositoryId, + numbers: candidates.map((pr) => pr.number), + }), + ); + const recent = candidates + .filter((pr) => !reviewed.has(pr.number)) + .slice(0, CATCH_UP_PULL_REQUESTS); + // observe reviews a pull request only until it has one finished review. + for (const [index, pr] of recent.entries()) + await ctx.scheduler.runAfter( + index * CATCH_UP_SPACING_MS, + internal.pullRequests.internalAction.observe, + { repositoryId: args.repositoryId, number: pr.number }, + ); + }, +}); + const REVIEW_COVERAGE_PREVIEW = 100; const MAX_FAILED_REVIEW_GROUPS = 2; const MAX_REFUSED_REVIEW_GROUPS = 5; diff --git a/convex/pullRequests/internalQuery.ts b/convex/pullRequests/internalQuery.ts index aeeeb18..2ed96f7 100644 --- a/convex/pullRequests/internalQuery.ts +++ b/convex/pullRequests/internalQuery.ts @@ -1,4 +1,5 @@ import { internalQuery } from "../_generated/server"; +import { checkAccountTokenAllowance } from "../subscriptions/internalQuery"; import { v } from "convex/values"; export const get = internalQuery({ args: { actionId: v.id("ticketActions") }, @@ -37,3 +38,34 @@ export const getTicket = internalQuery({ return ticket; }, }); + +/** Which of these pull requests already have a review of any kind. */ +export const reviewedNumbers = internalQuery({ + args: { repositoryId: v.id("repositories"), numbers: v.array(v.number()) }, + handler: async (ctx, args) => { + const reviewed: number[] = []; + for (const number of args.numbers.slice(0, 30)) { + const ticket = await ctx.db + .query("tickets") + .withIndex("by_github_number", (q) => + q.eq("repositoryId", args.repositoryId).eq("githubNumber", number), + ) + .first(); + if (!ticket || ticket.ticketType !== "pull_request") continue; + const review = await ctx.db + .query("ticketActions") + .withIndex("by_ticket", (q) => q.eq("ticketId", ticket._id)) + .filter((q) => q.eq(q.field("actionType"), "review")) + .first(); + if (review) reviewed.push(number); + } + return reviewed; + }, +}); + +/** Whether the account has tokens left for a review on Gitdog's key. */ +export const hasReviewAllowance = internalQuery({ + args: { userId: v.id("users") }, + handler: async (ctx, args) => + (await checkAccountTokenAllowance(ctx, args.userId)).allowed, +}); diff --git a/convex/repositories/internalMutation.ts b/convex/repositories/internalMutation.ts index 9aead7c..3a25047 100644 --- a/convex/repositories/internalMutation.ts +++ b/convex/repositories/internalMutation.ts @@ -1,3 +1,4 @@ +import { reviewOpenPullRequests } from "../pullRequests/catchUp"; import { paginationOptsValidator } from "convex/server"; import { v } from "convex/values"; import { internal } from "../_generated/api"; @@ -475,6 +476,11 @@ async function restoreRepositoryFromLiveAccess( indexIdentityVerifiedGithubRepo: undefined, updatedAt: now, }); + if ( + shouldReactivate && + repository.settings.pullRequestReviewsEnabled === true + ) + await reviewOpenPullRequests(ctx, repository); return shouldReactivate; } diff --git a/convex/repositories/mutation.ts b/convex/repositories/mutation.ts index 8d6bd48..d903494 100644 --- a/convex/repositories/mutation.ts +++ b/convex/repositories/mutation.ts @@ -19,6 +19,10 @@ import { } from "../github/installationAccess"; import { getCanonicalGitHubRepositoryFullNameKey } from "../github/repositoryIdentity"; import { pauseRepositoryForAccessRemoval } from "./accessLifecycle"; +import { + reviewOpenPullRequests, + turnsReviewsOn, +} from "../pullRequests/catchUp"; import { getRepositoryNameQuarantineFullName, readCanonicalRepositoryNameMatches, @@ -152,6 +156,8 @@ async function reactivateInstallationReconnect( internal.repoIndexing.action.indexRepository, { repositoryId: repository._id, expectedIndexVersion: indexVersion }, ); + if (repository.settings.pullRequestReviewsEnabled === true) + await reviewOpenPullRequests(ctx, repository); } return repository._id; } @@ -392,10 +398,13 @@ export const updateSettings = mutation({ throw new Error("repository disconnect cleanup is still in progress"); } + const settings = { ...repo.settings, ...args.settings }; await ctx.db.patch(args.repositoryId, { - settings: { ...repo.settings, ...args.settings }, + settings, updatedAt: Date.now(), }); + if (repo.isActive && turnsReviewsOn(repo.settings, settings)) + await reviewOpenPullRequests(ctx, repo); return args.repositoryId; }, @@ -547,6 +556,8 @@ export const toggleActive = mutation({ expectedIndexVersion: activationVersion, }, ); + if (repo.settings.pullRequestReviewsEnabled === true) + await reviewOpenPullRequests(ctx, repo); } return args.repositoryId; diff --git a/convex/schema.ts b/convex/schema.ts index 8bf109f..2a0df3a 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -776,6 +776,9 @@ const applicationTables = { ), // Issue Assistant settings (simplified) + // When open pull requests were last scheduled for review because + // reviews turned on. + pullRequestCatchUpAt: v.optional(v.number()), settings: v.object({ // Issue Assistant features issueAssistantEnabled: v.boolean(), // Master toggle diff --git a/docs/github-app-setup.md b/docs/github-app-setup.md index 9fae17b..c213fec 100644 --- a/docs/github-app-setup.md +++ b/docs/github-app-setup.md @@ -90,7 +90,13 @@ dropped in verification only when the verifier quotes source that refutes it. found there; `bun run test:e2e:recall` replays the reviewer on them. Gitdog reviews a pull request once, when it is opened or marked ready for -review. Later reviews run when the author or a collaborator comments +review. When pull request reviews are turned on (web, CLI or project key), or a +repository with them on is reactivated, reconnected or regains GitHub access, +up to five open pull requests updated in the last 30 days that were never +reviewed get their first review, 12 minutes apart. Drafts and bot pull requests +are skipped, it runs at most once an hour per repository, and nothing is +scheduled when the account has no tokens and no usable key of its own. Later +reviews run when the author or a collaborator comments `@ review` (changes since the last review) or `@ full review`; `@ resolve` resolves Gitdog's threads and `@ help` lists the commands; `gitdog reviews request|resolve` runs the same from the CLI. diff --git a/packages/cli/docs/REFERENCE.md b/packages/cli/docs/REFERENCE.md index 09d9df8..5cda4ec 100644 --- a/packages/cli/docs/REFERENCE.md +++ b/packages/cli/docs/REFERENCE.md @@ -384,7 +384,8 @@ npx gitdog@latest reviews recheck ACTION_ID ``` Gitdog reviews a pull request once, when it is opened or marked ready for -review. The `reviews` commands match the pull request comments +review, and turning `pullRequestReviewsEnabled` on also gives up to five open +pull requests from the last 30 days their first review. The `reviews` commands match the pull request comments `@gitdogai review` (commits since the last review), `@gitdogai full review` and `@gitdogai resolve` (resolves Gitdog's threads); the CLI prints the result instead of replying on GitHub. On GitHub, commands go in the pull request diff --git a/public/llms-full.txt b/public/llms-full.txt index 885aa7f..24022a9 100644 --- a/public/llms-full.txt +++ b/public/llms-full.txt @@ -40,7 +40,7 @@ When a pull request is opened or marked ready for review, Gitdog: ## Commands -Gitdog reviews a pull request once, when it is opened or marked ready for review. After that, the pull request author or a repository collaborator asks for more: +Gitdog reviews a pull request once, when it is opened or marked ready for review. When pull request reviews turn on, or a repository with them on is reactivated, reconnected or regains GitHub access, Gitdog also gives up to five open pull requests from the last 30 days their first review, skipping drafts and bots. After that, the pull request author or a repository collaborator asks for more: | Pull request comment | CLI | What it does | | --- | --- | --- | diff --git a/public/llms.txt b/public/llms.txt index ced3182..0077d8d 100644 --- a/public/llms.txt +++ b/public/llms.txt @@ -16,7 +16,7 @@ Gitdog is an AI automation platform for GitHub that helps developers automate re ## Commands -Gitdog reviews a pull request once, when it is opened or marked ready for review. After that, the pull request author or a repository collaborator asks for more: +Gitdog reviews a pull request once, when it is opened or marked ready for review. When pull request reviews turn on, or a repository with them on is reactivated, reconnected or regains GitHub access, Gitdog also gives up to five open pull requests from the last 30 days their first review, skipping drafts and bots. After that, the pull request author or a repository collaborator asks for more: | Pull request comment | CLI | What it does | | --- | --- | --- | diff --git a/test/convex/pullRequests/lifecycle.test.ts b/test/convex/pullRequests/lifecycle.test.ts index 3caf720..dac3c58 100644 --- a/test/convex/pullRequests/lifecycle.test.ts +++ b/test/convex/pullRequests/lifecycle.test.ts @@ -1273,6 +1273,182 @@ describe("PR review durable lifecycle", () => { }); }); +describe("open pull requests when reviews turn on", () => { + const scheduled = (f: Awaited>, name: string) => + f.t.run(async (ctx) => + (await ctx.db.system.query("_scheduled_functions").collect()).filter( + (job) => job.name.includes(name), + ), + ); + it("starts reviews of open pull requests once, when reviews turn on", async () => { + const f = await fixture(); + await f.t.run((ctx) => + ctx.db.patch(f.repositoryId, { + settings: { ...settings, pullRequestReviewsEnabled: false }, + }), + ); + const owner = f.t.withIdentity({ subject: f.userId }); + for (let i = 0; i < 2; i++) + await owner.mutation(api.repositories.mutation.updateSettings, { + repositoryId: f.repositoryId, + settings, + }); + expect(await scheduled(f, "reviewOpenPullRequests")).toHaveLength(1); + }); + it("reviews only a few recently updated pull requests by people, spaced out", async () => { + const f = await fixture(); + const github = await import("../../../convex/pullRequests/github"); + const repository = (await f.t.run((ctx) => ctx.db.get(f.repositoryId)))!; + vi.spyOn( + repositoryIdentity, + "resolveVerifiedRepositoryAccess", + ).mockResolvedValue({ + repository, + installation: {} as never, + githubRepositoryId: 123, + githubRepo: "owner/repo", + owner: "owner", + repo: "repo", + token: "fixture-token", + }); + const day = 24 * 60 * 60_000; + const pull = ( + number: number, + extra: { draft?: boolean; login?: string; daysAgo?: number } = {}, + ) => ({ + id: number, + number, + state: "open", + draft: extra.draft ?? false, + title: `PR ${number}`, + body: null, + user: { login: extra.login ?? "person" }, + head: { sha: "a".repeat(40) }, + base: { sha: "b".repeat(40), repo: { id: 123 } }, + changed_files: 1, + updated_at: new Date( + Date.now() - (extra.daysAgo ?? 1) * day, + ).toISOString(), + }); + vi.spyOn(github, "listOpenPullRequests").mockResolvedValue([ + pull(1, { draft: true }), + pull(2, { login: "dependabot[bot]" }), + pull(3), + pull(4), + pull(5), + pull(6), + pull(7), + pull(8), + pull(9, { daysAgo: 40 }), + ]); + await f.t.action( + internal.pullRequests.internalAction.reviewOpenPullRequests, + { repositoryId: f.repositoryId }, + ); + const jobs = (await scheduled(f, "internalAction:observe")).sort( + (a, b) => a.scheduledTime - b.scheduledTime, + ); + expect( + jobs.map((job) => (job.args[0] as { number: number }).number), + ).toEqual([3, 4, 5, 6, 7]); + // An account runs one review at a time. + expect( + jobs[1].scheduledTime - jobs[0].scheduledTime, + ).toBeGreaterThanOrEqual(12 * 60_000 - 1_000); + }); + it("starts at most one catch-up an hour, however often reviews toggle", async () => { + const f = await fixture(); + const owner = f.t.withIdentity({ subject: f.userId }); + for (const enabled of [false, true, false, true]) + await owner.mutation(api.repositories.mutation.updateSettings, { + repositoryId: f.repositoryId, + settings: { ...settings, pullRequestReviewsEnabled: enabled }, + }); + expect(await scheduled(f, "reviewOpenPullRequests")).toHaveLength(1); + }); + it("starts a catch-up when a repository with reviews on is reactivated", async () => { + const f = await fixture(); + const owner = f.t.withIdentity({ subject: f.userId }); + await owner.mutation(api.repositories.mutation.toggleActive, { + repositoryId: f.repositoryId, + isActive: false, + }); + await owner.mutation(api.repositories.mutation.toggleActive, { + repositoryId: f.repositoryId, + isActive: true, + }); + expect(await scheduled(f, "reviewOpenPullRequests")).toHaveLength(1); + }); + it("skips reviewed and malformed pull requests, and accounts without tokens", async () => { + const f = await fixture(); + const github = await import("../../../convex/pullRequests/github"); + const repository = (await f.t.run((ctx) => ctx.db.get(f.repositoryId)))!; + vi.spyOn( + repositoryIdentity, + "resolveVerifiedRepositoryAccess", + ).mockResolvedValue({ + repository, + installation: {} as never, + githubRepositoryId: 123, + githubRepo: "owner/repo", + owner: "owner", + repo: "repo", + token: "fixture-token", + }); + // Pull request 12 already has a review from the fixture. + const reviewed = await f.observe(); + expect(reviewed).not.toBeNull(); + const fresh = new Date().toISOString(); + const pull = (number: number, user: unknown = { login: "person" }) => + ({ + number, + draft: false, + user, + updated_at: fresh, + }) as never; + vi.spyOn(github, "listOpenPullRequests").mockResolvedValue([ + null as never, + pull(12), + pull(13, null), + pull(14), + ]); + await f.t.action( + internal.pullRequests.internalAction.reviewOpenPullRequests, + { repositoryId: f.repositoryId }, + ); + const jobs = await scheduled(f, "internalAction:observe"); + expect( + jobs.map((job) => (job.args[0] as { number: number }).number), + ).toEqual([14]); + // An account that cannot pay for reviews gets none scheduled. + vi.spyOn( + subscriptionQueries, + "checkAccountTokenAllowance", + ).mockResolvedValue({ allowed: false } as never); + await f.t.action( + internal.pullRequests.internalAction.reviewOpenPullRequests, + { repositoryId: f.repositoryId }, + ); + expect(await scheduled(f, "internalAction:observe")).toHaveLength(1); + }); + it("does nothing for a repository with reviews off", async () => { + const f = await fixture(); + const github = await import("../../../convex/pullRequests/github"); + const list = vi.spyOn(github, "listOpenPullRequests"); + await f.t.run((ctx) => + ctx.db.patch(f.repositoryId, { + settings: { ...settings, pullRequestReviewsEnabled: false }, + }), + ); + await f.t.action( + internal.pullRequests.internalAction.reviewOpenPullRequests, + { repositoryId: f.repositoryId }, + ); + expect(list).not.toHaveBeenCalled(); + expect(await scheduled(f, "internalAction:observe")).toHaveLength(0); + }); +}); + describe("bot review quota protection at the live GitHub boundary", () => { async function liveFixture(author: { login: string; type?: string }) { const f = await fixture(); diff --git a/test/convex/users/projectApiKeys.test.ts b/test/convex/users/projectApiKeys.test.ts index 807708f..88299ae 100644 --- a/test/convex/users/projectApiKeys.test.ts +++ b/test/convex/users/projectApiKeys.test.ts @@ -408,6 +408,33 @@ describe("project API key lifecycle and isolation", () => { ).toMatchObject({ _id: f.ids.ticketId }); }); + it("reviews open pull requests when a project key turns reviews on", async () => { + const f = await fixture(); + const writer = await f.create(["read", "write"]); + await f.t.run(async (ctx) => { + const repo = await ctx.db.get(f.ids.repositoryId); + await ctx.db.patch(f.ids.repositoryId, { + isActive: true, + settings: { ...repo!.settings, pullRequestReviewsEnabled: false }, + }); + }); + await f.execute( + writer.key, + "repositories/mutation:updateSettings", + { + repositoryId: f.ids.repositoryId, + settings: { pullRequestReviewsEnabled: true }, + }, + "mutation", + ); + const jobs = await f.t.run(async (ctx) => + (await ctx.db.system.query("_scheduled_functions").collect()).filter( + (job) => job.name.includes("reviewOpenPullRequests"), + ), + ); + expect(jobs).toHaveLength(1); + }); + it("enforces read/write scopes and preserves omitted settings", async () => { const f = await fixture(); const reader = await f.create();