diff --git a/README.md b/README.md index db1eeea..081c3b4 100644 --- a/README.md +++ b/README.md @@ -43,16 +43,16 @@ Maintainers answer the same questions, triage vague reports and review routine changes every day. Gitdog indexes your repository and drafts the work for you, with the actual code as evidence instead of generic advice. -| | What it does | -| ------------------------ | --------------------------------------------------------------------------------------- | -| **Issue assistant** | Turns rough reports into clear, structured issues and answers questions | -| **Discussion assistant** | Replies to GitHub Discussions with context from your codebase | -| **Pull request reviews** | Reviews changed code against the repository, suggests fixes and approves clean code | -| **Review commands** | `@gitdogai review`, `full review`, `resolve` and `help` in a pull request, or the CLI | -| **Maintainer tagging** | Notifies the right people when a request needs a code change | -| **Approval first** | With approval on (the default), drafts wait in your queue until you approve them | -| **Your own AI keys** | Run generation on your Anthropic or OpenAI key, encrypted at rest and never shown again | -| **Team workspace** | Share projects with members and keep per-project settings and usage in one place | +| | What it does | +| ------------------------ | ------------------------------------------------------------------------------------------- | +| **Issue assistant** | Turns rough reports into clear, structured issues and answers questions | +| **Discussion assistant** | Replies to GitHub Discussions with context from your codebase | +| **Pull request reviews** | Reviews changed code against the repository, suggests fixes and approves clean code | +| **Review commands** | `@gitdogai review`, `full review`, `resolve` and `help` in a pull request, or the CLI | +| **Maintainer tagging** | Notifies the right people when a request needs a code change | +| **Approval first** | With approval on (the default), drafts wait in your queue until you approve them | +| **Your own AI keys** | Run on your Anthropic or OpenAI key with the model and effort you choose, encrypted at rest | +| **Team workspace** | Share projects with members and keep per-project settings and usage in one place | ## Get started diff --git a/convex/_ai/config.ts b/convex/_ai/config.ts index f03ee1a..e5f63e0 100644 --- a/convex/_ai/config.ts +++ b/convex/_ai/config.ts @@ -82,6 +82,18 @@ export const CHAT_MODELS: Record< outputPer1M: 10, quotaWeight: 1, }, + // Available on an owner's own key only; priced for recorded cost. + "claude-haiku-5-5": { + provider: "anthropic", + inputPer1M: 0.1, + outputPer1M: 0.5, + quotaWeight: 0.05, + longContext: { + inputTokens: 100_000, + inputMultiplier: 5, + outputMultiplier: 5, + }, + }, "gpt-6.1-sol": { provider: "openai", inputPer1M: 2, @@ -112,6 +124,71 @@ export function getChatModel(model: string, provider: string) { return entry; } +export type ModelEffort = "low" | "medium" | "high"; +export type UserKeyModel = { + modelId: string; + name: string; + efforts: readonly ModelEffort[]; + defaultEffort: ModelEffort; +}; + +/** + * Models an owner can choose for their own provider key, the default first. + * The provider bills that usage, so it never counts against Gitdog tokens. + * Requests are not streamed and stop after two minutes, so settings that can + * think for longer, such as Fable 5.1 or xhigh and max effort, are left out. + */ +export const USER_KEY_MODELS: Record<"anthropic" | "openai", UserKeyModel[]> = { + anthropic: [ + { + modelId: "claude-opus-5-5", + name: "Claude Opus 5.5", + efforts: ["low", "medium", "high"], + defaultEffort: "medium", + }, + { + modelId: "claude-sonnet-5-5", + name: "Claude Sonnet 5.5", + efforts: ["low", "medium", "high"], + defaultEffort: "high", + }, + { + modelId: "claude-haiku-5-5", + name: "Claude Haiku 5.5", + efforts: ["low", "medium", "high"], + defaultEffort: "medium", + }, + ], + openai: [ + { + modelId: "gpt-6.1-sol", + name: "GPT-6.1 Sol", + efforts: ["low", "medium", "high"], + defaultEffort: "high", + }, + ], +}; + +/** + * The model and effort an owner's key runs with. A choice that is no longer + * offered falls back to the default, so a key never stops working. + */ +export function userKeyModel( + provider: "anthropic" | "openai", + modelId?: string, + effort?: string, +): { modelId: string; name: string; effort: ModelEffort } { + const models = USER_KEY_MODELS[provider]; + const model = models.find((m) => m.modelId === modelId) ?? models[0]; + return { + modelId: model.modelId, + name: model.name, + effort: model.efforts.includes(effort as ModelEffort) + ? (effort as ModelEffort) + : model.defaultEffort, + }; +} + /** * Quota weight held by every chat reservation: the highest weight among the * models a Gitdog request may be routed to, so a reservation stays valid diff --git a/convex/_ai/providerResolution.ts b/convex/_ai/providerResolution.ts index dfafcd5..a196d7f 100644 --- a/convex/_ai/providerResolution.ts +++ b/convex/_ai/providerResolution.ts @@ -1,3 +1,4 @@ +import { userKeyModel } from "./config"; import { internal } from "../_generated/api"; import type { Doc, Id } from "../_generated/dataModel"; import type { ActionCtx } from "../_generated/server"; @@ -193,13 +194,18 @@ async function userKeyLeaf( } return "unreadable"; } + // The owner's chosen model and effort; the provider bills this usage. + const model = userKeyModel(row.provider, row.modelId, row.effort); return new UserKeyProvider( ctx, row.userId, row.provider, row.provider === "anthropic" - ? new ClaudeProvider(apiKey, undefined, { enforceAccessDeadline: false }) - : new OpenAIProvider(apiKey), + ? new ClaudeProvider(apiKey, undefined, { + enforceAccessDeadline: false, + model, + }) + : new OpenAIProvider(apiKey, model), row.iv, ); } diff --git a/convex/_ai/providers.ts b/convex/_ai/providers.ts index 4ced7d9..73e1e27 100644 --- a/convex/_ai/providers.ts +++ b/convex/_ai/providers.ts @@ -176,17 +176,29 @@ function parseTool(value: RecordValue, input: unknown): ToolUseContent { return { type: "tool_use", id: value.id, name: value.name, input }; } +/** The model and effort a provider runs with; Gitdog's own by default. */ +export type ProviderModel = { modelId: string; name: string; effort: string }; + export class ClaudeProvider implements AiProvider { - readonly name = GENERATION_MODELS.anthropic.name; - readonly modelId = GENERATION_MODELS.anthropic.modelId; + readonly name: string; + readonly modelId: string; readonly providerId = "anthropic"; + private readonly effort: string; constructor( private readonly apiKey: string, private readonly authorize?: () => Promise>, // Gitdog's own Claude access ends at CLAUDE_API_ENDS_AT; a user's own - // key is not affected by that deadline. - private readonly routing: { enforceAccessDeadline?: boolean } = {}, - ) {} + // key is not affected by that deadline, and may choose its model. + private readonly routing: { + enforceAccessDeadline?: boolean; + model?: ProviderModel; + } = {}, + ) { + const model = routing.model ?? GENERATION_MODELS.anthropic; + this.name = model.name; + this.modelId = model.modelId; + this.effort = model.effort; + } private get enforceDeadline() { return this.routing.enforceAccessDeadline !== false; @@ -244,7 +256,7 @@ export class ClaudeProvider implements AiProvider { max_tokens: options?.maxTokens ?? 4096, thinking: { type: "adaptive" }, output_config: { - effort: GENERATION_MODELS.anthropic.effort, + effort: this.effort, ...(options?.responseFormat ? { format: { @@ -376,10 +388,18 @@ export class ClaudeProvider implements AiProvider { } export class OpenAIProvider implements AiProvider { - readonly name = GENERATION_MODELS.openai.name; - readonly modelId = GENERATION_MODELS.openai.modelId; + readonly name: string; + readonly modelId: string; readonly providerId = "openai"; - constructor(private readonly apiKey: string) {} + private readonly effort: string; + constructor( + private readonly apiKey: string, + model: ProviderModel = GENERATION_MODELS.openai, + ) { + this.name = model.name; + this.modelId = model.modelId; + this.effort = model.effort; + } async chat( messages: ChatMessage[], @@ -441,7 +461,7 @@ export class OpenAIProvider implements AiProvider { instructions: systemPrompt(messages), input, store: false, - reasoning: { effort: GENERATION_MODELS.openai.effort }, + reasoning: { effort: this.effort }, include: ["reasoning.encrypted_content"], max_output_tokens: options?.maxTokens ?? 4096, ...(options?.responseFormat diff --git a/convex/providerApiKeys/action.ts b/convex/providerApiKeys/action.ts index df4ab5d..e80cc19 100644 --- a/convex/providerApiKeys/action.ts +++ b/convex/providerApiKeys/action.ts @@ -3,7 +3,7 @@ import { ConvexError, v } from "convex/values"; import { internal } from "../_generated/api"; import type { Id } from "../_generated/dataModel"; import { action, type ActionCtx } from "../_generated/server"; -import { GENERATION_MODELS } from "../_ai/config"; +import { USER_KEY_MODELS, userKeyModel } from "../_ai/config"; import type { ApiErrorCode } from "../apiKeys/security"; import { decryptProviderKey, @@ -32,6 +32,7 @@ export type ProviderKeyErrorReason = | "unreadable" | "invalid_key" | "no_model_access" + | "unknown_model" | "validation_unavailable"; function keyError( @@ -45,6 +46,7 @@ function keyError( function rejectValidation( provider: KeyProvider, result: Exclude, + modelName: string, ): never { const name = PROVIDER_NAMES[provider]; switch (result.code) { @@ -58,7 +60,7 @@ function rejectValidation( return keyError( "BAD_REQUEST", "no_model_access", - `This ${name} API key has no access to ${GENERATION_MODELS[provider].name}.`, + `This ${name} API key has no access to ${modelName}.`, ); default: return keyError( @@ -87,7 +89,12 @@ export const save = action({ handler: async ( ctx, args, - ): Promise<{ provider: KeyProvider; last4: string; status: "active" }> => { + ): Promise<{ + provider: KeyProvider; + last4: string; + status: "active"; + modelId: string; + }> => { const userId = await getAuthUserId(ctx); if (!userId) keyError("UNAUTHORIZED", "unauthorized", "Sign in to save an API key."); @@ -123,17 +130,46 @@ export const save = action({ "Saving API keys is not available right now.", ); } + // A new key is checked against the model its owner chose, if any. + const existing = await ctx.runQuery( + internal.providerApiKeys.internalQuery.getEncryptedKey, + { userId, provider: args.provider }, + ); + let model = userKeyModel(args.provider, existing?.modelId); + let choice = { modelId: existing?.modelId, effort: existing?.effort }; await requireValidationSlot(ctx, userId); - const result = await validateProviderKey(args.provider, apiKey); - if (!result.ok) rejectValidation(args.provider, result); + let result = await validateProviderKey( + args.provider, + apiKey, + model.modelId, + ); + // A new key without access to the chosen model returns to the default + // model, so replacing a revoked key never needs the key removed first. + const defaultModel = userKeyModel(args.provider); + if ( + !result.ok && + result.code === "no_model_access" && + model.modelId !== defaultModel.modelId + ) { + model = defaultModel; + choice = { modelId: undefined, effort: undefined }; + result = await validateProviderKey(args.provider, apiKey, model.modelId); + } + if (!result.ok) rejectValidation(args.provider, result, model.name); const last4 = apiKey.slice(-4); await ctx.runMutation(internal.providerApiKeys.internalMutation.upsert, { userId, provider: args.provider, ...encrypted, last4, + ...choice, }); - return { provider: args.provider, last4, status: "active" }; + return { + provider: args.provider, + last4, + status: "active", + modelId: model.modelId, + }; }, }); @@ -189,8 +225,13 @@ export const revalidate = action({ "This API key can no longer be read. Enter it again.", ); } + const model = userKeyModel(args.provider, row.modelId); await requireValidationSlot(ctx, userId); - const result = await validateProviderKey(args.provider, apiKey); + const result = await validateProviderKey( + args.provider, + apiKey, + model.modelId, + ); if (!result.ok) { if (result.code !== "unavailable") await ctx.runMutation( @@ -203,7 +244,7 @@ export const revalidate = action({ keyIv: row.iv, }, ); - rejectValidation(args.provider, result); + rejectValidation(args.provider, result, model.name); } await ctx.runMutation( internal.providerApiKeys.internalMutation.markValidated, @@ -212,3 +253,87 @@ export const revalidate = action({ return { provider: args.provider, status: "active" }; }, }); + +/** + * Choose the model and effort generation uses on the owner's own key. The + * key must have access to that model; the provider bills the usage. + */ +export const setModel = action({ + args: { + provider: providerValidator, + modelId: v.string(), + effort: v.optional(v.string()), + }, + handler: async ( + ctx, + args, + ): Promise<{ provider: KeyProvider; modelId: string; effort: string }> => { + const userId = await getAuthUserId(ctx); + if (!userId) + keyError("UNAUTHORIZED", "unauthorized", "Sign in to choose a model."); + const offered = USER_KEY_MODELS[args.provider].find( + (m) => m.modelId === args.modelId, + ); + if ( + !offered || + (args.effort !== undefined && + !(offered.efforts as readonly string[]).includes(args.effort)) + ) + keyError( + "BAD_REQUEST", + "unknown_model", + `Choose one of: ${USER_KEY_MODELS[args.provider] + .map((m) => `${m.modelId} (${m.efforts.join(", ")})`) + .join("; ")}.`, + ); + const row = await ctx.runQuery( + internal.providerApiKeys.internalQuery.getEncryptedKey, + { userId, provider: args.provider }, + ); + if (!row) + keyError( + "NOT_FOUND", + "not_found", + "Save an API key for this provider before choosing its model.", + ); + const model = userKeyModel(args.provider, args.modelId, args.effort); + if (userKeyModel(args.provider, row.modelId).modelId !== model.modelId) { + let apiKey: string; + try { + apiKey = await decryptProviderKey(row); + } catch (error) { + if (!(error instanceof ProviderKeyEncryptionError)) throw error; + if (error.kind === "not_configured") + keyError( + "UNSUPPORTED_OPERATION", + "not_configured", + "Key checks are temporarily unavailable.", + ); + keyError( + "BAD_REQUEST", + "unreadable", + "This API key can no longer be read. Enter it again.", + ); + } + await requireValidationSlot(ctx, userId); + const result = await validateProviderKey( + args.provider, + apiKey, + model.modelId, + ); + if (!result.ok) rejectValidation(args.provider, result, model.name); + } + await ctx.runMutation(internal.providerApiKeys.internalMutation.setModel, { + userId, + provider: args.provider, + keyIv: row.iv, + modelId: model.modelId, + effort: model.effort, + }); + return { + provider: args.provider, + modelId: model.modelId, + effort: model.effort, + }; + }, +}); diff --git a/convex/providerApiKeys/internalMutation.ts b/convex/providerApiKeys/internalMutation.ts index f42bab0..3be01d2 100644 --- a/convex/providerApiKeys/internalMutation.ts +++ b/convex/providerApiKeys/internalMutation.ts @@ -79,6 +79,9 @@ export const upsert = internalMutation({ iv: v.string(), encryptionKeyId: v.string(), last4: v.string(), + // The model the new key was checked against; unset means the default. + modelId: v.optional(v.string()), + effort: v.optional(v.string()), }, handler: async (ctx, args) => { const now = Date.now(); @@ -93,6 +96,8 @@ export const upsert = internalMutation({ iv: args.iv, encryptionKeyId: args.encryptionKeyId, last4: args.last4, + modelId: args.modelId, + effort: args.effort, status: "active" as const, lastValidatedAt: now, lastError: undefined, @@ -284,3 +289,30 @@ export const setExpectedEncryptionKeyId = internalMutation({ return { expectedKeyId: args.keyId }; }, }); + +/** Records the owner's model choice on the key that was checked for it. */ +export const setModel = internalMutation({ + args: { + userId: v.id("users"), + provider: providerValidator, + keyIv: v.string(), + modelId: v.string(), + effort: v.string(), + }, + handler: async (ctx, args) => { + const row = await ctx.db + .query("providerApiKeys") + .withIndex("by_user_and_provider", (q) => + q.eq("userId", args.userId).eq("provider", args.provider), + ) + .unique(); + // A key replaced meanwhile was not checked for this model. + if (!row || row.iv !== args.keyIv) + throw new Error("The API key changed; choose the model again."); + await ctx.db.patch(row._id, { + modelId: args.modelId, + effort: args.effort, + updatedAt: Date.now(), + }); + }, +}); diff --git a/convex/providerApiKeys/query.ts b/convex/providerApiKeys/query.ts index 0dd3e0d..2b214ab 100644 --- a/convex/providerApiKeys/query.ts +++ b/convex/providerApiKeys/query.ts @@ -1,6 +1,7 @@ import { getAuthUserId } from "@convex-dev/auth/server"; import { query } from "../_generated/server"; import { readEncryptionState } from "./encryptionState"; +import { USER_KEY_MODELS, userKeyModel } from "../_ai/config"; export const list = query({ args: {}, @@ -24,6 +25,7 @@ export const list = query({ row.encryptionKeyId === state.keyId && row.unreadableAt === undefined), provider: row.provider, + ...userKeyModel(row.provider, row.modelId, row.effort), last4: row.last4, status: row.status, lastValidatedAt: row.lastValidatedAt, @@ -33,3 +35,9 @@ export const list = query({ })); }, }); + +/** The models and efforts an owner can choose for their own keys. */ +export const models = query({ + args: {}, + handler: async () => USER_KEY_MODELS, +}); diff --git a/convex/providerApiKeys/validation.ts b/convex/providerApiKeys/validation.ts index 5228630..f9c5836 100644 --- a/convex/providerApiKeys/validation.ts +++ b/convex/providerApiKeys/validation.ts @@ -1,4 +1,4 @@ -import { GENERATION_MODELS } from "../_ai/config"; +import { userKeyModel } from "../_ai/config"; import type { KeyProvider } from "./crypto"; const VALIDATION_TIMEOUT_MS = 10_000; @@ -16,18 +16,19 @@ export type KeyValidationResult = export async function validateProviderKey( provider: KeyProvider, apiKey: string, + modelId: string = userKeyModel(provider).modelId, ): Promise { const request: { url: string; headers: Record } = provider === "anthropic" ? { - url: `https://api.anthropic.com/v1/models/${GENERATION_MODELS.anthropic.modelId}`, + url: `https://api.anthropic.com/v1/models/${encodeURIComponent(modelId)}`, headers: { "x-api-key": apiKey, "anthropic-version": "2023-06-01", }, } : { - url: `https://api.openai.com/v1/models/${GENERATION_MODELS.openai.modelId}`, + url: `https://api.openai.com/v1/models/${encodeURIComponent(modelId)}`, headers: { Authorization: `Bearer ${apiKey}` }, }; let response: Response; diff --git a/convex/pullRequests/catchUp.ts b/convex/pullRequests/catchUp.ts index b7aa56b..226dff0 100644 --- a/convex/pullRequests/catchUp.ts +++ b/convex/pullRequests/catchUp.ts @@ -24,7 +24,7 @@ export async function reviewOpenPullRequests( await ctx.scheduler.runAfter( 0, internal.pullRequests.internalAction.reviewOpenPullRequests, - { repositoryId: repository._id }, + { repositoryId: repository._id, stampedAt: now }, ); } diff --git a/convex/pullRequests/internalAction.ts b/convex/pullRequests/internalAction.ts index 65572d1..2c4d23a 100644 --- a/convex/pullRequests/internalAction.ts +++ b/convex/pullRequests/internalAction.ts @@ -222,8 +222,20 @@ const CATCH_UP_WINDOW_MS = 30 * 24 * 60 * 60_000; const CATCH_UP_SPACING_MS = 12 * 60_000; export const reviewOpenPullRequests = internalAction({ - args: { repositoryId: v.id("repositories") }, + args: { + repositoryId: v.id("repositories"), + stampedAt: v.optional(v.number()), + }, handler: async (ctx, args): Promise => { + // A catch-up that schedules nothing frees the hourly allowance, so + // turning reviews on again after adding tokens works at once. + const release = async (): Promise => { + if (args.stampedAt !== undefined) + await ctx.runMutation( + internal.pullRequests.internalMutation.releaseCatchUp, + { repositoryId: args.repositoryId, stampedAt: args.stampedAt }, + ); + }; const repo = await ctx.runQuery( internal.repositories.internalQuery.getById, { repositoryId: args.repositoryId }, @@ -239,7 +251,7 @@ export const reviewOpenPullRequests = internalAction({ const provider = await resolveGenerationProvider(ctx, { userId: repo.userId, }).catch(() => null); - if (!provider) return; + if (!provider) return await release(); if ( !usesUserKey(provider) && !(await ctx.runQuery( @@ -247,13 +259,16 @@ export const reviewOpenPullRequests = internalAction({ { userId: repo.userId }, )) ) - return; + return await release(); const access = await resolveVerifiedRepositoryAccess( ctx, args.repositoryId, - ); + ).catch(() => null); + if (!access) return await release(); + const open = await listOpenPullRequests(access).catch(() => null); + if (!open) return await release(); const now = Date.now(); - const candidates = (await listOpenPullRequests(access)).filter( + const candidates = open.filter( (pr) => pr !== null && typeof pr === "object" && @@ -272,6 +287,7 @@ export const reviewOpenPullRequests = internalAction({ const recent = candidates .filter((pr) => !reviewed.has(pr.number)) .slice(0, CATCH_UP_PULL_REQUESTS); + if (recent.length === 0) return await release(); // observe reviews a pull request only until it has one finished review. for (const [index, pr] of recent.entries()) await ctx.scheduler.runAfter( diff --git a/convex/pullRequests/internalMutation.ts b/convex/pullRequests/internalMutation.ts index 6452d1a..07e4c4a 100644 --- a/convex/pullRequests/internalMutation.ts +++ b/convex/pullRequests/internalMutation.ts @@ -1429,7 +1429,16 @@ export const stampAutomaticNotice = internalMutation({ }, }); -/** Remember the status comment posted for a review, once. */ +/** Frees the hourly catch-up allowance a catch-up that did nothing took. */ +export const releaseCatchUp = internalMutation({ + args: { repositoryId: v.id("repositories"), stampedAt: v.number() }, + handler: async (ctx, args) => { + const repo = await ctx.db.get(args.repositoryId); + if (repo?.pullRequestCatchUpAt === args.stampedAt) + await ctx.db.patch(repo._id, { pullRequestCatchUpAt: undefined }); + }, +}); + /** The status comment was deleted because the review it described was posted. */ export const forgetStatusComment = internalMutation({ args: { actionId: v.id("ticketActions"), commentId: v.number() }, @@ -1440,6 +1449,7 @@ export const forgetStatusComment = internalMutation({ }, }); +/** Remember the status comment posted for a review, once. */ export const recordStatusComment = internalMutation({ args: { actionId: v.id("ticketActions"), diff --git a/convex/schema.ts b/convex/schema.ts index 2a0df3a..1e12179 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -34,6 +34,9 @@ const applicationTables = { v.literal("invalid"), v.literal("out_of_credit"), ), + // The owner's model and effort for this key; unset means the default. + modelId: v.optional(v.string()), + effort: v.optional(v.string()), lastValidatedAt: v.number(), // Set when this key failed to decrypt (the owner is told once); cleared // when the key is saved or checked again. diff --git a/docs/ai-models.md b/docs/ai-models.md index 4dd3b1a..a512901 100644 --- a/docs/ai-models.md +++ b/docs/ai-models.md @@ -28,6 +28,15 @@ AI provider keys**. Generation for repositories they own (issue and discussion replies, follow-ups, pull request reviews and learning extraction) then runs on those keys instead of Gitdog's: +- Each key runs the model and effort its owner chooses (web settings or + `gitdog provider-keys model`), checked against the key when chosen and when + the key is saved again. Anthropic: Claude Opus 5.5 (default, `medium`), + Sonnet 5.5 (`high`) or Haiku 5.5 (`medium`); OpenAI: GPT-6.1 Sol (`high`). + Efforts are `low`, `medium` or `high`: requests are not streamed and stop + after two minutes, so Fable 5.1 and `xhigh`/`max` are not offered. Haiku + 5.5 bills prompts over 100K tokens at 5x and has no server-side refusal + fallback. A new key without access to the chosen model is saved with the + default model, and a model no longer offered falls back to the default. - Claude runs on the user's Anthropic key and is not affected by Gitdog's Claude deadline. The user's OpenAI key is used if only it is saved, if the Anthropic key is rejected, or if Claude reports exhausted credit. @@ -50,7 +59,7 @@ those keys instead of Gitdog's: `printf %s "$BYOK_ENCRYPTION_KEY" | base64 -d | shasum -a 256`. Keys saved under the old server key then become unreadable, and their owners are asked to enter them again. There is no decrypt-only previous key. -- Keys are validated against the generation model on save, encrypted with +- Keys are validated against their chosen model on save, encrypted with AES-256-GCM using `BYOK_ENCRYPTION_KEY`, and never returned to clients. ## Authentication @@ -85,7 +94,8 @@ indexing costs no AI tokens. | ----------------- | ------------------- | -------------------- | ------------ | | Claude Opus 5.5 | $4.00 | $20.00 | 2 | | GPT-6.1 Sol | $2.00 | $10.00 | 1 | -| Claude Sonnet 5.5 | $2.00 | $10.00 | 1 (retired) | +| Claude Sonnet 5.5 | $2.00 | $10.00 | 1 (own keys) | +| Claude Haiku 5.5 | $0.10 | $0.50 | own keys | On Gitdog's keys, each provider token counts as its model's quota weight in Gitdog tokens: an Opus 5.5 call with 30K input and 5K output tokens uses 70K. diff --git a/packages/cli/docs/REFERENCE.md b/packages/cli/docs/REFERENCE.md index 8087525..e77aa20 100644 --- a/packages/cli/docs/REFERENCE.md +++ b/packages/cli/docs/REFERENCE.md @@ -139,6 +139,8 @@ gitdog provider-keys list gitdog provider-keys set anthropic # prompts without echoing the key printf %s "$OPENAI_API_KEY" | gitdog provider-keys set openai --stdin gitdog provider-keys check openai +gitdog provider-keys models # models and efforts you can choose +gitdog provider-keys model anthropic claude-sonnet-5-5 --effort high # no --effort: the model default gitdog provider-keys remove openai --yes ``` diff --git a/packages/cli/src/api-catalog.json b/packages/cli/src/api-catalog.json index 1db2d2f..8dae5e5 100644 --- a/packages/cli/src/api-catalog.json +++ b/packages/cli/src/api-catalog.json @@ -224,6 +224,11 @@ "kind": "action", "args": "{ provider: v.union(v.literal(\"anthropic\"), v.literal(\"openai\")), apiKey: v.string() }" }, + { + "path": "providerApiKeys/action:setModel", + "kind": "action", + "args": "{\n provider: v.union(v.literal(\"anthropic\"), v.literal(\"openai\")),\n modelId: v.string(),\n effort: v.optional(v.string()),\n }" + }, { "path": "providerApiKeys/mutation:remove", "kind": "mutation", @@ -234,6 +239,11 @@ "kind": "query", "args": "{}" }, + { + "path": "providerApiKeys/query:models", + "kind": "query", + "args": "{}" + }, { "path": "pullRequests/action:command", "kind": "action", diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index 6ab3dd4..22622db 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -87,7 +87,9 @@ const PROVIDER_KEY_ERRORS: Record = { "Gitdog can no longer read this saved key. Run provider-keys set to enter it again.", invalid_key: "The provider rejected this API key.", no_model_access: - "This API key has no access to the model Gitdog uses. Check the key's project or model permissions.", + "This API key has no access to the chosen model. Check the key's project or model permissions, or choose another with provider-keys model.", + unknown_model: + "That model or effort is not offered. Run provider-keys models to see the choices.", validation_unavailable: "The provider could not check this key right now. Try again shortly.", }; @@ -525,11 +527,15 @@ export function createProgram( serviceUnavailable?: boolean; lastValidatedAt: number; lastErrorCode: string | null; + modelId?: string; + effort?: string; }> >("query", "providerApiKeys/query:list"); const rows = keys.map((key) => ({ provider: key.provider, key: `…${key.last4}`, + model: key.modelId ?? null, + effort: key.effort ?? null, status: key.serviceUnavailable ? "checks_unavailable" : key.readable === false @@ -545,12 +551,75 @@ export function createProgram( ui.table(rows, [ "provider", "key", + "model", + "effort", "status", "lastChecked", "lastError", ]); }, ); + command( + providerKeys, + "models", + "List the models and efforts you can choose for your own keys", + async ({ api, ui }) => { + const models = await api.call< + Record< + string, + Array<{ + modelId: string; + name: string; + efforts: string[]; + defaultEffort: string; + }> + > + >("query", "providerApiKeys/query:models"); + const rows = Object.entries(models).flatMap(([provider, list]) => + list.map((m, index) => ({ + provider, + model: m.modelId, + name: m.name, + efforts: m.efforts.join(", "), + defaultEffort: m.defaultEffort, + default: index === 0, + })), + ); + if (ui.json) ui.output(rows); + else + ui.table(rows, [ + "provider", + "model", + "name", + "efforts", + "defaultEffort", + "default", + ]); + }, + ); + command( + providerKeys, + "model ", + "Choose the model (and --effort) generation uses on your own key", + async ({ api, ui }, args, opts) => { + const provider = keyProvider(args[0]); + ui.output( + await providerKeyCall( + api, + "action", + "providerApiKeys/action:setModel", + { + provider, + modelId: args[1], + ...(text(opts.effort) ? { effort: text(opts.effort) } : {}), + }, + ), + ); + }, + ).option( + "--effort ", + "Effort level, such as medium or high; defaults to the model's own", + ); command( providerKeys, "set [unexpected...]", diff --git a/packages/cli/test/commands.test.mjs b/packages/cli/test/commands.test.mjs index 8594531..286406a 100644 --- a/packages/cli/test/commands.test.mjs +++ b/packages/cli/test/commands.test.mjs @@ -418,7 +418,7 @@ test("all catalog paths are unique public functions and version matches the npm new Set(API_CATALOG.map((entry) => entry.path)).size, API_CATALOG.length, ); - assert.equal(API_CATALOG.length, 93); + assert.equal(API_CATALOG.length, 95); assert.equal( API_CATALOG.every( (entry) => diff --git a/packages/cli/test/provider-keys.test.mjs b/packages/cli/test/provider-keys.test.mjs index 7030560..5fa55ad 100644 --- a/packages/cli/test/provider-keys.test.mjs +++ b/packages/cli/test/provider-keys.test.mjs @@ -70,6 +70,8 @@ test("provider-keys list shows metadata and flags unreadable keys", async () => { provider: "anthropic", key: "…abcd", + model: null, + effort: null, status: "unreadable", lastChecked: "2026-11-01T00:00:00.000Z", lastError: null, @@ -77,6 +79,8 @@ test("provider-keys list shows metadata and flags unreadable keys", async () => { provider: "openai", key: "…wxyz", + model: null, + effort: null, status: "out_of_credit", lastChecked: "2026-11-02T00:00:00.000Z", lastError: "out_of_credit", @@ -149,7 +153,8 @@ test("provider-keys maps server reasons to clear messages", async () => { ["wrong_provider", /other provider/], ["rate_limited", /few minutes/], ["not_configured", /cannot store API keys/], - ["no_model_access", /no access to the model/], + ["no_model_access", /no access to the chosen model/], + ["unknown_model", /provider-keys models/], ["unreadable", /enter it again/], ]) { const api = recordingApi(() => { @@ -246,11 +251,21 @@ test("provider-keys list prints every column for people", async () => { await run(api, ui, ["provider-keys", "list"]); assert.deepEqual(ui.tables, [ { - keys: ["provider", "key", "status", "lastChecked", "lastError"], + keys: [ + "provider", + "key", + "model", + "effort", + "status", + "lastChecked", + "lastError", + ], rows: [ { provider: "openai", key: "…wxyz", + model: null, + effort: null, status: "checks_unavailable", lastChecked: "2026-11-02T00:00:00.000Z", lastError: null, @@ -299,3 +314,36 @@ test("output never shows an Anthropic or OpenAI key", () => { ]) assert.equal(redactSecrets(name), name); }); + +test("provider-keys models and model choose the model for your own key", async () => { + const ui = new KeyUi(); + const api = recordingApi(({ path }) => + path.endsWith("query:models") + ? { + anthropic: [ + { + modelId: "claude-opus-5-5", + name: "Claude Opus 5.5", + efforts: ["low", "medium"], + defaultEffort: "medium", + }, + ], + openai: [], + } + : { provider: "anthropic", modelId: "claude-opus-5-5", effort: "low" }, + ); + await run(api, ui, ["provider-keys", "models"]); + await run(api, ui, [ + "provider-keys", + "model", + "anthropic", + "claude-opus-5-5", + "--effort", + "low", + ]); + assert.deepEqual(api.calls.at(-1), { + kind: "action", + path: "providerApiKeys/action:setModel", + args: { provider: "anthropic", modelId: "claude-opus-5-5", effort: "low" }, + }); +}); diff --git a/public/llms-full.txt b/public/llms-full.txt index f1aaca5..70c1aa5 100644 --- a/public/llms-full.txt +++ b/public/llms-full.txt @@ -64,7 +64,7 @@ The `gitdog` npm CLI (`npx gitdog@latest`) covers the same workflows as the web ### Free Plan - $0/month - 100,000 one-time trial tokens on first sign-in -- Then use your own Anthropic or OpenAI API key (that usage never uses Gitdog tokens) +- Then use your own Anthropic or OpenAI API key (that usage never uses Gitdog tokens), with the model and effort you choose: Claude Opus 5.5, Sonnet 5.5 or Haiku 5.5, or GPT-6.1 Sol - Hard limit with no automatic overage charges - Prepaid token packs available for additional usage - All features included diff --git a/src/i18n/locales/en.json b/src/i18n/locales/en.json index 426a141..2eb1962 100644 --- a/src/i18n/locales/en.json +++ b/src/i18n/locales/en.json @@ -600,12 +600,13 @@ "errors": { "format": "This does not look like an {{provider}} API key.", "rejected": "{{provider}} rejected this API key.", - "noModelAccess": "This {{provider}} API key has no access to the model Gitdog uses.", + "noModelAccess": "This {{provider}} API key has no access to the chosen model.", "reenter": "This API key can no longer be read. Enter it again.", "unavailable": "{{provider}} could not check this API key right now. Try again shortly.", "rateLimited": "Too many API key checks. Try again in a few minutes.", "notConfigured": "Key checks are temporarily unavailable. Try again later.", - "wrongProvider": "This looks like an {{other}} key. Paste it into the {{other}} card." + "wrongProvider": "This looks like an {{other}} key. Paste it into the {{other}} card.", + "unknownModel": "Choose one of the listed models and efforts." }, "valueProp": "Run Gitdog on your own Anthropic or OpenAI key — that usage never touches your Gitdog tokens.", "bullets": { @@ -616,7 +617,11 @@ "getKey": "Get a key", "checked": "Checked {{time}}", "confirmRemove": "Remove your {{provider}} key? Gitdog will use its own tokens again.", - "moveTo": "Move to {{provider}}" + "moveTo": "Move to {{provider}}", + "modelLabel": "Model", + "effortLabel": "Effort", + "modelHint": "Runs on this key and is billed by {{provider}}. It never uses Gitdog tokens.", + "modelSaved": "Saved" } }, "footer": { diff --git a/src/i18n/locales/ja.json b/src/i18n/locales/ja.json index f75a769..6d4e7ab 100644 --- a/src/i18n/locales/ja.json +++ b/src/i18n/locales/ja.json @@ -599,12 +599,13 @@ "errors": { "format": "{{provider}} の API キーの形式ではありません。", "rejected": "{{provider}} がこの API キーを拒否しました。", - "noModelAccess": "この {{provider}} API キーでは Gitdog が使うモデルにアクセスできません。", + "noModelAccess": "この {{provider}} API キーでは選んだモデルにアクセスできません。", "reenter": "この API キーは読み取れなくなりました。もう一度入力してください。", "unavailable": "{{provider}} が現在この API キーを確認できません。しばらくしてからお試しください。", "rateLimited": "API キーの確認が多すぎます。数分後にもう一度お試しください。", "notConfigured": "現在キーを確認できません。しばらくしてからお試しください。", - "wrongProvider": "{{other}}のキーのようです。{{other}}のカードに貼り付けてください。" + "wrongProvider": "{{other}}のキーのようです。{{other}}のカードに貼り付けてください。", + "unknownModel": "一覧のモデルと推論の強さから選んでください。" }, "valueProp": "自分のAnthropicまたはOpenAIキーでGitdogを実行 — その利用分はGitdogのトークンを使いません。", "bullets": { @@ -615,7 +616,11 @@ "getKey": "キーを取得", "checked": "{{time}}に確認", "confirmRemove": "{{provider}}キーを削除しますか?以後はGitdogのトークンを使います。", - "moveTo": "{{provider}}に移動" + "moveTo": "{{provider}}に移動", + "modelLabel": "モデル", + "effortLabel": "推論の強さ", + "modelHint": "このキーで実行され、{{provider}} から請求されます。Gitdog のトークンは使いません。", + "modelSaved": "保存しました" } }, "footer": { diff --git a/src/i18n/locales/ko.json b/src/i18n/locales/ko.json index 79d669b..bcfdbb3 100644 --- a/src/i18n/locales/ko.json +++ b/src/i18n/locales/ko.json @@ -599,12 +599,13 @@ "errors": { "format": "{{provider}} API 키 형식이 아닙니다.", "rejected": "{{provider}}가 이 API 키를 거부했습니다.", - "noModelAccess": "이 {{provider}} API 키로는 Gitdog가 사용하는 모델에 접근할 수 없습니다.", + "noModelAccess": "이 {{provider}} API 키로는 선택한 모델에 접근할 수 없습니다.", "reenter": "이 API 키를 더 이상 읽을 수 없습니다. 다시 입력해 주세요.", "unavailable": "{{provider}}에서 지금 이 API 키를 확인할 수 없습니다. 잠시 후 다시 시도해 주세요.", "rateLimited": "API 키 확인 요청이 너무 많습니다. 몇 분 후 다시 시도해 주세요.", "notConfigured": "지금은 키를 확인할 수 없습니다. 잠시 후 다시 시도해 주세요.", - "wrongProvider": "{{other}} 키로 보입니다. {{other}} 카드에 붙여 넣어 주세요." + "wrongProvider": "{{other}} 키로 보입니다. {{other}} 카드에 붙여 넣어 주세요.", + "unknownModel": "목록에 있는 모델과 추론 강도 중에서 선택하세요." }, "valueProp": "내 Anthropic 또는 OpenAI 키로 Gitdog를 실행하세요 — 이 사용량은 Gitdog 토큰을 쓰지 않습니다.", "bullets": { @@ -615,7 +616,11 @@ "getKey": "키 발급받기", "checked": "{{time}} 확인", "confirmRemove": "{{provider}} 키를 삭제할까요? 이후에는 Gitdog 토큰을 다시 사용합니다.", - "moveTo": "{{provider}}로 옮기기" + "moveTo": "{{provider}}로 옮기기", + "modelLabel": "모델", + "effortLabel": "추론 강도", + "modelHint": "이 키로 실행되며 {{provider}}에서 청구됩니다. Gitdog 토큰은 사용하지 않습니다.", + "modelSaved": "저장됨" } }, "footer": { diff --git a/src/pages/profile/ProviderKeysSection.tsx b/src/pages/profile/ProviderKeysSection.tsx index 81058c9..dc7eca5 100644 --- a/src/pages/profile/ProviderKeysSection.tsx +++ b/src/pages/profile/ProviderKeysSection.tsx @@ -19,8 +19,25 @@ type SavedKey = { readable: boolean; serviceUnavailable?: boolean; lastValidatedAt: number; + modelId: string; + effort: string; }; +type ModelOption = { + modelId: string; + name: string; + efforts: readonly string[]; + defaultEffort: string; +}; + +const selectClass = cn( + "w-full px-3 py-2 text-sm", + "border border-gray-300 dark:border-gray-700", + "bg-white dark:bg-gray-900", + "focus:outline-hidden focus:border-black dark:focus:border-white", + "disabled:opacity-50", +); + const PROVIDERS: Record< KeyProvider, { label: string; placeholder: string; pattern: RegExp; keysUrl: string } @@ -86,6 +103,7 @@ const primaryButtonClass = cn( function ProviderKeyCard({ provider, saved, + models, draft, onDraftChange, editing, @@ -94,6 +112,7 @@ function ProviderKeyCard({ }: { provider: KeyProvider; saved: SavedKey | undefined; + models: ModelOption[] | undefined; draft: string; onDraftChange: (value: string) => void; editing: boolean; @@ -106,6 +125,8 @@ function ProviderKeyCard({ const saveKey = useAction(api.providerApiKeys.action.save); const revalidateKey = useAction(api.providerApiKeys.action.revalidate); const removeKey = useMutation(api.providerApiKeys.mutation.remove); + const setModel = useAction(api.providerApiKeys.action.setModel); + const [modelSaved, setModelSaved] = useState(false); const [busy, setBusy] = useState(false); const [confirmingRemove, setConfirmingRemove] = useState(false); const [error, setError] = useState(null); @@ -141,6 +162,8 @@ function ProviderKeyCard({ return t("profile.providerKeys.errors.unavailable", { provider: label, }); + case "unknown_model": + return t("profile.providerKeys.errors.unknownModel"); default: return t("common.error"); } @@ -178,6 +201,12 @@ function ProviderKeyCard({ : saved.status; const healthy = status === "active"; const showForm = !saved || editing; + const chosen = models?.find((m) => m.modelId === saved?.modelId); + const chooseModel = async (modelId: string, effort?: string) => { + setModelSaved(false); + if (await run(() => setModel({ provider, modelId, effort }))) + setModelSaved(true); + }; return (
)} + {saved?.readable && !saved.serviceUnavailable && models && !showForm && ( +
+ + +

+ {t("profile.providerKeys.modelHint", { provider: label })} + {modelSaved && ` ${t("profile.providerKeys.modelSaved")}`} +

+
+ )} + {showForm ? (
@@ -405,6 +479,7 @@ function ProviderKeyCard({ export function ProviderKeysSection() { const { t } = useTranslation(); const keys = useQuery(api.providerApiKeys.query.list); + const models = useQuery(api.providerApiKeys.query.models); const [drafts, setDrafts] = useState>({ anthropic: "", openai: "", @@ -477,6 +552,7 @@ export function ProviderKeysSection() { key={provider} provider={provider} saved={savedKey(provider)} + models={models?.[provider]} draft={drafts[provider]} onDraftChange={(value) => setDrafts((current) => ({ ...current, [provider]: value })) diff --git a/test/convex/ai/providerFallback.test.ts b/test/convex/ai/providerFallback.test.ts index a65b8fe..cb070fb 100644 --- a/test/convex/ai/providerFallback.test.ts +++ b/test/convex/ai/providerFallback.test.ts @@ -628,6 +628,26 @@ describe("Claude safety refusals", () => { totalTokens: 14, }); }); + it("runs an owner's own key on the model and effort they chose", async () => { + const fetch = vi.spyOn(globalThis, "fetch"); + fetch.mockResolvedValueOnce( + claude({ content: [{ type: "text", text: "ok" }] }), + ); + const provider = new ClaudeProvider("fixture-key", undefined, { + enforceAccessDeadline: false, + model: { + modelId: "claude-haiku-5-5", + name: "Claude Haiku 5.5", + effort: "low", + }, + }); + expect(provider.modelId).toBe("claude-haiku-5-5"); + await provider.chat(messages); + expect(JSON.parse(fetch.mock.calls[0][1]!.body as string)).toMatchObject({ + model: "claude-haiku-5-5", + output_config: { effort: "low" }, + }); + }); it("reports the serving model of a sticky fallback turn", async () => { const fetch = vi.spyOn(globalThis, "fetch"); fetch.mockResolvedValueOnce( diff --git a/test/convex/providerApiKeys/providerApiKeys.test.ts b/test/convex/providerApiKeys/providerApiKeys.test.ts index 5e5eb79..e4a2dcd 100644 --- a/test/convex/providerApiKeys/providerApiKeys.test.ts +++ b/test/convex/providerApiKeys/providerApiKeys.test.ts @@ -124,6 +124,7 @@ describe("provider key management", () => { provider: "anthropic", last4: ANTHROPIC_KEY.slice(-4), status: "active", + modelId: "claude-opus-5-5", }); const [url, init] = fetch.mock.calls[0]; expect(String(url)).toContain("/v1/models/"); @@ -348,6 +349,142 @@ describe("provider key management", () => { }); }); +describe("provider key model choice", () => { + it("checks the chosen model, stores it and lists it", async () => { + const t = convexTest(schema, modules); + const { as } = await signedIn(t); + const fetch = vi + .spyOn(globalThis, "fetch") + .mockResolvedValue(new Response("{}", { status: 200 })); + await as.action(api.providerApiKeys.action.save, { + provider: "anthropic", + apiKey: ANTHROPIC_KEY, + }); + // The default model is checked first. + expect(String(fetch.mock.calls[0][0])).toContain( + "/v1/models/claude-opus-5-5", + ); + expect( + (await as.query(api.providerApiKeys.query.list, {}))[0], + ).toMatchObject({ modelId: "claude-opus-5-5", effort: "medium" }); + + await expect( + as.action(api.providerApiKeys.action.setModel, { + provider: "anthropic", + modelId: "claude-sonnet-5-5", + effort: "high", + }), + ).resolves.toEqual({ + provider: "anthropic", + modelId: "claude-sonnet-5-5", + effort: "high", + }); + expect(String(fetch.mock.calls[1][0])).toContain( + "/v1/models/claude-sonnet-5-5", + ); + expect( + (await as.query(api.providerApiKeys.query.list, {}))[0], + ).toMatchObject({ + modelId: "claude-sonnet-5-5", + name: "Claude Sonnet 5.5", + effort: "high", + }); + // Changing only the effort needs no new check. + await as.action(api.providerApiKeys.action.setModel, { + provider: "anthropic", + modelId: "claude-sonnet-5-5", + effort: "low", + }); + expect(fetch).toHaveBeenCalledTimes(2); + // A re-saved key is checked against the chosen model. + await as.action(api.providerApiKeys.action.save, { + provider: "anthropic", + apiKey: ANTHROPIC_KEY, + }); + expect(String(fetch.mock.calls[2][0])).toContain( + "/v1/models/claude-sonnet-5-5", + ); + // A new key without access to the chosen model returns to the default. + fetch.mockResolvedValueOnce(new Response("{}", { status: 404 })); + await expect( + as.action(api.providerApiKeys.action.save, { + provider: "anthropic", + apiKey: ANTHROPIC_KEY, + }), + ).resolves.toMatchObject({ modelId: "claude-opus-5-5" }); + expect(String(fetch.mock.calls[4][0])).toContain( + "/v1/models/claude-opus-5-5", + ); + expect( + (await as.query(api.providerApiKeys.query.list, {}))[0], + ).toMatchObject({ modelId: "claude-opus-5-5", effort: "medium" }); + }); + + it("reports a server encryption problem on a model change as unavailable checks", async () => { + const t = convexTest(schema, modules); + const { as } = await signedIn(t); + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response("{}", { status: 200 }), + ); + await as.action(api.providerApiKeys.action.save, { + provider: "anthropic", + apiKey: ANTHROPIC_KEY, + }); + process.env.BYOK_ENCRYPTION_KEY = btoa("short"); + await expect( + as.action(api.providerApiKeys.action.setModel, { + provider: "anthropic", + modelId: "claude-sonnet-5-5", + }), + ).rejects.toMatchObject({ data: { reason: "not_configured" } }); + }); + + it("refuses models that are not offered, missing keys and models the key cannot use", async () => { + const t = convexTest(schema, modules); + const { as } = await signedIn(t); + const fetch = vi.spyOn(globalThis, "fetch"); + await expect( + as.action(api.providerApiKeys.action.setModel, { + provider: "openai", + modelId: "gpt-6.1-sol", + }), + ).rejects.toThrow("Save an API key"); + fetch.mockResolvedValue(new Response("{}", { status: 200 })); + await as.action(api.providerApiKeys.action.save, { + provider: "openai", + apiKey: OPENAI_KEY, + }); + await expect( + as.action(api.providerApiKeys.action.setModel, { + provider: "openai", + modelId: "gpt-4o", + }), + ).rejects.toThrow("Choose one of"); + await expect( + as.action(api.providerApiKeys.action.setModel, { + provider: "openai", + modelId: "gpt-6.1-sol", + effort: "max", + }), + ).rejects.toThrow("Choose one of"); + const { as: claude } = await signedIn(t); + await claude.action(api.providerApiKeys.action.save, { + provider: "anthropic", + apiKey: ANTHROPIC_KEY, + }); + fetch.mockResolvedValueOnce(new Response("{}", { status: 404 })); + await expect( + claude.action(api.providerApiKeys.action.setModel, { + provider: "anthropic", + modelId: "claude-haiku-5-5", + }), + ).rejects.toThrow("no access to Claude Haiku 5.5"); + expect( + (await claude.query(api.providerApiKeys.query.list, {}))[0], + ).toMatchObject({ modelId: "claude-opus-5-5" }); + }); +}); + describe("provider key failure notices", () => { it("notifies once per status change, not once per failed request", async () => { const t = convexTest(schema, modules);