diff --git a/CLAUDE.md b/CLAUDE.md index b1f9a4c..1d6caa3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,7 +9,7 @@ **Gitdog** is an AI-powered GitHub issue and discussion assistant. - Repository-aware issue and discussion formatting, answers, and maintainer tagging -- Claude Sonnet 5.5 generation with GPT-6.1 Sol fallback (see [docs/ai-models.md](docs/ai-models.md)) +- Claude Opus 5.5 generation with GPT-6.1 Sol fallback (see [docs/ai-models.md](docs/ai-models.md)) - Keyword repository index plus AI repository tools (search, read file) for code search - Private workspace per user - Free and Pro token quotas with optional prepaid token packs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7c416ea..2996476 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,13 +14,24 @@ organized, how to run it locally and what a pull request needs. | `e2e/` | Opt-in end-to-end runners against a development backend | | `docs/` | Conventions, GitHub App setup and AI model notes | +## Tech stack + +| Layer | Technology | +| -------- | -------------------------------------------------------------------- | +| Frontend | React 19, TypeScript, Vite, Tailwind CSS, react-i18next (en, ko, ja) | +| Backend | Convex | +| AI | Claude Opus 5.5 with GPT-6.1 Sol fallback; users' own keys optional | +| GitHub | GitHub App (OAuth, webhooks, repository access) | +| Payments | Lemon Squeezy | +| Hosting | Firebase Hosting | + ## How it works ```text GitHub webhook ─▶ Convex ─▶ repository index (keyword search) │ ▼ - Claude Sonnet 5.5 (GPT-6.1 Sol fallback) + Claude Opus 5.5 (GPT-6.1 Sol fallback) │ ▼ draft reply or review ─▶ approval ─▶ GitHub comment / PR review @@ -43,6 +54,18 @@ bun run dev # web app and Convex together A development GitHub App is required for repository features; follow [docs/github-app-setup.md](docs/github-app-setup.md). +## Commands + +| Command | Purpose | +| ---------------------- | ----------------------------------------------- | +| `bun run dev` | Run the web app and Convex dev backend together | +| `bun run typecheck` | Type-check the web app, backend and e2e runners | +| `bun run lint` | ESLint across the monorepo | +| `bun run format:check` | Prettier check | +| `bun run test` | Web and backend test suites | +| `bun run test:cli` | CLI test suite | +| `bun run check:api` | Verify the CLI API catalog matches the backend | + ## Checks Run these before opening a pull request: @@ -67,6 +90,14 @@ reports drift). For CLI changes, run `bun run typecheck:cli && bun run test:cli` - Follow [docs/convention.md](docs/convention.md) and [CLAUDE.md](CLAUDE.md) for code style. +## Documentation + +- [AI models and pricing](docs/ai-models.md): routing, quota weights and usage + accounting +- [GitHub App setup](docs/github-app-setup.md) +- [Code conventions](docs/convention.md) +- [CLI reference](packages/cli/docs/REFERENCE.md) + ## Security Please do not open public issues for security problems. Email diff --git a/README.md b/README.md index c9d937f..0386451 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,8 @@ ## Why Gitdog Maintainers answer the same questions, triage vague reports and review routine -changes every day. Gitdog indexes your repository and drafts the work for you — -with the actual code as evidence, not generic advice. +changes every day. Gitdog indexes your repository and drafts the work for you, +with the actual code as evidence instead of generic advice. | | What it does | | ------------------------ | --------------------------------------------------------------------------------------- | @@ -54,6 +54,20 @@ with the actual code as evidence, not generic advice. | **Your own AI keys** | Run generation on your Anthropic or OpenAI key, encrypted at rest and never shown again | | **Team workspace** | Share projects with members and keep per-project settings and usage in one place | +## Get started + +**On the web**: sign in with GitHub at [gitdog.dev](https://gitdog.dev), install +the GitHub App and connect a repository. + +**From the terminal**: the [`gitdog` CLI](packages/cli) manages projects, +settings, approvals and pull request reviews. + +```sh +npx gitdog@latest login +npx gitdog@latest init owner/repo +npx gitdog@latest tickets pending +``` + ## How it works ```mermaid @@ -67,21 +81,21 @@ flowchart LR ``` 1. **Connect** a repository through the Gitdog GitHub App. -2. **Index** — Gitdog builds a searchable index of your code and keeps it in - sync with the default branch. -3. **Draft** — new issues, discussions and pull requests get a reply or review +2. **Index**: Gitdog builds a searchable index of your code and keeps it in sync + with the default branch. +3. **Draft**: new issues, discussions and pull requests get a reply or review grounded in the indexed source. -4. **Approve** — review the draft in Gitdog or the CLI, then publish it. A pull - request review with no findings needs no approval: it posts at once and, when - every changed file was reviewed and no Gitdog thread is open, approves the - reviewed commit. A later push, including updating the branch, withdraws that - approval until `@gitdogbot review` reviews the new commits. +4. **Approve**: review the draft in Gitdog or the CLI, then publish it. + +A pull request review with no findings needs no approval. It posts at once and, +when every changed file was reviewed and no Gitdog thread is open, approves the +reviewed commit. A later push, including updating the branch, withdraws that +approval until the new commits are reviewed. -## Commands +## Pull request commands Gitdog reviews a pull request once, when it is opened or marked ready for -review. After that, the pull request author or a collaborator asks for more in -a pull request comment, or from the CLI: +review. After that, ask for more in a pull request comment or from the CLI: | Pull request comment | CLI | What it does | | ------------------------ | ------------------------------------------- | ------------------------------------------------ | @@ -90,90 +104,35 @@ a pull request comment, or from the CLI: | `@gitdogbot resolve` | `gitdog reviews resolve ` | Resolves every Gitdog review thread | | `@gitdogbot help` | `gitdog reviews --help` | Lists the commands | -A command is a comment line that starts with the mention, or follows another -review bot's command on the same line (`@coderabbitai review @gitdogbot -review`), in the pull request conversation; replies inside a review thread are -not read. `@gitdog` works as shorthand, but it also notifies the GitHub user -of that name, so `@gitdogbot` is preferred. While a review runs, Gitdog keeps -one status comment on the pull request up to date: reviewing, ready for -approval, reviewed, or why it could not finish. Collaborators can run -every command; the pull request author, who may work from a fork, can run all -but a full review. A review that failed or ran out of tokens can be requested -again, and pull requests opened by bots such as Dependabot are not reviewed. +- **Where**: a comment line in the pull request conversation that starts with + the mention, or follows another review bot's command on the same line + (`@coderabbitai review @gitdogbot review`). Replies inside a review thread + are not read. +- **Who**: collaborators can run every command. The pull request author, who + may work from a fork, can run all but `full review`. +- **`@gitdog`** works as shorthand, but it also notifies the GitHub user of that + name, so `@gitdogbot` is preferred. +- **Status**: while a review runs, one status comment on the pull request shows + whether it is reviewing, waiting for approval, reviewed, or why it could not + finish. A review that failed or ran out of tokens can be requested again. For + automatic reviews, Gitdog posts at most one failure comment a day per pull + request. +- **Limits**: someone who sends more than 10 commands an hour or 40 a day to + one repository is paused there for 24 hours. Pull requests opened by bots + such as Dependabot are not reviewed. In an issue or discussion, mention `@gitdogbot` with a question to get a reply -grounded in the repository. When a review cannot finish, for example because the account ran out of tokens, its status comment says why; for automatic reviews Gitdog posts at most one such comment a day per pull request. Someone who sends more than 10 commands -an hour or 40 a day to one repository is paused there for 24 hours. - -## Get started - -**On the web** — sign in with GitHub at [gitdog.dev](https://gitdog.dev), -install the GitHub App and connect a repository. Try Gitdog with 100K one-time -trial tokens, then use your own Anthropic or OpenAI API key or Pro; no credit -card required. - -**From the terminal** — the [`gitdog` CLI](packages/cli) manages projects, -settings, approvals and pull request reviews: - -```sh -npx gitdog@latest login -npx gitdog@latest init owner/repo -npx gitdog@latest tickets pending -``` - -## Repository layout - -This is a monorepo managed with bun workspaces. - -| Path | Description | -| ------------------------------- | -------------------------------------------------------------------------- | -| [`src/`](src) | Web app — React 19, TypeScript, Tailwind CSS | -| [`convex/`](convex) | Backend — Convex functions, GitHub App, AI workflows and billing | -| [`packages/cli/`](packages/cli) | The `gitdog` CLI, published to [npm](https://www.npmjs.com/package/gitdog) | -| [`docs/`](docs) | Conventions, GitHub App setup and AI model notes | +grounded in the repository. -## Development +## Plans -Requires Node.js 26 (`nvm use`) and Bun 1.4.4. +Start with 100K one-time trial tokens, no credit card required. After that, use +Pro (2.4M tokens a month), prepaid token packs, or your own Anthropic or OpenAI +API key. Gitdog runs on Claude Opus 5.5, and each Opus token counts as 2 Gitdog +tokens. Work on your own key never uses Gitdog tokens. See +[AI models and pricing](docs/ai-models.md). -```sh -bun install -cp .env.example .env.local -bun run dev -``` - -| Command | Purpose | -| ------------------- | ----------------------------------------------- | -| `bun run dev` | Run the web app and Convex dev backend together | -| `bun run typecheck` | Type-check the web app, backend and e2e runners | -| `bun run lint` | ESLint across the monorepo | -| `bun run test` | Web and backend test suites | -| `bun run test:cli` | CLI test suite | -| `bun run check:api` | Verify the CLI API catalog matches the backend | - -See [CONTRIBUTING.md](CONTRIBUTING.md) before opening a pull request. - -## Tech stack - -| Layer | Technology | -| -------- | -------------------------------------------------------------------- | -| Frontend | React 19, TypeScript, Vite, Tailwind CSS, react-i18next (en, ko, ja) | -| Backend | Convex | -| AI | Claude Sonnet 5.5 with GPT-6.1 Sol fallback; your own key optional | -| GitHub | GitHub App (OAuth, webhooks, repository access) | -| Payments | Lemon Squeezy | -| Hosting | Firebase Hosting | - -## Documentation - -- [AI models and pricing](docs/ai-models.md) -- [GitHub App setup](docs/github-app-setup.md) -- [Code conventions](docs/convention.md) -- [CLI reference](packages/cli/docs/REFERENCE.md) - -## Security - -### Your Anthropic and OpenAI keys +## Your Anthropic and OpenAI keys If you bring your own provider key, this is exactly how Gitdog handles it: @@ -203,7 +162,18 @@ If you bring your own provider key, this is exactly how Gitdog handles it: not cap it. Anyone who can open issues or pull requests on a connected public repository can trigger generation, so set a spend limit with the provider. -### Reporting +## Documentation + +- [CLI reference](packages/cli/docs/REFERENCE.md) +- [AI models and pricing](docs/ai-models.md) +- [GitHub App setup](docs/github-app-setup.md) + +## Contributing + +Development setup, repository layout and checks are in +[CONTRIBUTING.md](CONTRIBUTING.md). + +## Security Please report security issues privately to [hyo@hyo.dev](mailto:hyo@hyo.dev) instead of opening a public issue. diff --git a/convex/_ai/chatBilling.ts b/convex/_ai/chatBilling.ts index 2635048..6049688 100644 --- a/convex/_ai/chatBilling.ts +++ b/convex/_ai/chatBilling.ts @@ -1,4 +1,5 @@ import { internal } from "../_generated/api"; +import { RESERVATION_QUOTA_WEIGHT } from "./config"; import { chatRequestBytes } from "./requestSize"; import { ChatProviderRequestError } from "./providerErrors"; import { CreditFallbackProvider } from "./providers"; @@ -189,6 +190,10 @@ function parseChatResultCheckpoint( (typeof value.providerRequestId !== "string" || !value.providerRequestId || value.providerRequestId.length > 512)) || + (value.servedModelId !== undefined && + (typeof value.servedModelId !== "string" || + !value.servedModelId || + value.servedModelId.length > 128)) || (value.historyContent !== undefined && !hasValidCheckpointHistory(value.historyContent)) || (value.toolUse !== undefined && !hasValidCheckpointToolUse(value.toolUse)) @@ -199,7 +204,8 @@ function parseChatResultCheckpoint( expectedOutcome && (value.usage.promptTokens !== expectedOutcome.inputTokens || value.usage.completionTokens !== expectedOutcome.outputTokens || - value.providerRequestId !== expectedOutcome.providerRequestId) + value.providerRequestId !== expectedOutcome.providerRequestId || + value.servedModelId !== expectedOutcome.servedModel) ) { throw new Error("AI chat result checkpoint usage mismatch"); } @@ -295,6 +301,7 @@ type ChatSettlementOutcome = inputTokens: number; outputTokens: number; providerRequestId?: string; + servedModel?: string; } | { kind: "known_not_billed"; providerRequestId?: string }; @@ -304,6 +311,7 @@ const chatSettlementOutcomeValidator = v.union( inputTokens: v.number(), outputTokens: v.number(), providerRequestId: v.optional(v.string()), + servedModel: v.optional(v.string()), }), v.object({ kind: v.literal("known_not_billed"), @@ -321,7 +329,8 @@ function settlementOutcomesMatch( (left.kind === "known_not_billed" || (right.kind === "known_billed" && left.inputTokens === right.inputTokens && - left.outputTokens === right.outputTokens)) + left.outputTokens === right.outputTokens && + left.servedModel === right.servedModel)) ); } @@ -719,7 +728,7 @@ export const reserveChatUsage = internalMutation({ ) { throw new Error("AI chat reservation scope is invalid"); } - const reservedTokens = checkedReservationTokens( + const reservationLimit = checkedReservationTokens( args.inputTokenLimit, args.outputTokenLimit, ); @@ -737,6 +746,15 @@ export const reserveChatUsage = internalMutation({ ) .first(), ]); + // A reservation keeps the weight it was made with, so a retry after the + // weights change still matches it. + const quotaWeight = existing + ? (existing.quotaWeight ?? 1) + : RESERVATION_QUOTA_WEIGHT; + const reservedTokens = reservationLimit * quotaWeight; + if (!Number.isSafeInteger(reservedTokens)) { + throw new Error("AI chat reservation exceeds the supported range"); + } if (existing) { const sameProvider = existing.model === args.model && @@ -944,6 +962,7 @@ export const reserveChatUsage = internalMutation({ requirePendingTicketAction: args.requirePendingTicketAction, ticketActionProposalVersion: action.proposalVersion, ...quota.snapshot, + ...(quotaWeight !== 1 ? { quotaWeight } : {}), reservedTokens, status: "pending", expiresAt: now + CHAT_RESERVATION_TTL_MS, @@ -1218,6 +1237,7 @@ export const settleStagedChatOutcome = internalMutation({ inputTokens: outcome.inputTokens, outputTokens: outcome.outputTokens, model: reservation.model, + ...(outcome.servedModel ? { servedModel: outcome.servedModel } : {}), provider: reservation.provider ?? "google", reservationKey: reservation.reservationKey, reservationAttemptToken: reservation.attemptToken, @@ -2791,6 +2811,9 @@ export function createMeteredProvider( inputTokens: result.usage.promptTokens, outputTokens: result.usage.completionTokens, providerRequestId, + ...(result.servedModelId + ? { servedModel: result.servedModelId } + : {}), }; const resultCheckpointJson = serializeChatResultCheckpoint( result, diff --git a/convex/_ai/config.ts b/convex/_ai/config.ts index 130fb90..f03ee1a 100644 --- a/convex/_ai/config.ts +++ b/convex/_ai/config.ts @@ -22,20 +22,108 @@ export const CLAUDE_API_ENDS_AT = Date.parse("2027-01-17T15:00:00Z"); export const GENERATION_MODELS = { anthropic: { - modelId: "claude-sonnet-5-5", - name: "Claude Sonnet 5.5", - effort: "high", - inputPer1M: 2, - outputPer1M: 10, + modelId: "claude-opus-5-5", + name: "Claude Opus 5.5", + effort: "medium", }, openai: { modelId: "gpt-6.1-sol", name: "GPT-6.1 Sol", - effort: "xhigh", + effort: "high", + }, +} as const; + +/** + * Prices (USD per 1M tokens) and quota weights for every chat model Gitdog + * has billed. Retired models stay so historical usage rows remain priceable. + * + * A quota weight is how many quota tokens one provider token costs on + * Gitdog's key: 1 for the $2/$10 tier, 2 for a model that costs twice as + * much. Usage on the owner's own key is billed by the provider and is never + * weighted. + */ +export const CHAT_MODELS: Record< + string, + { + provider: "anthropic" | "openai" | "google"; + inputPer1M: number; + outputPer1M: number; + quotaWeight: number; + // Input above the threshold bills the whole request at the multipliers. + longContext?: { + inputTokens: number; + inputMultiplier: number; + outputMultiplier: number; + }; + } +> = { + "claude-opus-5-5": { + provider: "anthropic", + inputPer1M: 4, + outputPer1M: 20, + quotaWeight: 2, + }, + // Refusal fallback targets for Opus 5.5, priced only for recorded cost. + "claude-opus-5": { + provider: "anthropic", + inputPer1M: 5, + outputPer1M: 25, + quotaWeight: 2.5, + }, + "claude-opus-4-8": { + provider: "anthropic", + inputPer1M: 5, + outputPer1M: 25, + quotaWeight: 2.5, + }, + "claude-sonnet-5-5": { + provider: "anthropic", inputPer1M: 2, outputPer1M: 10, + quotaWeight: 1, }, -} as const; + "gpt-6.1-sol": { + provider: "openai", + inputPer1M: 2, + outputPer1M: 10, + quotaWeight: 1, + longContext: { + inputTokens: 272_000, + inputMultiplier: 2, + outputMultiplier: 1.5, + }, + }, + [LEGACY_GOOGLE_MODELS.chatModelId]: { + provider: LEGACY_GOOGLE_MODELS.provider, + inputPer1M: LEGACY_GOOGLE_MODELS.chatInputPer1M, + outputPer1M: LEGACY_GOOGLE_MODELS.chatOutputPer1M, + quotaWeight: 1, + }, +}; + +export function getChatModel(model: string, provider: string) { + const entry = Object.hasOwn(CHAT_MODELS, model) + ? CHAT_MODELS[model] + : undefined; + if (!entry || entry.provider !== provider) + throw new Error( + `Unsupported chat pricing: ${provider}/${model}. Add explicit pricing before enabling it.`, + ); + return entry; +} + +/** + * Quota weight held by every chat reservation: the highest weight among the + * models a Gitdog request may be routed to, so a reservation stays valid + * whichever provider serves it. + */ +export const RESERVATION_QUOTA_WEIGHT = Math.max( + ...Object.values(GENERATION_MODELS).map((model) => { + const entry = CHAT_MODELS[model.modelId]; + if (!entry) throw new Error(`${model.modelId} has no chat pricing`); + return entry.quotaWeight; + }), +); // ============================================ // TOKEN PRICING (per 1M tokens in USD) @@ -52,10 +140,10 @@ export const TOKEN_PRICING = { export const TOKEN_LIMITS = { // One-time trial credit granted on sign-in (stored as prepaid tokens). - // Enough for several issue and discussion replies and small pull request - // reviews. Large PRs are reviewed in batches that each hold - // REVIEW_BATCH_TOKEN_ESTIMATE up front, so a trial alone may not start one; - // the owner's own key has no such limit. + // Enough for a few issue and discussion replies and small pull request + // reviews; Opus 5.5 counts twice. Large PRs are reviewed in batches that + // each hold REVIEW_BATCH_TOKEN_ESTIMATE (200K at Opus's weight) up front, + // so a trial alone cannot start one; the owner's own key has no such limit. TRIAL_TOKENS: 100_000, // Pro tier: monthly allowance plus prepaid token packs diff --git a/convex/_ai/learning/extractLearning.ts b/convex/_ai/learning/extractLearning.ts index 8bd6440..19bc2c3 100644 --- a/convex/_ai/learning/extractLearning.ts +++ b/convex/_ai/learning/extractLearning.ts @@ -41,7 +41,8 @@ async function summarizeLinkContent( await beforeProviderCall?.(); const result = await provider.chat([{ role: "user", content: prompt }], { - maxTokens: 500, + // Thinking is always on and counts toward the limit. + maxTokens: 2000, temperature: 0.3, billingLabel, }); @@ -122,7 +123,8 @@ export async function extractLearning( await options.beforeProviderCall?.(); const result = await provider.chat([{ role: "user", content: prompt }], { - maxTokens: 500, + // Thinking is always on and counts toward the limit. + maxTokens: 2000, temperature: 0.3, billingLabel: "learning-extract", }); diff --git a/convex/_ai/model.ts b/convex/_ai/model.ts index f53994f..8d94d93 100644 --- a/convex/_ai/model.ts +++ b/convex/_ai/model.ts @@ -96,6 +96,9 @@ export type ChatCompletionResult = { }; provider?: string; modelId?: string; + // The model that actually served the request when a refusal fallback + // replaced modelId. Billing stays bound to modelId; cost uses this one. + servedModelId?: string; providerId?: string; providerRequestId?: string; /** Provider-native parts needed to replay stateless tool conversations. */ diff --git a/convex/_ai/providers.ts b/convex/_ai/providers.ts index e36dc16..4ced7d9 100644 --- a/convex/_ai/providers.ts +++ b/convex/_ai/providers.ts @@ -340,6 +340,16 @@ export class ClaudeProvider implements AiProvider { : "provider_error"; if (finishReason === "tool_use" && toolUse.length === 0) throw new Error("Missing tool call"); + // After a fallback, later turns of the conversation are served by the + // fallback model directly, with no fallback block, so read the model + // that answered rather than only the block. + const servedModelId = + typeof data.model === "string" && + data.model && + data.model !== this.modelId && + data.model.length <= 128 + ? data.model + : undefined; return { content: history .flatMap((p) => (p.type === "text" ? [p.text] : [])) @@ -350,6 +360,7 @@ export class ClaudeProvider implements AiProvider { usage: billed, provider: this.name, modelId: this.modelId, + ...(servedModelId ? { servedModelId } : {}), providerId: this.providerId, providerRequestId: requestId, }; diff --git a/convex/aiAuth/internalAction.ts b/convex/aiAuth/internalAction.ts index 3bafeea..704f9ce 100644 --- a/convex/aiAuth/internalAction.ts +++ b/convex/aiAuth/internalAction.ts @@ -55,9 +55,8 @@ export const checkClaudeGeneration = internalAction({ ); const result = await provider.chat( [{ role: "user", content: "Reply OK." }], - { - maxTokens: 512, - }, + // Thinking is always on and counts toward the limit. + { maxTokens: 2048 }, ); if (!result.usage) throw new Error("Claude generation QA did not return usage"); diff --git a/convex/pullRequests/github.ts b/convex/pullRequests/github.ts index 4901427..e8e4d5b 100644 --- a/convex/pullRequests/github.ts +++ b/convex/pullRequests/github.ts @@ -1,3 +1,4 @@ +import { RESERVATION_QUOTA_WEIGHT } from "../_ai/config"; import { fetchGitHubAppJsonWithRetry } from "../github/helpers"; import type { VerifiedRepositoryAccess } from "../github/repositoryIdentity"; import type { ChangedFile } from "./review"; @@ -263,8 +264,9 @@ export async function changedSince( } } -// Production reviews of up to 60 files used a median of about 90k tokens. -export const REVIEW_BATCH_TOKEN_ESTIMATE = 100_000; +// Production reviews of up to 60 files used a median of about 90k provider +// tokens. The hold is in quota tokens, so it scales with the model's weight. +export const REVIEW_BATCH_TOKEN_ESTIMATE = 100_000 * RESERVATION_QUOTA_WEIGHT; export function minimumReviewBatches(changedFiles: number): number { return Math.ceil(changedFiles / REVIEW_BATCH_FILES); diff --git a/convex/schema.ts b/convex/schema.ts index 0d1e26f..0f68929 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -246,7 +246,11 @@ const applicationTables = { // Token counts inputTokens: v.number(), outputTokens: v.number(), + // Quota tokens: provider tokens times quotaWeight for chat, or the + // cost-equivalent amount for embeddings. totalTokens: v.number(), + // Set on chat rows whose model counts more than once per provider token. + quotaWeight: v.optional(v.number()), embeddingTokens: v.optional(v.number()), embeddingBillingTier: v.optional( v.union(v.literal("free"), v.literal("paid")), @@ -273,6 +277,9 @@ const applicationTables = { // Metadata model: v.string(), // Chat or embedding model identifier + // Set when a refusal fallback served the request instead of model. Cost + // is priced at this model when Gitdog knows its price. + servedModel: v.optional(v.string()), provider: v.union( v.literal("google"), v.literal("anthropic"), @@ -491,6 +498,9 @@ const applicationTables = { keySource: v.optional(v.literal("user")), inputTokenLimit: v.number(), outputTokenLimit: v.number(), + // reservedTokens is (inputTokenLimit + outputTokenLimit) * quotaWeight. + // Missing on reservations made before weights, which hold a weight of 1. + quotaWeight: v.optional(v.number()), reservedTokens: v.number(), subscriptionId: v.optional(v.id("subscriptions")), entitlementPlan: v.union(v.literal("free"), v.literal("pro")), @@ -516,6 +526,7 @@ const applicationTables = { inputTokens: v.number(), outputTokens: v.number(), providerRequestId: v.optional(v.string()), + servedModel: v.optional(v.string()), }), v.object({ kind: v.literal("known_not_billed"), diff --git a/convex/subscriptions/query.ts b/convex/subscriptions/query.ts index f231247..36f8955 100644 --- a/convex/subscriptions/query.ts +++ b/convex/subscriptions/query.ts @@ -2,7 +2,12 @@ const MAX_WALLET_ROWS = 100; import { query } from "../_generated/server"; import { getAuthUserId } from "@convex-dev/auth/server"; -import { PRICING, CLAUDE_API_ENDS_AT, GENERATION_MODELS } from "../_ai/config"; +import { + PRICING, + CHAT_MODELS, + CLAUDE_API_ENDS_AT, + GENERATION_MODELS, +} from "../_ai/config"; import { getSafeQuotaTokenLimit, hasValidPurchasedTokenDebitCoverage, @@ -388,8 +393,9 @@ export const getPricingInfo = query({ }, }, tokenPricing: { - inputPer1M: generation.inputPer1M, - outputPer1M: generation.outputPer1M, + inputPer1M: CHAT_MODELS[generation.modelId].inputPer1M, + outputPer1M: CHAT_MODELS[generation.modelId].outputPer1M, + quotaWeight: CHAT_MODELS[generation.modelId].quotaWeight, model: generation.name, }, }; diff --git a/convex/tokenUsage/internalMutation.ts b/convex/tokenUsage/internalMutation.ts index 96671d6..7fc18fd 100644 --- a/convex/tokenUsage/internalMutation.ts +++ b/convex/tokenUsage/internalMutation.ts @@ -5,7 +5,8 @@ import type { Doc, Id } from "../_generated/dataModel"; import type { MutationCtx } from "../_generated/server"; import { v } from "convex/values"; import { - GENERATION_MODELS, + CHAT_MODELS, + getChatModel, LEGACY_GOOGLE_MODELS, PRICING, } from "../_ai/config"; @@ -210,6 +211,7 @@ type ChatUsageWriteArgs = { inputTokens: number; outputTokens: number; model: string; + servedModel?: string; provider: "google" | "anthropic" | "openai"; reservationKey?: string; reservationAttemptToken?: string; @@ -262,36 +264,6 @@ function assertFiniteNonNegativeCosts( } } -/** - * Get per-1M-token pricing based on model - */ -export function getModelPricing( - model: string, - provider: string, -): { - inputPer1M: number; - outputPer1M: number; -} { - if (provider === "anthropic" || provider === "openai") { - const pricing = GENERATION_MODELS[provider]; - if (model !== pricing.modelId) - throw new Error(`Unsupported ${provider} model pricing: ${model}`); - return { inputPer1M: pricing.inputPer1M, outputPer1M: pricing.outputPer1M }; - } - if ( - provider !== LEGACY_GOOGLE_MODELS.provider || - model !== LEGACY_GOOGLE_MODELS.chatModelId - ) { - throw new Error( - `Unsupported chat pricing: ${provider}/${model}. Add explicit pricing before enabling it.`, - ); - } - return { - inputPer1M: LEGACY_GOOGLE_MODELS.chatInputPer1M, - outputPer1M: LEGACY_GOOGLE_MODELS.chatOutputPer1M, - }; -} - /** Calculate uncached generation costs in cents, including context tiers. */ export function calculateChatCost( inputTokens: number, @@ -306,20 +278,23 @@ export function calculateChatCost( serviceFeeCents: number; totalCostCents: number; } { - const pricing = getModelPricing(model, provider); - const longContext = provider === "openai" && inputTokens > 272_000; + const pricing = getChatModel(model, provider); + const longContext = + pricing.longContext && inputTokens > pricing.longContext.inputTokens + ? pricing.longContext + : undefined; // Preserve fractional cents per request. Rounding each small request to whole // cents makes repeated low-token calls disappear from cost accounting. const inputCostCents = (inputTokens / 1_000_000) * pricing.inputPer1M * - (longContext ? 2 : 1) * + (longContext?.inputMultiplier ?? 1) * 100; const outputCostCents = (outputTokens / 1_000_000) * pricing.outputPer1M * - (longContext ? 1.5 : 1) * + (longContext?.outputMultiplier ?? 1) * 100; const baseCostCents = inputCostCents + outputCostCents; const serviceFeeCents = baseCostCents * serviceFeeRate; @@ -471,6 +446,18 @@ export async function recordChatUsageInTransaction( } } const userKey = reservation?.keySource === "user"; + // A reservation made before quota weights holds a weight of 1, so its + // usage is never charged more than it reserved. + const quotaWeight = userKey + ? 1 + : Math.min( + getChatModel(args.model, args.provider).quotaWeight, + reservation ? (reservation.quotaWeight ?? 1) : Infinity, + ); + const quotaTokens = Math.ceil(totalTokens * quotaWeight); + if (!Number.isSafeInteger(quotaTokens)) { + throw new Error("Total token count exceeds the supported range"); + } const requestedAction = args.ticketActionId ? await ctx.db.get(args.ticketActionId) @@ -502,7 +489,7 @@ export async function recordChatUsageInTransaction( subscription: reservation?.subscriptionId ? await ctx.db.get(reservation.subscriptionId) : await getSubscriptionForEntitlement(ctx, args.userId, now), - totalTokens, + totalTokens: quotaTokens, now, ...(reservation ? { @@ -512,11 +499,19 @@ export async function recordChatUsageInTransaction( : {}), }); const serviceFeeRate = accounting.entitlement.serviceFeeRate; + // A refusal fallback bills at the serving model's rates. An unknown served + // model is still recorded, priced at the requested model. + const pricedModel = + args.servedModel && + Object.hasOwn(CHAT_MODELS, args.servedModel) && + CHAT_MODELS[args.servedModel].provider === args.provider + ? args.servedModel + : args.model; const costs = calculateChatCost( args.inputTokens, args.outputTokens, serviceFeeRate, - args.model, + pricedModel, args.provider, ); assertFiniteNonNegativeCosts(costs); @@ -536,9 +531,13 @@ export async function recordChatUsageInTransaction( usageType: "chat", inputTokens: args.inputTokens, outputTokens: args.outputTokens, - totalTokens, + totalTokens: quotaTokens, + ...(quotaWeight !== 1 ? { quotaWeight } : {}), ...costs, model: args.model, + ...(args.servedModel && args.servedModel !== args.model + ? { servedModel: args.servedModel } + : {}), provider: args.provider, serviceFeeRate, chatReservationKey: reservationKey, diff --git a/docs/ai-models.md b/docs/ai-models.md index e11cf3e..4dd3b1a 100644 --- a/docs/ai-models.md +++ b/docs/ai-models.md @@ -7,8 +7,10 @@ and CLI never store provider keys. ## Generation routing -- Primary: **Claude Sonnet 5.5** (`claude-sonnet-5-5`). -- Fallback: **GPT-6.1 Sol** (`gpt-6.1-sol`) through the OpenAI Responses API. +- Primary: **Claude Opus 5.5** (`claude-opus-5-5`) at `medium` effort, with + Anthropic's server-side refusal fallback. +- Fallback: **GPT-6.1 Sol** (`gpt-6.1-sol`) at `high` reasoning effort through + the OpenAI Responses API. - Claude is used strictly before **2027-01-18 00:00 Asia/Seoul** (`2027-01-17T15:00:00Z`); after that, new requests go to OpenAI. - Only confirmed credit exhaustion or a provider spending limit selects OpenAI. @@ -79,10 +81,27 @@ indexing costs no AI tokens. ## Token pricing -| Model | Input per 1M tokens | Output per 1M tokens | -| ----------------- | ------------------- | -------------------- | -| Claude Sonnet 5.5 | $2.00 | $10.00 | -| GPT-6.1 Sol | $2.00 | $10.00 | +| Model | Input per 1M tokens | Output per 1M tokens | Quota weight | +| ----------------- | ------------------- | -------------------- | ------------ | +| Claude Opus 5.5 | $4.00 | $20.00 | 2 | +| GPT-6.1 Sol | $2.00 | $10.00 | 1 | +| Claude Sonnet 5.5 | $2.00 | $10.00 | 1 (retired) | + +On Gitdog's keys, each provider token counts as its model's quota weight in +Gitdog tokens: an Opus 5.5 call with 30K input and 5K output tokens uses 70K. +Every reservation holds the highest weight a request could be routed to, so a +call that falls back to Sol holds 2x but is charged 1x. A call is charged the +lower of its model's weight and its reservation's, so a reservation made before +weights existed is charged at most 1x. Usage on your own key is billed by the +provider and is never weighted. + +A request Opus 5.5 declines may be served inside the same call by Anthropic's +refusal fallback model (Claude Opus 5 or Opus 4.8, $5/$25), and later turns +of that conversation go to the fallback model directly for a while. Whenever +the response names another serving model, the usage row keeps +`claude-opus-5-5` and its weight, since the reservation is bound to it, stores +the serving model as `servedModel`, and records its cost at that model's +price. Sol prompts above 272K input tokens use 2x input and 1.5x output pricing. Prompt caching is disabled, so recorded token counts map directly to the listed @@ -121,6 +140,9 @@ Token usage is recorded per request from the provider's own usage report: - Input includes the reported prompt and tool-use prompt tokens. - Output is the remaining provider-reported total. - The actual provider and model are stored with every usage record. +- `totalTokens` is the quota amount: provider tokens times the weight charged, + which is stored as `quotaWeight` when it is not 1. Input and output stay + raw provider counts, so on Opus rows they add up to half of `totalTokens`. - Usage consumes included tokens first, then the prepaid balance. ## Configuration diff --git a/e2e/runPrReview.ts b/e2e/runPrReview.ts index aab7013..c152126 100644 --- a/e2e/runPrReview.ts +++ b/e2e/runPrReview.ts @@ -160,6 +160,8 @@ try { inputTokens: number; outputTokens: number; totalTokens: number; + quotaWeight?: number; + servedModel?: string; baseCostCents: number; serviceFeeCents: number; totalCostCents: number; @@ -169,13 +171,26 @@ try { assert(["anthropic", "openai"].includes(row.provider)); assert.equal( row.model, - row.provider === "anthropic" ? "claude-sonnet-5-5" : "gpt-6.1-sol", + row.provider === "anthropic" ? "claude-opus-5-5" : "gpt-6.1-sol", ); - assert.equal(row.totalTokens, row.inputTokens + row.outputTokens); + // Opus 5.5 counts twice against quota on Gitdog's key. + assert.equal( + row.totalTokens, + (row.inputTokens + row.outputTokens) * (row.quotaWeight ?? 1), + ); + // Per-1M prices; a refusal fallback is priced at the model that served it. + const prices: Record = { + "claude-opus-5-5": [4, 20], + "claude-opus-5": [5, 25], + "claude-opus-4-8": [5, 25], + "gpt-6.1-sol": [2, 10], + }; + const [inputPer1M, outputPer1M] = + prices[row.servedModel ?? row.model] ?? prices[row.model]; const long = row.provider === "openai" && row.inputTokens > 272000; const expected = - (row.inputTokens / 1000000) * 2 * (long ? 2 : 1) * 100 + - (row.outputTokens / 1000000) * 10 * (long ? 1.5 : 1) * 100; + (row.inputTokens / 1000000) * inputPer1M * (long ? 2 : 1) * 100 + + (row.outputTokens / 1000000) * outputPer1M * (long ? 1.5 : 1) * 100; assert( Math.abs(row.baseCostCents - expected) < 1e-8, "Provider usage price mismatch", diff --git a/packages/cli/docs/REFERENCE.md b/packages/cli/docs/REFERENCE.md index 5c4f243..eae955a 100644 --- a/packages/cli/docs/REFERENCE.md +++ b/packages/cli/docs/REFERENCE.md @@ -240,7 +240,10 @@ Approval executes the approved action on GitHub. Project deletion disconnects the Gitdog project and removes cached Gitdog data; the GitHub repository is preserved. Team features and ownership restrictions follow the web app's plan and access rules. Checkout returns a secure link; purchasing requires completing -the provider's checkout. Account eligibility and policy consent are completed in +the provider's checkout. In `usage summary`, `totalTokens` is the quota used: +Claude Opus 5.5 counts each provider token twice, so it is larger than +`totalInputTokens` plus `totalOutputTokens`, which are raw provider counts. +Account eligibility and policy consent are completed in the browser using `account consent`, or through an explicitly selected public API operation. Consent is never inferred from login. diff --git a/src/i18n/locales/en.json b/src/i18n/locales/en.json index 03db164..7378622 100644 --- a/src/i18n/locales/en.json +++ b/src/i18n/locales/en.json @@ -768,7 +768,7 @@ "purchase": "Purchase" }, "tokenPricing": "Token Pricing", - "tokenPricingDescription": "Free starts with 100k one-time trial tokens and Pro includes 2.4M each month. Both stop at the available limit, which prepaid token packs can extend. Work that runs on your own API key never uses Gitdog tokens.", + "tokenPricingDescription": "Free starts with 100k one-time trial tokens and Pro includes 2.4M each month. Both stop at the available limit, which prepaid token packs can extend. Gitdog runs on Claude Opus 5.5, and each Opus token counts as 2 Gitdog tokens because it costs twice as much as the GPT-6.1 Sol fallback. Work that runs on your own API key never uses Gitdog tokens.", "freeTokensLabel": "Trial tokens (one-time)", "platformFeeLabel": "Prepaid token packs", "billingLabel": "Billing", diff --git a/src/i18n/locales/ja.json b/src/i18n/locales/ja.json index 4e8e580..a82846b 100644 --- a/src/i18n/locales/ja.json +++ b/src/i18n/locales/ja.json @@ -767,7 +767,7 @@ "purchase": "購入" }, "tokenPricing": "トークン価格", - "tokenPricingDescription": "Freeは1回限りの100k体験トークンから始まり、Proには毎月2.4Mトークンが含まれます。どちらも利用可能な上限で停止し、プリペイドトークンパックで追加できます。自分のAPIキーで実行される処理はGitdogのトークンを使いません。", + "tokenPricingDescription": "Freeは1回限りの100k体験トークンから始まり、Proには毎月2.4Mトークンが含まれます。どちらも利用可能な上限で停止し、プリペイドトークンパックで追加できます。GitdogはClaude Opus 5.5で動作し、OpusのトークンはフォールバックのGPT-6.1 Solの2倍のコストがかかるため、1トークンをGitdogトークン2つとして数えます。自分のAPIキーで実行される処理はGitdogのトークンを使いません。", "freeTokensLabel": "体験トークン(1回限り)", "platformFeeLabel": "プリペイドトークンパック", "billingLabel": "課金", diff --git a/src/i18n/locales/ko.json b/src/i18n/locales/ko.json index 54868d1..b256fae 100644 --- a/src/i18n/locales/ko.json +++ b/src/i18n/locales/ko.json @@ -767,7 +767,7 @@ "purchase": "구매" }, "tokenPricing": "토큰 가격", - "tokenPricingDescription": "Free는 1회 100k 체험 토큰으로 시작하고, Pro는 매월 2.4M 토큰을 포함합니다. 두 플랜 모두 사용 가능한 한도에서 중지되며 선불 토큰 팩으로 늘릴 수 있습니다. 내 API 키로 실행되는 작업은 Gitdog 토큰을 쓰지 않습니다.", + "tokenPricingDescription": "Free는 1회 100k 체험 토큰으로 시작하고, Pro는 매월 2.4M 토큰을 포함합니다. 두 플랜 모두 사용 가능한 한도에서 중지되며 선불 토큰 팩으로 늘릴 수 있습니다. Gitdog은 Claude Opus 5.5로 실행되며, Opus 토큰은 GPT-6.1 Sol 대체 모델보다 비용이 두 배라 1개당 Gitdog 토큰 2개로 계산됩니다. 내 API 키로 실행되는 작업은 Gitdog 토큰을 쓰지 않습니다.", "freeTokensLabel": "체험 토큰 (1회)", "platformFeeLabel": "선불 토큰 팩", "billingLabel": "결제", diff --git a/test/convex/ai/federation.test.ts b/test/convex/ai/federation.test.ts index c2c9fdd..c59a34b 100644 --- a/test/convex/ai/federation.test.ts +++ b/test/convex/ai/federation.test.ts @@ -392,7 +392,7 @@ describe("deployment-admin authentication diagnostic", () => { expect(result).toEqual({ ok: true, inputTokens: 9, outputTokens: 2 }); const request = JSON.parse(fetch.mock.calls[1][1]!.body as string); expect(request).toMatchObject({ - max_tokens: 512, + max_tokens: 2048, messages: [ { role: "user", content: [{ type: "text", text: "Reply OK." }] }, ], diff --git a/test/convex/ai/legacyGoogleUsage.test.ts b/test/convex/ai/legacyGoogleUsage.test.ts index 8807e01..38282d8 100644 --- a/test/convex/ai/legacyGoogleUsage.test.ts +++ b/test/convex/ai/legacyGoogleUsage.test.ts @@ -5,7 +5,10 @@ import { internal } from "../../../convex/_generated/api"; import type { Id } from "../../../convex/_generated/dataModel"; import schema from "../../../convex/schema"; import { hasOpenUsageReservation } from "../../../convex/_ai/chatBilling"; -import { LEGACY_GOOGLE_MODELS } from "../../../convex/_ai/config"; +import { + LEGACY_GOOGLE_MODELS, + RESERVATION_QUOTA_WEIGHT, +} from "../../../convex/_ai/config"; import { getDefaultAiModel } from "../../../convex/_ai/model"; import { CURRENT_PRIVACY_POLICY_VERSION, @@ -193,7 +196,10 @@ describe("legacy Google usage", () => { q.eq("reservationKey", old.reservationKey), ) .unique(); - expect(row).toMatchObject({ status: "ambiguous", reservedTokens: 2000 }); + expect(row).toMatchObject({ + status: "ambiguous", + reservedTokens: 2000 * RESERVATION_QUOTA_WEIGHT, + }); expect(row?.usageId).toBeUndefined(); expect(row?.reconciliation).toBeUndefined(); expect(await hasOpenUsageReservation(ctx, f.userId)).toBe(true); @@ -202,7 +208,10 @@ describe("legacy Google usage", () => { it("keeps the full retired Google ceiling reserved against future quota", async () => { const t = convexTest(schema, modules); const f = await insertFixture(t); - await seedHold(t, argsFor(f, "large-old-hold", "google", 49950)); + await seedHold( + t, + argsFor(f, "large-old-hold", "google", 49950 / RESERVATION_QUOTA_WEIGHT), + ); expect( await t.mutation( internal._ai.chatBilling.reserveChatUsage, diff --git a/test/convex/ai/providerFallback.test.ts b/test/convex/ai/providerFallback.test.ts index d852c81..a65b8fe 100644 --- a/test/convex/ai/providerFallback.test.ts +++ b/test/convex/ai/providerFallback.test.ts @@ -76,7 +76,7 @@ describe("credit fallback and exclusive Claude deadline", () => { expect(claudeBody.messages[0].content[1].text.endsWith(source)).toBe(true); expect(openaiBody.input[1].content.endsWith(source)).toBe(true); }); - it("requests schema-constrained JSON for both providers without lowering review effort", async () => { + it("requests schema-constrained JSON for both providers at the configured review effort", async () => { const responseFormat = { name: "review_result", schema: { @@ -95,12 +95,12 @@ describe("credit fallback and exclusive Claude deadline", () => { expect(JSON.parse(fetch.mock.calls[0][1]!.body as string)).toMatchObject({ thinking: { type: "adaptive" }, output_config: { - effort: "high", + effort: "medium", format: { type: "json_schema", schema: responseFormat.schema }, }, }); expect(JSON.parse(fetch.mock.calls[1][1]!.body as string)).toMatchObject({ - reasoning: { effort: "xhigh" }, + reasoning: { effort: "high" }, text: { format: { type: "json_schema", @@ -124,9 +124,9 @@ describe("credit fallback and exclusive Claude deadline", () => { ); expect(provider.providerId).toBe("anthropic"); expect(JSON.parse(fetch.mock.calls[0][1]!.body as string)).toMatchObject({ - model: "claude-sonnet-5-5", + model: "claude-opus-5-5", thinking: { type: "adaptive" }, - output_config: { effort: "high" }, + output_config: { effort: "medium" }, }); }); @@ -185,7 +185,7 @@ describe("credit fallback and exclusive Claude deadline", () => { expect(body).toMatchObject({ model: "gpt-6.1-sol", store: false, - reasoning: { effort: "xhigh" }, + reasoning: { effort: "high" }, include: ["reasoning.encrypted_content"], instructions: "Be precise", prompt_cache_options: { mode: "explicit", ttl: "30m" }, @@ -556,7 +556,7 @@ it.each([ }, ); -it("counts high-effort reasoning inside output usage exactly once", async () => { +it("counts reasoning-effort reasoning inside output usage exactly once", async () => { const fetch = vi.spyOn(globalThis, "fetch").mockResolvedValue( openai({ usage: { @@ -576,7 +576,7 @@ it("counts high-effort reasoning inside output usage exactly once", async () => totalTokens: 3100, }); expect(JSON.parse(fetch.mock.calls[0][1]!.body as string)).toMatchObject({ - reasoning: { effort: "xhigh" }, + reasoning: { effort: "high" }, max_output_tokens: 16384, }); }); @@ -594,12 +594,12 @@ describe("Claude safety refusals", () => { it("reads the fallback model's answer and bills the attempt that produced it", async () => { vi.spyOn(globalThis, "fetch").mockResolvedValueOnce( claude({ - model: "claude-sonnet-5", + model: "claude-opus-4-8", content: [ { type: "fallback", - from: { model: "claude-sonnet-5-5" }, - to: { model: "claude-sonnet-5" }, + from: { model: "claude-opus-5-5" }, + to: { model: "claude-opus-4-8" }, }, { type: "text", text: "Fallback answer" }, ], @@ -615,6 +615,10 @@ describe("Claude safety refusals", () => { ); const result = await new ClaudeProvider("fixture-key").chat(messages); expect(result.content).toBe("Fallback answer"); + expect(result).toMatchObject({ + modelId: "claude-opus-5-5", + servedModelId: "claude-opus-4-8", + }); expect(result.historyContent).toEqual([ { type: "text", text: "Fallback answer" }, ]); @@ -624,6 +628,27 @@ describe("Claude safety refusals", () => { totalTokens: 14, }); }); + it("reports the serving model of a sticky fallback turn", async () => { + const fetch = vi.spyOn(globalThis, "fetch"); + fetch.mockResolvedValueOnce( + claude({ + model: "claude-opus-4-8", + content: [{ type: "text", text: "Sticky answer" }], + }), + ); + expect( + await new ClaudeProvider("fixture-key").chat(messages), + ).toMatchObject({ servedModelId: "claude-opus-4-8" }); + fetch.mockResolvedValueOnce( + claude({ + model: "claude-opus-5-5", + content: [{ type: "text", text: "Primary answer" }], + }), + ); + expect( + await new ClaudeProvider("fixture-key").chat(messages), + ).not.toHaveProperty("servedModelId"); + }); it("reports a decline with its category instead of a processing failure", async () => { vi.spyOn(globalThis, "fetch").mockResolvedValueOnce( claude({ @@ -676,8 +701,8 @@ describe("Claude safety refusals", () => { { type: "text", text: "partial" }, { type: "fallback", - from: { model: "claude-sonnet-5-5" }, - to: { model: "claude-sonnet-5" }, + from: { model: "claude-opus-5-5" }, + to: { model: "claude-opus-4-8" }, }, { type: "text", text: "Served" }, ], diff --git a/test/convex/lemonSqueezy/billingSettlement.test.ts b/test/convex/lemonSqueezy/billingSettlement.test.ts index 9d4f7b0..01c7357 100644 --- a/test/convex/lemonSqueezy/billingSettlement.test.ts +++ b/test/convex/lemonSqueezy/billingSettlement.test.ts @@ -2582,10 +2582,11 @@ describe("authoritative subscription and invoice settlement", () => { requirePendingTicketAction: true, generationKey: "renewal-chat-generation-4203", operation: "renewal billing hold regression", - model: "claude-sonnet-5-5", + // Opus 5.5 holds and charges two quota tokens per provider token. + model: "claude-opus-5-5", provider: "anthropic" as const, requestFingerprint: "1".repeat(64), - inputTokenLimit: PRO_INCLUDED_TOKENS + 40, + inputTokenLimit: PRO_INCLUDED_TOKENS / 2 + 10, outputTokenLimit: 20, }; const safeChatReservation = { @@ -2593,7 +2594,7 @@ describe("authoritative subscription and invoice settlement", () => { reservationKey: "renewal-safe-chat-hold-4203", attemptToken: "renewal-safe-chat-hold-4203-attempt-1", generationKey: "renewal-safe-chat-generation-4203", - inputTokenLimit: 100, + inputTokenLimit: 30, }; await expect( t.mutation( @@ -2680,7 +2681,7 @@ describe("authoritative subscription and invoice settlement", () => { userId, repositoryId, ticketActionId, - inputTokens: PRO_INCLUDED_TOKENS + 40, + inputTokens: PRO_INCLUDED_TOKENS / 2 + 10, outputTokens: 20, model: chatReservation.model, provider: "anthropic", diff --git a/test/convex/pullRequests/lifecycle.test.ts b/test/convex/pullRequests/lifecycle.test.ts index b97b92d..3caf720 100644 --- a/test/convex/pullRequests/lifecycle.test.ts +++ b/test/convex/pullRequests/lifecycle.test.ts @@ -12,6 +12,7 @@ import { CURRENT_PRIVACY_POLICY_VERSION, } from "../../../convex/users/legal"; import { drainScheduledFunctions } from "../scheduling"; +import { REVIEW_BATCH_TOKEN_ESTIMATE } from "../../../convex/pullRequests/github"; import { PRICING, getFreeIncludedTokens } from "../../../convex/_ai/config"; import { encryptProviderKey } from "../../../convex/providerApiKeys/crypto"; const modules = import.meta.glob("../../../convex/**/*.ts"); @@ -2751,6 +2752,9 @@ describe("explicit verification of failed reviews", () => { }); describe("batched PR reviews", () => { + // Holds are in quota tokens, so they scale with the model weight. + const BATCH = REVIEW_BATCH_TOKEN_ESTIMATE; + const tokens = (n: number) => n.toLocaleString("en-US"); async function batchFixture( failedPage?: number | number[], largeFiles = 0, @@ -3020,7 +3024,7 @@ describe("batched PR reviews", () => { expect(await f.getAction()).toMatchObject({ status: "pending", reviewBatch: { page: 1, group: 1, claimed: false }, - reviewQuotaHold: 200_000, + reviewQuotaHold: 2 * BATCH, }); await f.generate(1, 1); await drainScheduledFunctions(f.t); @@ -3075,7 +3079,7 @@ describe("batched PR reviews", () => { status: "pending", reviewBatch: { page: 1, group: 1, claimed: false }, // Two groups left on this page plus three pages for the other 45 files. - reviewQuotaHold: 500_000, + reviewQuotaHold: 5 * BATCH, }); expect(f.batches.map((key) => key.split("/batch/")[1])).toEqual(["1/1"]); await f.generate(1, 1); @@ -3151,7 +3155,7 @@ describe("batched PR reviews", () => { }); expect(action?.reviewFailureDetails).toContain("Stage · Token quota check"); expect(action?.reviewFailureDetails).toContain( - "at least 4 more review batches, estimated at 400,000 tokens; 100,000 tokens remain.", + `at least 4 more review batches, estimated at ${tokens(4 * BATCH)} tokens; 100,000 tokens remain.`, ); expect(f.batches).toEqual([]); expect(f.fetch).not.toHaveBeenCalled(); @@ -3206,11 +3210,11 @@ describe("batched PR reviews", () => { await f.generate(); expect(await f.getAction()).toMatchObject({ reviewQuotaUserId: f.userId, - reviewQuotaHold: 300_000, + reviewQuotaHold: 3 * BATCH, reviewBatch: { page: 2 }, }); await f.generate(2); - expect((await f.getAction())?.reviewQuotaHold).toBe(200_000); + expect((await f.getAction())?.reviewQuotaHold).toBe(2 * BATCH); }); it.each([ @@ -3219,7 +3223,7 @@ describe("batched PR reviews", () => { ] as const)( "accounts for a %s batched review that holds tokens", async (status, refused) => { - const f = await batchFixture(undefined, 0, 400_000); + const f = await batchFixture(undefined, 0, 4 * BATCH); const ticketId = (await f.getAction())!.ticketId; const holderId = await f.t.run((ctx) => ctx.db.insert("ticketActions", { @@ -3232,7 +3236,7 @@ describe("batched PR reviews", () => { chatHistory: [], createdAt: 1, reviewQuotaUserId: f.userId, - reviewQuotaHold: 400_000, + reviewQuotaHold: 4 * BATCH, }), ); await f.generate(); @@ -3240,7 +3244,7 @@ describe("batched PR reviews", () => { if (refused) { expect(action?.status).toBe("failed"); expect(action?.reviewFailureDetails).toContain( - "100,000 tokens remain after 400,000 held by other reviews in progress.", + `100,000 tokens remain after ${tokens(4 * BATCH)} held by other reviews in progress.`, ); expect(f.batches).toEqual([]); } else { @@ -3257,18 +3261,23 @@ describe("batched PR reviews", () => { ); it("reserves for the first page's real groups before any model call", async () => { - const f = await batchFixture(undefined, 2, 300_000); + const f = await batchFixture(undefined, 2, 3 * BATCH); await f.generate(); const action = await f.getAction(); expect(action?.status).toBe("failed"); expect(action?.reviewFailureDetails).toContain( - "at least 6 more review batches, estimated at 600,000 tokens; 400,000 tokens remain.", + `at least 6 more review batches, estimated at ${tokens(6 * BATCH)} tokens; ${tokens(3 * BATCH + 100_000)} tokens remain.`, ); expect(f.batches).toEqual([]); }); it("stops before spending more when a later page needs more than the hold", async () => { - const f = await batchFixture(undefined, 0, 300_000, [32, 33, 34]); + const f = await batchFixture( + undefined, + 0, + 4 * BATCH - 100_000, + [32, 33, 34], + ); await f.generate(); const ticketId = (await f.getAction())!.ticketId; await f.t.run((ctx) => @@ -3281,7 +3290,7 @@ describe("batched PR reviews", () => { chatHistory: [], createdAt: 1, reviewQuotaUserId: f.userId, - reviewQuotaHold: 200_000, + reviewQuotaHold: 2 * BATCH, }), ); await f.generate(2); diff --git a/test/convex/subscriptions/tokenLimits.test.ts b/test/convex/subscriptions/tokenLimits.test.ts index e95bdfe..b3a8988 100644 --- a/test/convex/subscriptions/tokenLimits.test.ts +++ b/test/convex/subscriptions/tokenLimits.test.ts @@ -1,10 +1,12 @@ import { describe, it, expect, beforeEach } from "vitest"; import { - GENERATION_MODELS, + CHAT_MODELS, PRICING, LEGACY_FREE_MONTHLY_TOKENS, } from "../../../convex/_ai/config"; +const SONNET = CHAT_MODELS["claude-sonnet-5-5"]; + /** * Token Limit Tests * @@ -489,8 +491,8 @@ describe("Cost Calculation", () => { const inputTokens = 10_000; const outputTokens = 2_000; const totalCost = - (inputTokens / 1_000_000) * GENERATION_MODELS.anthropic.inputPer1M + - (outputTokens / 1_000_000) * GENERATION_MODELS.anthropic.outputPer1M; + (inputTokens / 1_000_000) * SONNET.inputPer1M + + (outputTokens / 1_000_000) * SONNET.outputPer1M; const platformFee = totalCost * PRICING.PLATFORM_FEE_FREE; const totalWithFee = totalCost + platformFee; @@ -503,8 +505,8 @@ describe("Cost Calculation", () => { const inputTokens = 80_000; const outputTokens = 20_000; const totalCost = - (inputTokens / 1_000_000) * GENERATION_MODELS.anthropic.inputPer1M + - (outputTokens / 1_000_000) * GENERATION_MODELS.anthropic.outputPer1M; + (inputTokens / 1_000_000) * SONNET.inputPer1M + + (outputTokens / 1_000_000) * SONNET.outputPer1M; const platformFee = totalCost * PRICING.PLATFORM_FEE_PRO; const totalWithFee = totalCost + platformFee; @@ -518,8 +520,8 @@ describe("Cost Calculation", () => { const inputTokens = 800_000; const outputTokens = 200_000; const totalCost = - (inputTokens / 1_000_000) * GENERATION_MODELS.anthropic.inputPer1M + - (outputTokens / 1_000_000) * GENERATION_MODELS.anthropic.outputPer1M; + (inputTokens / 1_000_000) * SONNET.inputPer1M + + (outputTokens / 1_000_000) * SONNET.outputPer1M; const platformFee = totalCost * PRICING.PLATFORM_FEE_TOKEN_PACK_SMALL; const totalWithFee = totalCost + platformFee; diff --git a/test/convex/subscriptions/trialCredit.test.ts b/test/convex/subscriptions/trialCredit.test.ts index 6cb2d71..f29453e 100644 --- a/test/convex/subscriptions/trialCredit.test.ts +++ b/test/convex/subscriptions/trialCredit.test.ts @@ -330,7 +330,7 @@ describe("trial grant rules", () => { ctx.db.query("tokenUsage").collect(), ); expect(usage).toEqual([ - expect.objectContaining({ totalTokens: 15, purchasedTokensDebited: 0 }), + expect.objectContaining({ totalTokens: 30, purchasedTokensDebited: 0 }), ]); }); }); diff --git a/test/convex/tokenUsage/chatUsage.test.ts b/test/convex/tokenUsage/chatUsage.test.ts index 783b181..69593e0 100644 --- a/test/convex/tokenUsage/chatUsage.test.ts +++ b/test/convex/tokenUsage/chatUsage.test.ts @@ -588,9 +588,10 @@ describe("AI chat usage reservations", () => { }); }); + // Each reserved token holds RESERVATION_QUOTA_WEIGHT quota tokens. const chat = reservationArgs(fixture, "pro-prepaid-chat", 1, { - inputTokenLimit: 40, - outputTokenLimit: 20, + inputTokenLimit: 20, + outputTokenLimit: 10, }); await expect(reserve(t, chat)).resolves.toEqual({ allowed: true, @@ -604,8 +605,8 @@ describe("AI chat usage reservations", () => { reservedTokens: 40, }); const fill = reservationArgs(fixture, "pro-prepaid-chat-fill", 1, { - inputTokenLimit: 20, - outputTokenLimit: 20, + inputTokenLimit: 10, + outputTokenLimit: 10, }); await expect(reserve(t, fill)).resolves.toMatchObject({ allowed: true }); @@ -836,12 +837,12 @@ describe("AI chat usage reservations", () => { const fixture = await insertFixture(t); const attempts = [ reservationArgs(fixture, "concurrent-a", 1, { - inputTokenLimit: 30_000, - outputTokenLimit: 30_000, + inputTokenLimit: 15_000, + outputTokenLimit: 15_000, }), reservationArgs(fixture, "concurrent-b", 1, { - inputTokenLimit: 30_000, - outputTokenLimit: 30_000, + inputTokenLimit: 15_000, + outputTokenLimit: 15_000, }), ]; const results = await Promise.all(attempts.map((args) => reserve(t, args))); @@ -851,6 +852,138 @@ describe("AI chat usage reservations", () => { ]); }); + it("holds and charges Opus usage at its quota weight", async () => { + const t = convexTest(schema, modules); + const args = reservationArgs(await insertFixture(t), "weighted"); + await expect(reserve(t, args)).resolves.toMatchObject({ allowed: true }); + await expect(markStarted(t, args)).resolves.toBe(true); + const usageId = await finalize(t, args, 100, 20); + const state = await t.run(async (ctx) => ({ + usage: await ctx.db.get(usageId), + reservation: await ctx.db + .query("chatUsageReservations") + .withIndex("by_reservation_key", (q) => + q.eq("reservationKey", args.reservationKey), + ) + .unique(), + })); + expect(state.reservation).toMatchObject({ + quotaWeight: 2, + reservedTokens: 8_000, + }); + expect(state.usage).toMatchObject({ + inputTokens: 100, + outputTokens: 20, + totalTokens: 240, + quotaWeight: 2, + }); + }); + + it.each([ + ["claude-opus-4-8", 0.15], + ["claude-future-model", 0.12], + ] as const)( + "prices a refusal fallback served by %s at its own rates", + async (servedModel, baseCostCents) => { + const t = convexTest(schema, modules); + const args = reservationArgs(await insertFixture(t), "served"); + await reserve(t, args); + await markStarted(t, args); + await t.mutation(internal._ai.chatBilling.stageChatProviderOutcome, { + reservationKey: args.reservationKey, + attemptToken: args.attemptToken, + userId: args.userId, + repositoryId: args.repositoryId, + ticketActionId: args.ticketActionId, + outcome: { + kind: "known_billed", + inputTokens: 100, + outputTokens: 40, + providerRequestId: "served-request", + servedModel, + }, + }); + await t.mutation(internal._ai.chatBilling.settleStagedChatOutcome, { + reservationKey: args.reservationKey, + attemptToken: args.attemptToken, + }); + const usage = await t.run((ctx) => ctx.db.query("tokenUsage").first()); + // Quota stays bound to the requested model's weight. + expect(usage).toMatchObject({ + model: "claude-opus-5-5", + totalTokens: 280, + quotaWeight: 2, + servedModel, + }); + // Opus 4.8 at $5/$25; an unknown model falls back to Opus 5.5's $4/$20. + expect(usage?.baseCostCents).toBeCloseTo(baseCostCents); + }, + ); + + it("charges a weighted reservation served by Sol at Sol's weight", async () => { + const t = convexTest(schema, modules); + const args = { + ...reservationArgs(await insertFixture(t), "sol-served"), + model: "gpt-6.1-sol", + provider: "openai" as const, + }; + await reserve(t, args); + await markStarted(t, args); + const usageId = await t.mutation( + internal.tokenUsage.internalMutation.recordUsage, + { + userId: args.userId, + repositoryId: args.repositoryId, + ticketActionId: args.ticketActionId, + inputTokens: 100, + outputTokens: 20, + model: args.model, + provider: "openai", + reservationKey: args.reservationKey, + reservationAttemptToken: args.attemptToken, + providerRequestId: "sol-request", + }, + ); + const state = await t.run(async (ctx) => ({ + usage: await ctx.db.get(usageId), + reservation: await ctx.db + .query("chatUsageReservations") + .withIndex("by_reservation_key", (q) => + q.eq("reservationKey", args.reservationKey), + ) + .unique(), + })); + expect(state.reservation).toMatchObject({ quotaWeight: 2 }); + expect(state.usage).toMatchObject({ totalTokens: 120 }); + expect(state.usage).not.toHaveProperty("quotaWeight"); + }); + + it("keeps a reservation made before weights at its original weight", async () => { + const t = convexTest(schema, modules); + const args = reservationArgs(await insertFixture(t), "pre-weight"); + await expect(reserve(t, args)).resolves.toMatchObject({ allowed: true }); + // A reservation written by the previous deployment has no weight. + await t.run(async (ctx) => { + const row = await ctx.db + .query("chatUsageReservations") + .withIndex("by_reservation_key", (q) => + q.eq("reservationKey", args.reservationKey), + ) + .unique(); + await ctx.db.patch(row!._id, { + quotaWeight: undefined, + reservedTokens: 4_000, + }); + }); + // A retry after the deploy still matches the stored reservation. + await expect(reserve(t, args)).resolves.toMatchObject({ allowed: true }); + await expect(markStarted(t, args)).resolves.toBe(true); + const usageId = await finalize(t, args, 100, 20); + const usage = await t.run((ctx) => ctx.db.get(usageId)); + expect(usage).toMatchObject({ totalTokens: 120 }); + expect(usage).not.toHaveProperty("quotaWeight"); + }); + it("finalizes one usage row idempotently and rejects payload changes", async () => { const t = convexTest(schema, modules); const args = reservationArgs(await insertFixture(t), "finalize-once"); @@ -2277,7 +2410,8 @@ describe("AI chat usage reservations", () => { })); expect(recovered.reservation).toMatchObject({ status: "finalized" }); expect(recovered.reservation?.settlementState).toBeUndefined(); - expect(recovered.usage).toMatchObject({ totalTokens: 15 }); + // Opus 5.5 counts twice against quota. + expect(recovered.usage).toMatchObject({ totalTokens: 30, quotaWeight: 2 }); }); it("periodically reschedules an exact outcome without letting expiry overwrite it", async () => { @@ -3089,7 +3223,10 @@ describe("metered multi-provider replay", () => { inputTokens: 10, outputTokens: 5, }); - expect(state.usage[0].baseCostCents).toBeCloseTo(0.007); + // Sol: 10 * $2 + 5 * $10 per 1M; Opus 5.5: twice that. + expect(state.usage[0].baseCostCents).toBeCloseTo( + scenario === "fallback-replenished" ? 0.007 : 0.014, + ); }, ); }); diff --git a/test/convex/tokenUsage/providerPricing.test.ts b/test/convex/tokenUsage/providerPricing.test.ts index 5624716..04b8626 100644 --- a/test/convex/tokenUsage/providerPricing.test.ts +++ b/test/convex/tokenUsage/providerPricing.test.ts @@ -1,9 +1,15 @@ import { describe, expect, it } from "vitest"; import { - calculateChatCost, - getModelPricing, -} from "../../../convex/tokenUsage/internalMutation"; + getChatModel, + RESERVATION_QUOTA_WEIGHT, +} from "../../../convex/_ai/config"; +import { calculateChatCost } from "../../../convex/tokenUsage/internalMutation"; + +function getModelPricing(model: string, provider: string) { + const { inputPer1M, outputPer1M } = getChatModel(model, provider); + return { inputPer1M, outputPer1M }; +} describe("provider-aware chat pricing", () => { it("uses the configured paid AI 2.5 Flash pricing", () => { @@ -47,6 +53,10 @@ describe("provider-aware chat pricing", () => { describe("Claude and OpenAI generation pricing", () => { it("prices both current providers and rejects cross-provider model attribution", () => { + expect(getModelPricing("claude-opus-5-5", "anthropic")).toEqual({ + inputPer1M: 4, + outputPer1M: 20, + }); expect(getModelPricing("claude-sonnet-5-5", "anthropic")).toEqual({ inputPer1M: 2, outputPer1M: 10, @@ -56,6 +66,20 @@ describe("Claude and OpenAI generation pricing", () => { outputPer1M: 10, }); expect(() => getModelPricing("claude-sonnet-5-5", "openai")).toThrow(); + expect(() => getModelPricing("toString", "anthropic")).toThrow( + "Unsupported chat pricing", + ); + }); + it("weighs Opus twice and reserves at the highest routed weight", () => { + expect(getChatModel("claude-opus-5-5", "anthropic").quotaWeight).toBe(2); + expect(getChatModel("gpt-6.1-sol", "openai").quotaWeight).toBe(1); + expect(RESERVATION_QUOTA_WEIGHT).toBe(2); + }); + it("does not apply long-context pricing to Claude", () => { + expect( + calculateChatCost(500_000, 0, 0, "claude-opus-5-5", "anthropic") + .baseCostCents, + ).toBeCloseTo(200); }); it("uses Sol long-context pricing only above 272K input tokens", () => { expect(