From e5b4d9863c62cfdd1269ce7b5967fd7239ea5f45 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 22 Oct 2026 11:16:08 +0000 Subject: [PATCH 1/3] Cap each AI feature's reply length and retry cut-off replies Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-debatelab/SKILL.md | 2 +- .../verify-debatelab/features/README.md | 4 +- .../verify-debatelab/features/ai-tools.md | 1 + .../features/case-generation.md | 1 + .../verify-debatelab/recipes/ai-tools.spec.ts | 20 ++- .../recipes/case-generation.spec.ts | 16 +++ docs/architecture.md | 2 +- e2e/full-stack/fake-openrouter.mjs | 31 +++-- e2e/full-stack/stack-core.ts | 6 +- src/lib/ai/streaming.ts | 12 +- src/locales/en.json | 1 + src/locales/es.json | 1 + src/locales/hu.json | 1 + supabase/README.md | 2 +- supabase/functions/_shared/model-catalog.ts | 1 - supabase/functions/_shared/output-caps.ts | 60 +++++++++ supabase/functions/ai-stream/index.ts | 29 ++++- supabase/functions/ai/features/ask.ts | 2 +- supabase/functions/ai/features/attack.ts | 2 +- supabase/functions/ai/features/brainstorm.ts | 2 +- supabase/functions/ai/features/crossfire.ts | 1 + supabase/functions/ai/features/factoids.ts | 2 +- supabase/functions/ai/features/generate.ts | 2 +- supabase/functions/ai/features/sparring.ts | 11 +- .../functions/ai/handler-research.test.ts | 8 +- supabase/functions/ai/index.ts | 7 +- supabase/functions/ai/provider.ts | 117 ++++++++++++------ supabase/functions/ai/stream.test.ts | 47 ++++++- 28 files changed, 314 insertions(+), 77 deletions(-) create mode 100644 supabase/functions/_shared/output-caps.ts diff --git a/.claude/skills/verify-debatelab/SKILL.md b/.claude/skills/verify-debatelab/SKILL.md index 28f399011..2baa3efcc 100644 --- a/.claude/skills/verify-debatelab/SKILL.md +++ b/.claude/skills/verify-debatelab/SKILL.md @@ -81,7 +81,7 @@ It also prints the commit the stack started at and whether there are uncommitted - The Current Debate helpers: `startDebate(page, motion)` starts one from the header, `chooseForMotion(page, "create" | "once")` answers the motion dialog, and `debateControl(page)` is the visible header control on either viewport. - The voice helpers: `stubSpeech(page)` installs a scripted microphone, recogniser and browser voice; `say(page, words)` speaks; `spokenByBrowser(page)` reads back what the browser voice said. - `admin`: a service-role client for reading rows as a second view. -- `fake` / `providerCall(feature)`: what the Edge Function actually sent upstream. Use `fake.fail(feature, status)` to force a provider error. +- `fake` / `providerCall(feature)`: what the Edge Function actually sent upstream. Use `fake.fail(feature, status)` to force a provider error and `fake.truncate(feature)` (fake mode only) to cut the next reply off at its length cap. Each call records the `maxTokens` it asked for. - `aiResponse(page, feature, operation?)`: the parsed body of the next `ai` Edge Function response. Pages ask for live generation, so most `ai` responses are event streams; `aiResponse` and `readAIResponse(response)` read the final `result` event (or a plain JSON body) for you, so never call `response.json()` on an `ai` response. Assert that its fields reach the screen, never on model wording, so a recipe passes in both modes. `LIVE` and `AI_TIMEOUT` (30 s fake, 240 s live) let a recipe branch where a check only makes sense live, and a fake-mode branch adds an `unverified` annotation instead of passing silently. - `expectStreamed([...])`: every provider call for those features asked the model to stream, which proves the page showed the answer live. - `expectProviderCalls([...])`: what reached the provider. Each test writes `provider-calls.json`, and in live mode the fixture fails the test if any chat call was answered by a model other than the pin. diff --git a/.claude/skills/verify-debatelab/features/README.md b/.claude/skills/verify-debatelab/features/README.md index 565c095c3..e4ce71338 100644 --- a/.claude/skills/verify-debatelab/features/README.md +++ b/.claude/skills/verify-debatelab/features/README.md @@ -40,9 +40,9 @@ Each feature file starts with an H1 title and one paragraph describing the user- ## Features - [Demo-code sign-in](./auth-demo-login.md): sign-in, reload, rejected code, sign-out (`e2e/full-stack/auth.spec.ts`), and email one-time-code sign-in through Mailpit (`recipes/auth.spec.ts`). -- [Case generation](./case-generation.md): generate, web research, save and reopen from History on a fresh device, PDF and DOCX export, single-section regeneration, provider failure and retry, quota block (`recipes/case-generation.spec.ts`). +- [Case generation](./case-generation.md): generate, web research, save and reopen from History on a fresh device, PDF and DOCX export, single-section regeneration, provider failure and retry, a reply cut off at its length cap, quota block (`recipes/case-generation.spec.ts`). - [Library sources](./library-sources.md): add, pause, delete, grounding, and URL import (`recipes/library.spec.ts`). -- [AI tools](./ai-tools.md): Attack, Brainstorm, Factoids, and Ask streamed, cited, and with web search (`recipes/ai-tools.spec.ts`). +- [AI tools](./ai-tools.md): Attack, Brainstorm, Factoids, and Ask streamed, cut off at its length cap, cited, and with web search (`recipes/ai-tools.spec.ts`). - [Live generation](./live-generation.md): every AI tool shows its answer while the model writes it, Cancel mid-stream, and the event stream's shape (`recipes/live-generation.spec.ts`); Crossfire and Sparring requests stream too (`recipes/practice.spec.ts`). - [Current Debate](./current-debate.md): start, the first-debate offer and the different-motion choice, sources and results scoped to a debate, switching, and the first onboarding step (`recipes/current-debate.spec.ts`); debate context review and pins (`e2e/full-stack/debate-context.spec.ts`). - [Model defaults](./model-defaults.md): Pro accounts start on Claude Opus 5.5, Basic accounts on Claude Haiku 5.5 with Pro models locked, and retired model IDs move to Opus 5.5 (`recipes/model-defaults.spec.ts`). diff --git a/.claude/skills/verify-debatelab/features/ai-tools.md b/.claude/skills/verify-debatelab/features/ai-tools.md index 28742eea5..b3f02c083 100644 --- a/.claude/skills/verify-debatelab/features/ai-tools.md +++ b/.claude/skills/verify-debatelab/features/ai-tools.md @@ -25,6 +25,7 @@ Preconditions: - **Brainstorm.** On `/app/brainstorm`, fill the field labelled `Motion / Topic *` and click `Start Brainstorming`. With no Current Debate, the dialog `Start a debate with this motion?` opens first: `chooseForMotion(page, "create")` (or `"once"`). The text `FAKE-PROVIDER idea 1` appears. `providerCall("brainstorm").user` contains `Motion: `, and the `brainstorms` row is filed in the new debate. With a Current Debate, the field is prefilled and no dialog opens unless the motion differs. - **Factoids.** On `/app/factoids`, fill the field labelled `Statement *` and click `Find Factoids`. The text `FAKE-PROVIDER search suggestion` appears. - **Ask, streamed.** On `/app/ask`, fill the placeholder `Ask your debate coach...` and click the button named `Send message`. The text `FAKE-PROVIDER streamed answer: ...` appears. `fake.requests()` features contain `ask-stream` and not `ask`. `ai_requests` gets `{feature: "ask", status: "success"}`. +- **Ask, cut off.** Call `await fake.truncate("ask-stream")` and send. The stream ends with `finish_reason: "length"`, its `final` event carries `meta.replyTruncated: true`, and the partial answer stays with `This answer hit its length limit and stops early...` under `Uncertainties:`. No `Response was interrupted` toast. Fake mode only. - **Ask, cited.** Open the `Options` button above the message box, click `#citation-mode`, close it with Escape, then send. The text `FAKE-PROVIDER answer` appears, and `providerCall("ask")` fired. - **Stop and errors.** While Ask answers, the send button becomes `Stop answering`; Stop keeps the partial reply and returns focus to the box, which stays editable throughout. A failed request leaves an `alert` above the box with `Retry`, which asks again without adding the question twice. The four form tools show one status line (`Reviewing your case…`, `Finding angles…`, `Searching for evidence…`, `Writing your case…`, or `Researching sources…` while web research runs) with a `Cancel` button until the model starts writing, then the live draft (see [live-generation](./live-generation.md)). Failures stay on the page with `Retry`. diff --git a/.claude/skills/verify-debatelab/features/case-generation.md b/.claude/skills/verify-debatelab/features/case-generation.md index aca27a4ff..1a57f2e3c 100644 --- a/.claude/skills/verify-debatelab/features/case-generation.md +++ b/.claude/skills/verify-debatelab/features/case-generation.md @@ -29,6 +29,7 @@ Preconditions: - **Filed in the debate.** `cases` for the user is one row whose `debates(motion)` is the motion. On a fresh browser context, `/app/history` shows it as a heading in the `Cases` tab panel, and `Load` reopens it. - **Save (one-off).** Generate with no debate and choose `chooseForMotion(page, "once")`. Click the button named `Save` (use `exact`), fill the dialog field labelled `Case title`, and click the dialog's `Save` button. `cases` for the user is `[{title, debate_id: null}]`. On a fresh browser context, `/app/history` shows `One-off and unsorted work` with a list item holding the title. - **Error.** Call `await fake.fail("generate", 500)`, start the debate, and generate. An `alert` holding `Generation Failed` stays above the results with a `Retry` button; clicking `Retry` succeeds and removes it. +- **Cut off.** Call `await fake.truncate("generate")`, start the debate, and generate. The first reply stops halfway with `finish_reason: "length"`; the Edge Function retries once with a larger cap, and the case appears with no error. `fake.requests()` has two `generate` entries, the second with a larger `maxTokens`. Fake mode only. - **Cancel.** While a generation runs, the results column shows `Writing your case…` and a `Cancel` button. Cancel stops the request and shows no error. - **Quota.** Update `user_settings` to set `daily_limit_override: 0` through `admin`, start the debate, then generate. The dialog shows `Daily Access Paused`. `fake.requests()` has no `generate` entry. - **Regenerate a section.** After generating, click the button named `Regenerate Arguments` (the section titles are `Case Structure`, `Arguments`, `Predicted Opponent Case`, `Prepared Counters`). Hold the request with `page.route("**/functions/v1/ai", ...)` to check the in-flight state: a `status` with `Regenerating Arguments` is visible and `Regenerate Case Structure` is disabled. The toast `Section regenerated` follows. A full generation shows no toast: the result appears in place (and scrolls into view on phones). diff --git a/.claude/skills/verify-debatelab/recipes/ai-tools.spec.ts b/.claude/skills/verify-debatelab/recipes/ai-tools.spec.ts index 907fcfb6d..9822c67e2 100644 --- a/.claude/skills/verify-debatelab/recipes/ai-tools.spec.ts +++ b/.claude/skills/verify-debatelab/recipes/ai-tools.spec.ts @@ -1,4 +1,4 @@ -import { admin, AI_TIMEOUT, aiResponse, chooseForMotion, expect, expectProviderCalls, LIVE, providerCalls, test } from "./fixtures"; +import { admin, AI_TIMEOUT, aiResponse, chooseForMotion, expect, expectProviderCalls, fake, LIVE, providerCalls, test } from "./fixtures"; // Feature map: features/ai-tools.md. @@ -91,6 +91,24 @@ test("ask-stream: Ask streams a coaching answer and records usage", async ({ sig }, { timeout: AI_TIMEOUT }).toContainEqual(expect.objectContaining({ feature: "ask", status: "success" })); }); +test("ask-cutoff: a streamed answer cut off at its length cap says so under the answer", async ({ signedIn: page, evidence }) => { + test.skip(LIVE, "only the fake provider can force a cut-off reply"); + await fake.truncate("ask-stream"); + await page.goto("/app/ask"); + await page.getByPlaceholder("Ask your debate coach...").fill("How should I weigh access against cost?"); + const stream = page.waitForResponse((response) => response.url().endsWith("/functions/v1/ai-stream"), { timeout: AI_TIMEOUT }); + await page.getByRole("button", { name: "Send message" }).click(); + const events = await (await stream).text(); + expect(events).toContain('"replyTruncated":true'); + // The partial answer stays, with the notice under it rather than an error. + await expect(page.getByText("weigh access against cost.").first()).toBeVisible(); + await expect(page.getByText("This answer hit its length limit and stops early.", { exact: false })).toBeVisible(); + await expect(page.getByText("Response was interrupted")).toHaveCount(0); + await evidence.shot("01-cut-off-notice"); + const features = (await providerCalls()).map((call) => call.feature); + expect(features).toEqual(["ask-stream"]); +}); + test("ask-cited: Ask in citation mode uses the structured endpoint", async ({ signedIn: page, evidence }) => { await page.goto("/app/ask"); // The Ask switches live in the Options popover above the message box. diff --git a/.claude/skills/verify-debatelab/recipes/case-generation.spec.ts b/.claude/skills/verify-debatelab/recipes/case-generation.spec.ts index c3f04f707..fa85e2f08 100644 --- a/.claude/skills/verify-debatelab/recipes/case-generation.spec.ts +++ b/.claude/skills/verify-debatelab/recipes/case-generation.spec.ts @@ -206,6 +206,22 @@ test("gen-error: a provider failure shows an error and the retry succeeds", asyn await evidence.shot("02-retried"); }); +test("gen-cutoff: a case cut off at its length cap is retried once with a larger cap and still appears", async ({ signedIn: page, evidence }) => { + test.skip(LIVE, "only the fake provider can force a cut-off reply"); + await fake.truncate("generate"); + await openGenerate(page); + const response = aiResponse(page, "generate"); + await page.getByRole("button", { name: "Generate Debate Case" }).click(); + const body = await response; + await expect(page.getByText((body.output as unknown as CaseOutput).caseStructure.thesis).first()).toBeVisible(); + await expect(page.getByRole("alert").filter({ hasText: "Generation Failed" })).toHaveCount(0); + const caps = (await fake.requests()).filter((request) => request.feature === "generate").map((request) => request.maxTokens ?? 0); + await evidence.json("01-generate-max-tokens", caps); + expect(caps).toHaveLength(2); + expect(caps[1]).toBeGreaterThan(caps[0]); + await evidence.shot("02-case-after-retry"); +}); + test("gen-quota: an account without quota is told why and the provider is never called", async ({ signedIn: page, user, evidence }) => { await admin.from("user_settings").update({ daily_limit_override: 0 }).eq("user_id", await user.userId()); await openGenerate(page); diff --git a/docs/architecture.md b/docs/architecture.md index 30b508c00..4d5962d1d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -132,7 +132,7 @@ Supabase Auth owns user sessions. Protected routes provide a client navigation b Checkout and portal functions create Stripe sessions after authenticated ownership and redirect validation. The webhook updates subscription state and records event processing for idempotency. AI handlers record the provider-reported usage and charge for each request. -All model requests use the single OpenRouter backend route and `OPENROUTER_API_KEY`. The shared ten-model catalog, its defaults (Claude Haiku 5.5 for Basic, Claude Opus 5.5 for Pro), Pro-only entries, retired-ID successors, and request controls live in [`supabase/functions/_shared/model-catalog.ts`](../supabase/functions/_shared/model-catalog.ts). OpenRouter usage remains raw in the audit row, and the reported charge is unknown when the provider does not return a complete cost. The opponent's voice is the one exception to the OpenRouter route: the `voice` function calls ElevenLabs with `ELEVENLABS_API_KEY`. ElevenLabs reports billed characters, not dollars, so [`supabase/functions/_shared/cost.ts`](../supabase/functions/_shared/cost.ts) prices those rows from a dated list-price snapshot; promotions and plan credits are not modelled, so a voice row can overstate the charge. +All model requests use the single OpenRouter backend route and `OPENROUTER_API_KEY`. The shared ten-model catalog, its defaults (Claude Haiku 5.5 for Basic, Claude Opus 5.5 for Pro), Pro-only entries, retired-ID successors, and request controls live in [`supabase/functions/_shared/model-catalog.ts`](../supabase/functions/_shared/model-catalog.ts). Each feature's reply length cap lives in [`supabase/functions/_shared/output-caps.ts`](../supabase/functions/_shared/output-caps.ts); `max_tokens` adds room for the reasoning effort on top. A JSON reply cut off at its cap is retried once with twice the budget, and a streamed Ask answer cut off there ends with a notice under it. OpenRouter usage remains raw in the audit row, and the reported charge is unknown when the provider does not return a complete cost. The opponent's voice is the one exception to the OpenRouter route: the `voice` function calls ElevenLabs with `ELEVENLABS_API_KEY`. ElevenLabs reports billed characters, not dollars, so [`supabase/functions/_shared/cost.ts`](../supabase/functions/_shared/cost.ts) prices those rows from a dated list-price snapshot; promotions and plan credits are not modelled, so a voice row can overstate the charge. `Auto Fast` and `Auto Pro` are virtual selections, not provider model IDs. The Edge Functions resolve them through TypeSafe Jev on OpenRouter before calling the concrete writing model. Auto Fast is available to Basic accounts, routes only among the Basic models (Claude Haiku 5.5, GPT-6 Luna, Muse Spark 1.3 Contributor), and normally resolves to Claude Haiku 5.5. Auto Pro remains Pro-only and normally resolves to Claude Opus 5.5. Provider calls, usage rows, and response metadata record the concrete resolved model, while response metadata also retains the requested automatic selection. A Jev router call uses the same OpenRouter key and does not consume a second daily quota unit. diff --git a/e2e/full-stack/fake-openrouter.mjs b/e2e/full-stack/fake-openrouter.mjs index 2c036e568..824b2d524 100644 --- a/e2e/full-stack/fake-openrouter.mjs +++ b/e2e/full-stack/fake-openrouter.mjs @@ -20,6 +20,8 @@ const requests = []; // Subscription state the fake Stripe returns, set by tests through /__subscription. const subscriptions = new Map(); let failures = []; +// Replies cut off at max_tokens (finish_reason "length"), set through /__truncate. +let truncations = []; let upstreamCalls = 0; // Delay between streamed chunks; a test can slow it to keep a draft on screen. let streamDelayMs; @@ -204,26 +206,26 @@ function json(res, status, value) { res.end(JSON.stringify(value)); } -function stream(res) { +function stream(res, finishReason = "stop") { res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); const chunks = [`${MARK} streamed answer: `, "weigh access ", "against cost."]; for (const content of chunks) { res.write(`data: ${JSON.stringify({ choices: [{ delta: { content } }] })}\n\n`); } - res.write(`data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: "stop" }] })}\n\n`); + res.write(`data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: finishReason }] })}\n\n`); res.write(`data: ${JSON.stringify({ choices: [], usage })}\n\n`); res.end("data: [DONE]\n\n"); } // Live generation: a structured answer streamed in small chunks, the way a // model writes tokens, so the client's partial rendering runs for real. -async function streamJson(res, content) { +async function streamJson(res, content, finishReason = "stop") { res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); for (let index = 0; index < content.length; index += STREAM_CHUNK_CHARS) { res.write(`data: ${JSON.stringify({ choices: [{ delta: { content: content.slice(index, index + STREAM_CHUNK_CHARS) } }] })}\n\n`); await new Promise((resolve) => setTimeout(resolve, streamDelayMs ?? STREAM_CHUNK_DELAY_MS)); } - res.write(`data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: "stop" }] })}\n\n`); + res.write(`data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: finishReason }] })}\n\n`); res.write(`data: ${JSON.stringify({ choices: [], usage })}\n\n`); res.end("data: [DONE]\n\n"); } @@ -238,7 +240,7 @@ async function proxy(req, res, path) { const body = JSON.parse(await readBody(req)); const chat = path === "/api/v1/chat/completions"; const record = chat - ? { ...classify(body), requestedModel: body.model, model: chatModel(body.model), stream: body.stream === true } + ? { ...classify(body), requestedModel: body.model, model: chatModel(body.model), stream: body.stream === true, maxTokens: body.max_tokens } : { feature: "jev", model: body.model }; delete record.output; requests.push(record); @@ -311,6 +313,7 @@ createServer(async (req, res) => { if (url.pathname === "/__reset" && req.method === "POST") { requests.length = 0; failures = []; + truncations = []; streamDelayMs = undefined; return json(res, 200, { ok: true }); } @@ -324,6 +327,11 @@ createServer(async (req, res) => { subscriptions.set(subscription.id, subscription); return json(res, 200, { ok: true }); } + if (url.pathname === "/__truncate" && req.method === "POST") { + const { feature, count = 1 } = JSON.parse(await readBody(req) || "{}"); + truncations.push({ feature, count }); + return json(res, 200, { ok: true }); + } if (url.pathname === "/__fail" && req.method === "POST") { const { feature, count = 1, status = 500 } = JSON.parse(await readBody(req) || "{}"); failures.push({ feature, count, status }); @@ -359,7 +367,7 @@ createServer(async (req, res) => { const body = JSON.parse(await readBody(req)); const { feature, system, user, output } = classify(body); - requests.push({ feature, model: body.model, stream: body.stream === true, system, user }); + requests.push({ feature, model: body.model, stream: body.stream === true, maxTokens: body.max_tokens, system, user }); if (!feature) return json(res, 500, { error: { message: "Fake OpenRouter does not recognize this prompt" } }); const failure = failures.find((item) => item.count > 0 && (!item.feature || item.feature === feature)); @@ -367,13 +375,18 @@ createServer(async (req, res) => { failure.count -= 1; return json(res, failure.status, { error: { message: `Fake ${feature} failure` } }); } - if (feature === "ask-stream") return stream(res); + // A cut-off reply stops halfway with finish_reason "length", as a model does at max_tokens. + const truncation = truncations.find((item) => item.count > 0 && (!item.feature || item.feature === feature)); + if (truncation) truncation.count -= 1; + const finishReason = truncation ? "length" : "stop"; + if (feature === "ask-stream") return stream(res, finishReason); if (output === null) return json(res, 500, { error: { message: `Fake ${feature} could not parse the request` } }); - if (body.stream === true) return streamJson(res, JSON.stringify(output)); + const content = truncation ? JSON.stringify(output).slice(0, Math.floor(JSON.stringify(output).length / 2)) : JSON.stringify(output); + if (body.stream === true) return streamJson(res, content, finishReason); return json(res, 200, { id: `fake-${requests.length}`, model: body.model, - choices: [{ index: 0, finish_reason: "stop", message: { role: "assistant", content: JSON.stringify(output) } }], + choices: [{ index: 0, finish_reason: finishReason, message: { role: "assistant", content } }], usage, }); }).listen(port, "0.0.0.0", () => console.log(`${upstream ? "live proxy" : "fake OpenRouter"} listening on ${port}`)); diff --git a/e2e/full-stack/stack-core.ts b/e2e/full-stack/stack-core.ts index a88a247ae..0b5480f24 100644 --- a/e2e/full-stack/stack-core.ts +++ b/e2e/full-stack/stack-core.ts @@ -22,7 +22,7 @@ export const admin = stackReady ? createClient(supabaseUrl!, serviceKey!, { auth: { persistSession: false } }) : (null as never); -export type FakeRequest = { feature: string | null; model?: string; stream?: boolean; system: string; user: string }; +export type FakeRequest = { feature: string | null; model?: string; stream?: boolean; maxTokens?: number; system: string; user: string }; export const fake = { async reset() { @@ -34,6 +34,10 @@ export const fake = { async fail(feature: string, status = 500, count = 1) { await fetch(`${fakeUrl}/__fail`, { method: "POST", body: JSON.stringify({ feature, status, count }) }); }, + /** Cuts the next `count` replies for a feature off at max_tokens (fake mode only). */ + async truncate(feature: string, count = 1) { + await fetch(`${fakeUrl}/__truncate`, { method: "POST", body: JSON.stringify({ feature, count }) }); + }, /** Sets what the fake Stripe returns for GET /v1/subscriptions/:id. */ async stripeSubscription(subscription: { id: string; status: string }) { await fetch(`${fakeUrl}/__subscription`, { method: "POST", body: JSON.stringify(subscription) }); diff --git a/src/lib/ai/streaming.ts b/src/lib/ai/streaming.ts index ec72f1a68..0c1213c65 100644 --- a/src/lib/ai/streaming.ts +++ b/src/lib/ai/streaming.ts @@ -1,3 +1,4 @@ +import i18n from '@/i18n'; import { getActiveAccessToken, supabase, SUPABASE_URL } from '@/lib/supabase/client'; import { captureProductEvent } from '@/lib/observability'; import { AIAuthError, AIResponseError, REQUEST_TIMEOUT_MS, TimeoutError, boundAskHistory, boundAskMessage, debateContextForRequest, mapAIError, refreshDebatesIfMissing, reportAIFailure, withOutputLanguage } from './client'; @@ -212,8 +213,15 @@ export async function streamAskChat( contextStore.recordUsage('ask', parseDebateContextUsage(meta?.debateContext)); contextStore.resetOverride('ask'); } - const enrichedOutput = finalEvent.output && typeof finalEvent.output === 'object' && !Array.isArray(finalEvent.output) - ? { ...finalEvent.output, model: resolvedModel, provider } + const finalFields = finalEvent.output && typeof finalEvent.output === 'object' && !Array.isArray(finalEvent.output) + ? finalEvent.output as Record + : undefined; + // The server flags an answer cut off at its length cap; say so under it. + const cutOff = meta?.replyTruncated === true && Array.isArray(finalFields?.uncertainties) + ? { uncertainties: [...finalFields.uncertainties, i18n.t('ask.replyCutOff')] } + : {}; + const enrichedOutput = finalFields + ? { ...finalFields, ...cutOff, model: resolvedModel, provider } : finalEvent.output; finalOutput = acceptFinal(enrichedOutput, finalOutput); } diff --git a/src/locales/en.json b/src/locales/en.json index 7a2b4ac96..b0b4daee7 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -419,6 +419,7 @@ "streamInterruptedDesc": "You can ask again to get a complete answer.", "chatSaved": "Chat saved", "chatSavedDesc": "You can load it from History.", + "replyCutOff": "This answer hit its length limit and stops early. Ask a narrower question or ask it to continue.", "retryableFailedDesc": "Couldn't reach the AI. Check your connection and retry.", "saveFailed": "Save failed", "saveFailedDesc": "The chat could not be saved. Please try again.", diff --git a/src/locales/es.json b/src/locales/es.json index e03254f8e..7ad28b88b 100644 --- a/src/locales/es.json +++ b/src/locales/es.json @@ -419,6 +419,7 @@ "streamInterruptedDesc": "Puedes preguntar de nuevo para obtener una respuesta completa.", "chatSaved": "Chat guardado", "chatSavedDesc": "Puedes abrirlo desde el Historial.", + "replyCutOff": "Esta respuesta alcanzó su límite de longitud y termina antes de tiempo. Haz una pregunta más concreta o pide que continúe.", "retryableFailedDesc": "No se pudo conectar con la IA. Comprueba tu conexión e inténtalo de nuevo.", "saveFailed": "No se pudo guardar", "saveFailedDesc": "No se pudo guardar el chat. Inténtalo de nuevo.", diff --git a/src/locales/hu.json b/src/locales/hu.json index 46a0e4953..bf2f52d77 100644 --- a/src/locales/hu.json +++ b/src/locales/hu.json @@ -419,6 +419,7 @@ "streamInterruptedDesc": "Kérdezz újra a teljes válaszért.", "chatSaved": "Chat mentve", "chatSavedDesc": "Az Előzményekből újra megnyithatod.", + "replyCutOff": "Ez a válasz elérte a hosszkorlátot, ezért idő előtt véget ér. Tegyél fel szűkebb kérdést, vagy kérd, hogy folytassa.", "retryableFailedDesc": "Nem sikerült elérni az MI-t. Ellenőrizd a kapcsolatot, majd próbáld újra.", "saveFailed": "A mentés sikertelen", "saveFailedDesc": "Nem sikerült menteni a chatet. Próbáld újra.", diff --git a/supabase/README.md b/supabase/README.md index 37e2e17a1..79ac8e6e3 100644 --- a/supabase/README.md +++ b/supabase/README.md @@ -32,7 +32,7 @@ supabase/tests/ pgTAP database and policy tests - `ai-stream`: authenticated server-sent events endpoint for Ask. It emits deltas, a final event, or an error and records usage. - `voice`: authenticated text-to-speech for the opponent in voice rounds. It accepts one segment of at most 1,000 characters with the token `ai` signed for it, so it speaks only server-written text. It answers `voice_unavailable` without `ELEVENLABS_API_KEY`, enforces the per-user daily voice cap through `consume_daily_quota` under the `voice` key, calls ElevenLabs (`eleven_v4` for sparring, `eleven_v4_turbo` for Crossfire), and records each request in `ai_requests` as `voice_sparring` or `voice_crossfire`. Every failure is a JSON error with a code, and the browser then reads the reply with its own voice. -`ai/index.ts` is the entrypoint and dispatcher; `ai/features/` holds one handler per feature, `ai/provider.ts` the OpenRouter call and JSON repair, `ai/request.ts` input hygiene, and `ai/runtime.ts` the environment and service clients. Each feature's prompt lives in `_shared/prompts/.ts`, with shared rules in `_shared/prompts/common.ts` and composition in `_shared/prompts/system.ts`. +`ai/index.ts` is the entrypoint and dispatcher; `ai/features/` holds one handler per feature, `ai/provider.ts` the OpenRouter call, JSON repair, and the retry for a reply cut off at its cap (`_shared/output-caps.ts` holds the caps), `ai/request.ts` input hygiene, and `ai/runtime.ts` the environment and service clients. Each feature's prompt lives in `_shared/prompts/.ts`, with shared rules in `_shared/prompts/common.ts` and composition in `_shared/prompts/system.ts`. Both endpoints accept an optional `debateContext` of IDs (`debateId`, `side`, `sourceIds`, `outputs`). `_shared/debate-context-loader.ts` loads the matching material with the service role, filtering every query by the caller's user ID and the one debate, and returns 404 for a debate the caller does not own before any quota is reserved. Requests without it still accept the legacy `sourceIds` field. diff --git a/supabase/functions/_shared/model-catalog.ts b/supabase/functions/_shared/model-catalog.ts index 9ac386965..7657a49a4 100644 --- a/supabase/functions/_shared/model-catalog.ts +++ b/supabase/functions/_shared/model-catalog.ts @@ -4,7 +4,6 @@ */ export type ReasoningEffort = "low" | "medium" | "high"; -export const MAX_REQUEST_OUTPUT_TOKENS = 32_768; export interface ModelCatalogEntry { readonly id: string; diff --git a/supabase/functions/_shared/output-caps.ts b/supabase/functions/_shared/output-caps.ts new file mode 100644 index 000000000..8854ddc0a --- /dev/null +++ b/supabase/functions/_shared/output-caps.ts @@ -0,0 +1,60 @@ +import type { ReasoningEffort } from "./model-catalog.ts"; + +/** + * How long each AI reply may be. Change the numbers here; every provider call + * picks its cap from this table. + * + * Each value is the visible reply budget in tokens. The request's max_tokens + * adds room for hidden reasoning on top (see outputTokenLimit), because + * OpenRouter counts reasoning and reply against the same limit. + */ +export const REPLY_TOKEN_CAPS = { + /** Ask, both the streamed chat and the structured JSON modes. */ + ask: 1_500, + /** One Crossfire question or answer. */ + crossfireTurn: 1_000, + /** The turn-cited Crossfire debrief: six arrays of findings. */ + crossfireDebrief: 2_000, + /** One Sparring opponent speech. */ + sparringSpeech: 1_500, + /** The Sparring scorecard: four categories, moments, areas, verdict. */ + sparringScore: 2_000, + /** A full case, any length setting. */ + generate: 6_000, + attack: 4_000, + brainstorm: 4_000, + factoids: 4_000, + /** Research sub-calls: the search decision and follow-up queries. */ + research: 400, +} as const; + +export type ReplyCap = keyof typeof REPLY_TOKEN_CAPS; + +/** + * A reply cut off at its cap is retried once with this many times the budget. + * Truncated JSON cannot be parsed, so the retry is the only way to save it. + */ +export const TRUNCATION_RETRY_FACTOR = 2; + +/** + * The share of max_tokens a provider may spend on reasoning at each effort, + * and the smallest reasoning budget it allows. These mirror OpenRouter's + * conversion for budget-based models (Anthropic, Gemini): + * budget = max(max_tokens * share, 1024). + * https://openrouter.ai/docs/use-cases/reasoning-tokens + */ +const REASONING_SHARE: Record = { low: 0.2, medium: 0.5, high: 0.8 }; +const MIN_REASONING_TOKENS = 1_024; + +/** + * The max_tokens to send so the provider's full reasoning allowance still + * leaves `replyTokens` for the visible reply, bounded by the model's limit. + */ +export function outputTokenLimit( + replyTokens: number, + effort: ReasoningEffort, + modelMaxTokens: number, +): number { + const withShare = Math.ceil(replyTokens / (1 - REASONING_SHARE[effort])); + return Math.min(modelMaxTokens, Math.max(withShare, replyTokens + MIN_REASONING_TOKENS)); +} diff --git a/supabase/functions/ai-stream/index.ts b/supabase/functions/ai-stream/index.ts index f595da149..ed7ccdcdf 100644 --- a/supabase/functions/ai-stream/index.ts +++ b/supabase/functions/ai-stream/index.ts @@ -30,13 +30,13 @@ import { isAutoModelSelection, isModelSelection, isReasoningEffort, - MAX_REQUEST_OUTPUT_TOKENS, retiredModelSuccessor, type ModelCatalogEntry, type ModelSelection, type ReasoningEffort, } from "../_shared/model-catalog.ts"; import { resolveModelSelection } from "../_shared/auto-model-router.ts"; +import { outputTokenLimit, REPLY_TOKEN_CAPS } from "../_shared/output-caps.ts"; import { JEV_MODEL } from "../_shared/jev.ts"; type ChatRole = "user" | "assistant"; @@ -239,6 +239,7 @@ export async function handleRequest(req: Request): Promise { let streamUsageRaw: Record | undefined; let streamSawDone = false; let streamSawFinishReason = false; + let streamFinishReason: string | undefined; const collectUsage = (): NormalizedUsage => { return streamUsageRaw ? normalizeOpenRouterUsage(streamUsageRaw) @@ -265,7 +266,7 @@ export async function handleRequest(req: Request): Promise { const payload: Record = { model, messages, - max_tokens: Math.min(modelEntry.maxTokens, MAX_REQUEST_OUTPUT_TOKENS), + max_tokens: outputTokenLimit(REPLY_TOKEN_CAPS.ask, reasoningEffort, modelEntry.maxTokens), reasoning: { effort: reasoningEffort, exclude: true }, // Omitting sort and order keeps OpenRouter's balanced provider load balancing. provider: { require_parameters: true }, @@ -305,7 +306,10 @@ export async function handleRequest(req: Request): Promise { throw new Error(typeof message === "string" ? message : "OpenRouter stream error"); } const delta = (Array.isArray(parsed.choices) ? parsed.choices[0] : undefined) as { delta?: { content?: string }; finish_reason?: unknown } | undefined; - if (typeof delta?.finish_reason === "string" && delta.finish_reason) streamSawFinishReason = true; + if (typeof delta?.finish_reason === "string" && delta.finish_reason) { + streamSawFinishReason = true; + streamFinishReason = delta.finish_reason; + } if (typeof delta?.delta?.content === "string" && delta.delta.content) { accumulated += delta.delta.content; emit("delta", { text: delta.delta.content }); @@ -340,6 +344,13 @@ export async function handleRequest(req: Request): Promise { throw new Error("OpenRouter stream ended before completion"); } + // The text already reached the reader, so a cut-off answer cannot be + // retried. The client tells the user it stopped short. + const replyTruncated = streamFinishReason === "length"; + if (replyTruncated) { + console.warn(`[AI_STREAM] Reply hit its ask cap (${REPLY_TOKEN_CAPS.ask} tokens, ${model}, effort ${reasoningEffort})`); + } + const output = { response: accumulated.trim(), model, @@ -351,7 +362,17 @@ export async function handleRequest(req: Request): Promise { }; if (!req.signal.aborted) { - try { emit("final", { output, meta: { requestedModel, resolvedModel: model, ...(debateMeta ? { debateContext: debateMeta } : {}) } }); } catch { /* client disconnected after provider completion */ } + try { + emit("final", { + output, + meta: { + requestedModel, + resolvedModel: model, + ...(replyTruncated ? { replyTruncated: true } : {}), + ...(debateMeta ? { debateContext: debateMeta } : {}), + }, + }); + } catch { /* client disconnected after provider completion */ } } await logUsage(user.id, "ask", OPENROUTER_PROVIDER, model, collectUsage(), Date.now() - startedAt); } catch (err) { diff --git a/supabase/functions/ai/features/ask.ts b/supabase/functions/ai/features/ask.ts index 9211efbfb..5e7695621 100644 --- a/supabase/functions/ai/features/ask.ts +++ b/supabase/functions/ai/features/ask.ts @@ -98,7 +98,7 @@ export async function handleAsk(ctx: FeatureContext): Promise { const systemPrompt = await buildSystemPromptFor(feature, effectiveResearch); const prompt = `${basePrompt}${urlContext}${webSearchContext}`; const messages = [...history, { role: "user", content: prompt }]; - const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, reasoningEffort, workSignal, ctx.stream)); + const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, reasoningEffort, "ask", workSignal, ctx.stream)); const output = cleanGeneratedText(aiResult.output); if (output && typeof output === "object") { diff --git a/supabase/functions/ai/features/attack.ts b/supabase/functions/ai/features/attack.ts index 7193e88e0..9a75b7361 100644 --- a/supabase/functions/ai/features/attack.ts +++ b/supabase/functions/ai/features/attack.ts @@ -29,7 +29,7 @@ export async function handleAttack(ctx: FeatureContext): Promise { const systemPrompt = await buildSystemPromptFor(feature, research); const userPrompt = buildUserPrompt(feature, sanitizedInput, searchContext); const messages = [{ role: "user", content: userPrompt }]; - const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, reasoningEffort, workSignal, ctx.stream)); + const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, reasoningEffort, "attack", workSignal, ctx.stream)); const output = cleanGeneratedText(aiResult.output); enforceAttackControls(output, detectFallacies, analyzeWeighing); diff --git a/supabase/functions/ai/features/brainstorm.ts b/supabase/functions/ai/features/brainstorm.ts index f26b3dcbf..78f974566 100644 --- a/supabase/functions/ai/features/brainstorm.ts +++ b/supabase/functions/ai/features/brainstorm.ts @@ -30,7 +30,7 @@ export async function handleBrainstorm(ctx: FeatureContext): Promise { const systemPrompt = await buildSystemPromptFor(feature, research); const userPrompt = buildUserPrompt(feature, sanitizedInput, searchContext); const messages = [{ role: "user", content: userPrompt }]; - const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, reasoningEffort, workSignal, ctx.stream)); + const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, reasoningEffort, "brainstorm", workSignal, ctx.stream)); const output = cleanGeneratedText(aiResult.output); enforceBrainstormControls(output, side, outputMode); diff --git a/supabase/functions/ai/features/crossfire.ts b/supabase/functions/ai/features/crossfire.ts index 125c1e65f..75068fe18 100644 --- a/supabase/functions/ai/features/crossfire.ts +++ b/supabase/functions/ai/features/crossfire.ts @@ -63,6 +63,7 @@ export async function handleCrossfire(ctx: FeatureContext, options: CrossfireHan systemPrompt, [{ role: "user", content: userPrompt }], reasoningEffort, + crossfireInput.operation === "next-turn" ? "crossfireTurn" : "crossfireDebrief", workSignal, ctx.stream, )); diff --git a/supabase/functions/ai/features/factoids.ts b/supabase/functions/ai/features/factoids.ts index 6db847223..c89aac360 100644 --- a/supabase/functions/ai/features/factoids.ts +++ b/supabase/functions/ai/features/factoids.ts @@ -35,7 +35,7 @@ export async function handleFactoids(ctx: FeatureContext): Promise { const userPrompt = buildUserPrompt(feature, sanitizedInput, searchContext); const messages = [{ role: "user", content: userPrompt }]; - const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, reasoningEffort, workSignal, ctx.stream)); + const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, reasoningEffort, "factoids", workSignal, ctx.stream)); const output = cleanGeneratedText(aiResult.output); // Post-process: quality floor. Only citations whose URL matches fetched, diff --git a/supabase/functions/ai/features/generate.ts b/supabase/functions/ai/features/generate.ts index 3b2af8b2e..09a69e9f2 100644 --- a/supabase/functions/ai/features/generate.ts +++ b/supabase/functions/ai/features/generate.ts @@ -82,7 +82,7 @@ export async function handleGenerate(ctx: FeatureContext): Promise { const systemPrompt = await buildSystemPromptFor(feature, research); const userPrompt = buildUserPrompt(feature, sanitizedInput, searchContext); const messages = [{ role: "user", content: userPrompt }]; - const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, reasoningEffort, workSignal, ctx.stream)); + const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, reasoningEffort, "generate", workSignal, ctx.stream)); const output = cleanGeneratedText(aiResult.output); // Preserve model evidence warnings when research exists. If research diff --git a/supabase/functions/ai/features/sparring.ts b/supabase/functions/ai/features/sparring.ts index 877b9b3bf..3449e095c 100644 --- a/supabase/functions/ai/features/sparring.ts +++ b/supabase/functions/ai/features/sparring.ts @@ -35,7 +35,16 @@ export async function handleSparring(ctx: FeatureContext, options: SparringHandl const systemPrompt = await buildSystemPromptFor(feature); const userPrompt = buildUserPrompt(feature, sparringRequest as unknown as Record, ""); const messages = [{ role: "user", content: userPrompt }]; - const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, reasoningEffort, workSignal, ctx.stream)); + const aiResult = recordMainResult(await callAI( + provider, + model, + systemPrompt, + messages, + reasoningEffort, + sparringRequest.action === "score" ? "sparringScore" : "sparringSpeech", + workSignal, + ctx.stream, + )); const output = cleanGeneratedText(aiResult.output); const normalizedOutput = normalizeSparringOutput(output, sparringRequest.action); const parsedOutput = parseSparringProviderOutput(normalizedOutput, sparringRequest.action); diff --git a/supabase/functions/ai/handler-research.test.ts b/supabase/functions/ai/handler-research.test.ts index d9ab9383c..93404c1c6 100644 --- a/supabase/functions/ai/handler-research.test.ts +++ b/supabase/functions/ai/handler-research.test.ts @@ -18,7 +18,8 @@ for (const [key, value] of Object.entries(environment)) Deno.env.set(key, value) const { createRequestBudgetSignal, handleRequest } = await import("./index.ts"); const { handleRequest: handleStreamRequest } = await import("../ai-stream/index.ts"); -const { MAX_REQUEST_OUTPUT_TOKENS, MODEL_CATALOG } = await import("../_shared/model-catalog.ts"); +const { MODEL_CATALOG } = await import("../_shared/model-catalog.ts"); +const { outputTokenLimit, REPLY_TOKEN_CAPS } = await import("../_shared/output-caps.ts"); Deno.test("AI request work shares one deadline signal", () => { const caller = new AbortController(); @@ -722,7 +723,8 @@ Deno.test("AI handlers use strict OpenRouter routing and server-side reasoning t const entry = MODEL_CATALOG[index]; const body = openRouterBodies[index]; if (body.model !== entry.id) throw new Error(`wrong routed model: ${String(body.model)}`); - if (body.max_tokens !== MAX_REQUEST_OUTPUT_TOKENS) throw new Error(`output cap was not enforced for ${entry.id}`); + const effort = entry.isProOnly ? "high" : "low"; + if (body.max_tokens !== outputTokenLimit(REPLY_TOKEN_CAPS.ask, effort, entry.maxTokens)) throw new Error(`Ask output cap was not enforced for ${entry.id}`); const reasoning = body.reasoning as Record; const provider = body.provider as Record; if (reasoning.effort !== (entry.isProOnly ? "high" : "low")) throw new Error(`tier effort was not enforced for ${entry.id}`); @@ -791,7 +793,7 @@ Deno.test("streaming OpenRouter usage records zero and missing costs distinctly" body: JSON.stringify({ feature: "ask", model: "openai/gpt-6-luna", input: { message: "Hello", mode: "general" } }), })); await withZero.text(); - if (streamBodies[0]?.max_tokens !== MAX_REQUEST_OUTPUT_TOKENS) throw new Error("stream output cap was not enforced"); + if (streamBodies[0]?.max_tokens !== outputTokenLimit(REPLY_TOKEN_CAPS.ask, "low", 128_000)) throw new Error("stream Ask output cap was not enforced"); if (usageRows.at(-1)?.cost_usd !== 0) throw new Error(`zero stream cost was not logged: ${JSON.stringify(usageRows.at(-1))}`); includeUsage = false; diff --git a/supabase/functions/ai/index.ts b/supabase/functions/ai/index.ts index 786ccb8f2..fc9993748 100644 --- a/supabase/functions/ai/index.ts +++ b/supabase/functions/ai/index.ts @@ -638,7 +638,7 @@ export async function handleRequest(req: Request): Promise { }, callModel: async (args) => { try { - const result = await callAI(args.provider, args.model, args.systemPrompt, args.messages, args.reasoningEffort, args.signal); + const result = await callAI(args.provider, args.model, args.systemPrompt, args.messages, args.reasoningEffort, "research", args.signal); // A provider that ignores cancellation must not append usage after the // research timeout has already been finalized by the parent request. if (result.usage && !args.signal?.aborted) researchUsage.push(result.usage); @@ -755,11 +755,12 @@ export async function handleRequest(req: Request): Promise { return await handleAttack(ctx); } - // Fallback for any other feature + // Fallback for any other feature. Without its own cap it gets the + // largest one, a full case. const systemPrompt = await buildSystemPromptFor(feature); const userPrompt = buildUserPrompt(feature, sanitizedInput, ""); const messages = [{ role: "user", content: userPrompt }]; - const fallbackResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, reasoningEffort, workSignal, stream)); + const fallbackResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, reasoningEffort, "generate", workSignal, stream)); const output = cleanGeneratedText(fallbackResult.output); const latencyMs = Date.now() - startTime; diff --git a/supabase/functions/ai/provider.ts b/supabase/functions/ai/provider.ts index 52e01bcda..c7d3191ca 100644 --- a/supabase/functions/ai/provider.ts +++ b/supabase/functions/ai/provider.ts @@ -8,10 +8,15 @@ import { } from "../_shared/cost.ts"; import { getModelCatalogEntry, - MAX_REQUEST_OUTPUT_TOKENS, type ModelCatalogEntry, type ReasoningEffort, } from "../_shared/model-catalog.ts"; +import { + outputTokenLimit, + REPLY_TOKEN_CAPS, + type ReplyCap, + TRUNCATION_RETRY_FACTOR, +} from "../_shared/output-caps.ts"; import { fetchJsonWithTimeout, fetchWithTimeoutAndConsume, @@ -45,6 +50,18 @@ export class InvalidJsonResponseError extends Error { } } +/** The provider stopped at max_tokens, so the reply is incomplete. */ +export class OutputTruncatedError extends Error { + /** Token usage of the cut-off attempt so the retry can bill it. */ + usage?: NormalizedUsage; + + constructor(usage?: NormalizedUsage) { + super("AI reply was cut off at its token cap"); + this.name = "OutputTruncatedError"; + this.usage = usage; + } +} + /** * Parses provider content, attaching the attempt's token usage to a parse * failure so callAI's repair retry can fold it into the final accounting. @@ -100,7 +117,7 @@ export interface GenerationStream { export type GenerationPhase = "research" | "writing" | "checking"; type OpenRouterResponse = { - choices?: Array<{ message?: { content?: string | null } }>; + choices?: Array<{ message?: { content?: string | null }; finish_reason?: unknown }>; usage?: unknown; }; @@ -126,52 +143,73 @@ export async function callAI( systemPrompt: string, messages: Array<{ role: string; content: string }>, reasoningEffort: ReasoningEffort, + cap: ReplyCap, signal?: AbortSignal, stream?: GenerationStream, ): Promise { + const replyTokens = REPLY_TOKEN_CAPS[cap]; try { - return await callAIOnce(provider, model, systemPrompt, messages, reasoningEffort, signal, stream); + return await callAIOnce(provider, model, systemPrompt, messages, reasoningEffort, replyTokens, signal, stream); } catch (err) { + if (err instanceof OutputTruncatedError) { + console.warn(`[AI] Reply hit its ${cap} cap (${replyTokens} tokens, ${model}, effort ${reasoningEffort}); retrying with ${replyTokens * TRUNCATION_RETRY_FACTOR}`); + stream?.reset(); + return await retryOnce(err.usage, "truncation_retry", () => + callAIOnce(provider, model, systemPrompt, messages, reasoningEffort, replyTokens * TRUNCATION_RETRY_FACTOR, signal, stream) + ); + } if (!(err instanceof InvalidJsonResponseError)) throw attachMissingUsage(err, "call"); console.warn(`[AI] ${err.providerName} returned invalid JSON; retrying with repair prompt`); stream?.reset(); - let retry: AIResult; - try { - retry = await callAIOnce( + return await retryOnce(err.usage, "repair_retry", () => + callAIOnce( provider, model, `${systemPrompt}\n\nJSON REPAIR INSTRUCTION: Return only valid JSON matching this schema, no prose.`, withJsonRepairPrompt(messages), reasoningEffort, + replyTokens, signal, stream, - ); - } catch (retryError) { - // Preserve the completed first attempt if cancellation or another - // failure prevents the repair attempt from returning a result. - const retryUsage = retryError && typeof retryError === "object" && "usage" in retryError - ? (retryError as { usage?: NormalizedUsage }).usage ?? missingOpenRouterUsage("repair_retry") - : missingOpenRouterUsage("repair_retry"); - const usage = mergeNormalizedUsage([err.usage, retryUsage]); - if (usage && retryError && typeof retryError === "object") { - const errorWithUsage = retryError as { - usage?: NormalizedUsage; - completedUsage?: NormalizedUsage; - }; - errorWithUsage.usage = usage; - // The first response was parsed before the retry began, so retain it - // even if the retry is the operation cancelled by the caller. - errorWithUsage.completedUsage = err.usage; - } - throw attachMissingUsage(retryError, "repair_retry"); + ) + ); + } +} + +/** Runs the single retry after a failed attempt and bills both attempts. */ +async function retryOnce( + firstUsage: NormalizedUsage | undefined, + reason: string, + attempt: () => Promise, +): Promise { + let retry: AIResult; + try { + retry = await attempt(); + } catch (retryError) { + // Preserve the completed first attempt if cancellation or another + // failure prevents the retry from returning a result. + const retryUsage = retryError && typeof retryError === "object" && "usage" in retryError + ? (retryError as { usage?: NormalizedUsage }).usage ?? missingOpenRouterUsage(reason) + : missingOpenRouterUsage(reason); + const usage = mergeNormalizedUsage([firstUsage, retryUsage]); + if (usage && retryError && typeof retryError === "object") { + const errorWithUsage = retryError as { + usage?: NormalizedUsage; + completedUsage?: NormalizedUsage; + }; + errorWithUsage.usage = usage; + // The first response was received before the retry began, so retain it + // even if the retry is the operation cancelled by the caller. + errorWithUsage.completedUsage = firstUsage; } - // The failed attempt still consumed tokens; bill both. - return { - output: retry.output, - usage: mergeNormalizedUsage([err.usage, retry.usage]), - }; + throw attachMissingUsage(retryError, reason); } + // The failed attempt still consumed tokens; bill both. + return { + output: retry.output, + usage: mergeNormalizedUsage([firstUsage, retry.usage]), + }; } export function withJsonRepairPrompt(messages: Array<{ role: string; content: string }>) { @@ -195,6 +233,7 @@ export async function callAIOnce( systemPrompt: string, messages: Array<{ role: string; content: string }>, reasoningEffort: ReasoningEffort, + replyTokens: number, signal?: AbortSignal, stream?: GenerationStream, ): Promise { @@ -203,6 +242,7 @@ export async function callAIOnce( [{ role: "system", content: systemPrompt }, ...messages], model, reasoningEffort, + replyTokens, signal, stream, ); @@ -225,6 +265,7 @@ export async function callOpenRouter( messages: Array<{ role: string; content: string }>, model: string, reasoningEffort: ReasoningEffort, + replyTokens: number, signal?: AbortSignal, stream?: GenerationStream, ): Promise { @@ -235,7 +276,7 @@ export async function callOpenRouter( const payload: Record = { model, messages, - max_tokens: Math.min(modelEntry.maxTokens, MAX_REQUEST_OUTPUT_TOKENS), + max_tokens: outputTokenLimit(replyTokens, reasoningEffort, modelEntry.maxTokens), response_format: { type: "json_object" }, reasoning: { effort: reasoningEffort, exclude: true }, // Omitting sort and order keeps OpenRouter's balanced provider load balancing. @@ -254,6 +295,7 @@ export async function callOpenRouter( } if (!data?.choices || !Array.isArray(data.choices) || data.choices.length === 0) throw new Error("OpenRouter returned no choices"); const usage = normalizeProviderUsage(data.usage) ?? missingOpenRouterUsage("response"); + if (data.choices[0]?.finish_reason === "length") throw new OutputTruncatedError(usage); return parseProviderContent(data.choices[0]?.message?.content ?? "", "OpenRouter", usage); } @@ -280,7 +322,7 @@ async function streamOpenRouter( signal?: AbortSignal, ): Promise { stream.phase("writing"); - const { text, usage } = await fetchWithTimeoutAndConsume(`${OPENROUTER_API_URL}/v1/chat/completions`, { + const { text, usage, finishReason } = await fetchWithTimeoutAndConsume(`${OPENROUTER_API_URL}/v1/chat/completions`, { method: "POST", headers: { "Content-Type": "application/json", "Authorization": `Bearer ${apiKey}` }, body: JSON.stringify({ ...payload, stream: true, stream_options: { include_usage: true } }), @@ -294,6 +336,7 @@ async function streamOpenRouter( } return await readOpenRouterStream(response.body, stream, bodySignal); }); + if (finishReason === "length") throw new OutputTruncatedError(usage); return parseProviderContent(text, "OpenRouter", usage); } @@ -301,7 +344,7 @@ export async function readOpenRouterStream( body: ReadableStream, stream: GenerationStream, signal: AbortSignal, -): Promise<{ text: string; usage: NormalizedUsage }> { +): Promise<{ text: string; usage: NormalizedUsage; finishReason?: string }> { const reader = body.getReader(); const decoder = new TextDecoder(); let buffer = ""; @@ -311,7 +354,7 @@ export async function readOpenRouterStream( const encoder = new TextEncoder(); let usageRaw: Record | undefined; let done = false; - let finished = false; + let finishReason: string | undefined; const processLine = (line: string) => { const trimmed = line.trim(); @@ -332,7 +375,7 @@ export async function readOpenRouterStream( throw new Error("AI provider request failed"); } const choice = Array.isArray(chunk.choices) ? chunk.choices[0] : undefined; - if (typeof choice?.finish_reason === "string" && choice.finish_reason) finished = true; + if (typeof choice?.finish_reason === "string" && choice.finish_reason) finishReason = choice.finish_reason; const content = choice?.delta?.content; if (typeof content === "string" && content) { text += content; @@ -380,10 +423,10 @@ export async function readOpenRouterStream( } finally { void reader.cancel().catch(() => undefined); } - if (!done && !finished) { + if (!done && !finishReason) { const error = new Error("OpenRouter stream ended before completion") as Error & { usage?: NormalizedUsage }; error.usage = usageRaw ? { ...normalizeOpenRouterUsage(usageRaw), incomplete: true } : missingOpenRouterUsage("stream"); throw error; } - return { text, usage: usageRaw ? normalizeOpenRouterUsage(usageRaw) : missingOpenRouterUsage("stream") }; + return { text, usage: usageRaw ? normalizeOpenRouterUsage(usageRaw) : missingOpenRouterUsage("stream"), finishReason }; } diff --git a/supabase/functions/ai/stream.test.ts b/supabase/functions/ai/stream.test.ts index 1aa85929f..34a2a10fa 100644 --- a/supabase/functions/ai/stream.test.ts +++ b/supabase/functions/ai/stream.test.ts @@ -14,6 +14,7 @@ for (const [key, value] of Object.entries(environment)) Deno.env.set(key, value) const { handleRequest } = await import("./index.ts"); const { readOpenRouterStream } = await import("./provider.ts"); +const { outputTokenLimit, REPLY_TOKEN_CAPS, TRUNCATION_RETRY_FACTOR } = await import("../_shared/output-caps.ts"); const USER = "user-1"; const SESSION = "5e551000-0000-4000-8000-000000000002"; @@ -29,7 +30,7 @@ function jsonResponse(body: unknown, status = 200): Response { const USAGE = { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15, cost: 0.000001 }; /** One provider reply: text to stream (or return whole), or an HTTP failure. */ -type ProviderReply = { content: string } | { status: number }; +type ProviderReply = { content: string; finishReason?: string } | { status: number }; interface Harness { providerBodies: Array>; @@ -37,13 +38,13 @@ interface Harness { restore: () => void; } -function sseBody(content: string): ReadableStream { +function sseBody(content: string, finishReason = "stop"): ReadableStream { const encoder = new TextEncoder(); const chunks: string[] = [": OPENROUTER PROCESSING\n\n"]; for (let index = 0; index < content.length; index += 7) { chunks.push(`data: ${JSON.stringify({ choices: [{ delta: { content: content.slice(index, index + 7) } }] })}\n\n`); } - chunks.push(`data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: "stop" }] })}\n\n`); + chunks.push(`data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: finishReason }] })}\n\n`); chunks.push(`data: ${JSON.stringify({ choices: [], usage: USAGE })}\n\n`, "data: [DONE]\n\n"); return new ReadableStream({ start(controller) { @@ -80,8 +81,8 @@ function install(replies: ProviderReply[]): Harness { harness.providerBodies.push(body); const reply = replies[Math.min(harness.providerBodies.length - 1, replies.length - 1)]; if ("status" in reply) return jsonResponse({ error: { message: "Provider overloaded" } }, reply.status); - if (body.stream === true) return new Response(sseBody(reply.content), { headers: { "Content-Type": "text/event-stream" } }); - return jsonResponse({ choices: [{ message: { content: reply.content } }], usage: USAGE }); + if (body.stream === true) return new Response(sseBody(reply.content, reply.finishReason), { headers: { "Content-Type": "text/event-stream" } }); + return jsonResponse({ choices: [{ message: { content: reply.content }, finish_reason: reply.finishReason ?? "stop" }], usage: USAGE }); } throw new Error(`Unhandled request: ${url}`); }) as typeof fetch; @@ -189,6 +190,42 @@ Deno.test("a repair retry voids the streamed text and bills both attempts", asyn } }); +Deno.test("a reply cut off at its cap is retried once with a larger cap and both attempts are billed", async () => { + const cutOff = { content: REPLY.slice(0, 60), finishReason: "length" }; + // Basic accounts always reason at low effort. + const firstCap = outputTokenLimit(REPLY_TOKEN_CAPS.sparringSpeech, "low", 128_000); + const retryCap = outputTokenLimit(REPLY_TOKEN_CAPS.sparringSpeech * TRUNCATION_RETRY_FACTOR, "low", 128_000); + for (const stream of [false, true]) { + const harness = install([cutOff, { content: REPLY }]); + try { + const response = await handleRequest(aiRequest(stream)); + if (stream) { + const events = await readEvents(response); + assert(events.some((event) => event.event === "reset"), "the cut-off text was not voided"); + assert(events.at(-1)?.event === "result" && events.at(-1)?.data.status === 200, "the retried stream did not succeed"); + } else { + assert(response.status === 200, `the retried request returned ${response.status}`); + } + const caps = harness.providerBodies.map((body) => body.max_tokens); + assert(caps.length === 2 && caps[0] === firstCap && caps[1] === retryCap, `unexpected caps: ${JSON.stringify(caps)}`); + const usage = harness.usageRows.find((row) => row.feature === "sparring"); + assert(usage?.input_tokens === 20 && usage.output_tokens === 10, `both attempts were not billed: ${JSON.stringify(usage)}`); + } finally { + harness.restore(); + } + } + + // A second cut-off fails the request instead of retrying again. + const harness = install([cutOff, cutOff, { content: REPLY }]); + try { + const response = await handleRequest(aiRequest(false)); + assert(response.status >= 500, `a twice cut-off reply returned ${response.status}`); + assert(harness.providerBodies.length === 2, `expected one retry, got ${harness.providerBodies.length - 1}`); + } finally { + harness.restore(); + } +}); + Deno.test("a provider failure ends the stream with the same error a plain request returns", async () => { const harness = install([{ status: 503 }]); try { From 78164d2b4662e990669b2a3b726bceaa5945471e Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 22 Oct 2026 11:40:30 +0000 Subject: [PATCH 2/3] Keep reasoning room for effort-based models and translate the cut-off notice on render Co-Authored-By: Claude Opus 5.5 --- .../verify-debatelab/features/ai-tools.md | 2 +- packages/core/src/ai/schemas/ask.ts | 2 ++ src/components/features/ask/AskChat.tsx | 5 ++-- src/lib/ai/streaming.ts | 12 ++------- supabase/functions/_shared/output-caps.ts | 25 ++++++++++++------- supabase/functions/ai-stream/index.ts | 15 +++-------- 6 files changed, 27 insertions(+), 34 deletions(-) diff --git a/.claude/skills/verify-debatelab/features/ai-tools.md b/.claude/skills/verify-debatelab/features/ai-tools.md index b3f02c083..f30340f73 100644 --- a/.claude/skills/verify-debatelab/features/ai-tools.md +++ b/.claude/skills/verify-debatelab/features/ai-tools.md @@ -25,7 +25,7 @@ Preconditions: - **Brainstorm.** On `/app/brainstorm`, fill the field labelled `Motion / Topic *` and click `Start Brainstorming`. With no Current Debate, the dialog `Start a debate with this motion?` opens first: `chooseForMotion(page, "create")` (or `"once"`). The text `FAKE-PROVIDER idea 1` appears. `providerCall("brainstorm").user` contains `Motion: `, and the `brainstorms` row is filed in the new debate. With a Current Debate, the field is prefilled and no dialog opens unless the motion differs. - **Factoids.** On `/app/factoids`, fill the field labelled `Statement *` and click `Find Factoids`. The text `FAKE-PROVIDER search suggestion` appears. - **Ask, streamed.** On `/app/ask`, fill the placeholder `Ask your debate coach...` and click the button named `Send message`. The text `FAKE-PROVIDER streamed answer: ...` appears. `fake.requests()` features contain `ask-stream` and not `ask`. `ai_requests` gets `{feature: "ask", status: "success"}`. -- **Ask, cut off.** Call `await fake.truncate("ask-stream")` and send. The stream ends with `finish_reason: "length"`, its `final` event carries `meta.replyTruncated: true`, and the partial answer stays with `This answer hit its length limit and stops early...` under `Uncertainties:`. No `Response was interrupted` toast. Fake mode only. +- **Ask, cut off.** Call `await fake.truncate("ask-stream")` and send. The stream ends with `finish_reason: "length"`, its `final` event's output carries `replyTruncated: true`, and the partial answer stays with `This answer hit its length limit and stops early...` under `Uncertainties:`. No `Response was interrupted` toast. Fake mode only. - **Ask, cited.** Open the `Options` button above the message box, click `#citation-mode`, close it with Escape, then send. The text `FAKE-PROVIDER answer` appears, and `providerCall("ask")` fired. - **Stop and errors.** While Ask answers, the send button becomes `Stop answering`; Stop keeps the partial reply and returns focus to the box, which stays editable throughout. A failed request leaves an `alert` above the box with `Retry`, which asks again without adding the question twice. The four form tools show one status line (`Reviewing your case…`, `Finding angles…`, `Searching for evidence…`, `Writing your case…`, or `Researching sources…` while web research runs) with a `Cancel` button until the model starts writing, then the live draft (see [live-generation](./live-generation.md)). Failures stay on the page with `Retry`. diff --git a/packages/core/src/ai/schemas/ask.ts b/packages/core/src/ai/schemas/ask.ts index 9a6554c6a..ee07860fd 100644 --- a/packages/core/src/ai/schemas/ask.ts +++ b/packages/core/src/ai/schemas/ask.ts @@ -53,6 +53,8 @@ export const AskOutputSchema = z.object({ assumptions: z.array(z.string()), }).optional(), uncertainties: z.array(z.string()), + /** A streamed answer stopped at its length cap; the notice is translated when shown. */ + replyTruncated: z.boolean().optional(), }); export type AskOutput = z.infer; diff --git a/src/components/features/ask/AskChat.tsx b/src/components/features/ask/AskChat.tsx index 8c4816cb6..b786bd6e1 100644 --- a/src/components/features/ask/AskChat.tsx +++ b/src/components/features/ask/AskChat.tsx @@ -288,13 +288,14 @@ const MessageBubble = React.memo(function MessageBubble({ message, onSave }: { m )} {/* Uncertainties */} - {output?.uncertainties && output.uncertainties.length > 0 && ( + {output && (output.uncertainties?.length > 0 || output.replyTruncated) && (
{t('askChat.uncertainties')}
    - {output.uncertainties.map((uncertainty: string, i: number) => ( + {output.uncertainties?.map((uncertainty: string, i: number) => (
  • • {uncertainty}
    • ))} + {output.replyTruncated &&
  • • {t('ask.replyCutOff')}
    • }
)} diff --git a/src/lib/ai/streaming.ts b/src/lib/ai/streaming.ts index 0c1213c65..ec72f1a68 100644 --- a/src/lib/ai/streaming.ts +++ b/src/lib/ai/streaming.ts @@ -1,4 +1,3 @@ -import i18n from '@/i18n'; import { getActiveAccessToken, supabase, SUPABASE_URL } from '@/lib/supabase/client'; import { captureProductEvent } from '@/lib/observability'; import { AIAuthError, AIResponseError, REQUEST_TIMEOUT_MS, TimeoutError, boundAskHistory, boundAskMessage, debateContextForRequest, mapAIError, refreshDebatesIfMissing, reportAIFailure, withOutputLanguage } from './client'; @@ -213,15 +212,8 @@ export async function streamAskChat( contextStore.recordUsage('ask', parseDebateContextUsage(meta?.debateContext)); contextStore.resetOverride('ask'); } - const finalFields = finalEvent.output && typeof finalEvent.output === 'object' && !Array.isArray(finalEvent.output) - ? finalEvent.output as Record - : undefined; - // The server flags an answer cut off at its length cap; say so under it. - const cutOff = meta?.replyTruncated === true && Array.isArray(finalFields?.uncertainties) - ? { uncertainties: [...finalFields.uncertainties, i18n.t('ask.replyCutOff')] } - : {}; - const enrichedOutput = finalFields - ? { ...finalFields, ...cutOff, model: resolvedModel, provider } + const enrichedOutput = finalEvent.output && typeof finalEvent.output === 'object' && !Array.isArray(finalEvent.output) + ? { ...finalEvent.output, model: resolvedModel, provider } : finalEvent.output; finalOutput = acceptFinal(enrichedOutput, finalOutput); } diff --git a/supabase/functions/_shared/output-caps.ts b/supabase/functions/_shared/output-caps.ts index 8854ddc0a..2592a4121 100644 --- a/supabase/functions/_shared/output-caps.ts +++ b/supabase/functions/_shared/output-caps.ts @@ -37,24 +37,31 @@ export type ReplyCap = keyof typeof REPLY_TOKEN_CAPS; export const TRUNCATION_RETRY_FACTOR = 2; /** - * The share of max_tokens a provider may spend on reasoning at each effort, - * and the smallest reasoning budget it allows. These mirror OpenRouter's - * conversion for budget-based models (Anthropic, Gemini): - * budget = max(max_tokens * share, 1024). + * The share of max_tokens a budget-based model (Claude) may spend on + * reasoning at each effort. OpenRouter sets its budget to + * max(max_tokens * share, 1024). * https://openrouter.ai/docs/use-cases/reasoning-tokens */ const REASONING_SHARE: Record = { low: 0.2, medium: 0.5, high: 0.8 }; -const MIN_REASONING_TOKENS = 1_024; /** - * The max_tokens to send so the provider's full reasoning allowance still - * leaves `replyTokens` for the visible reply, bounded by the model's limit. + * Reasoning room kept on top of the reply at each effort. Effort-based models + * (OpenAI, Gemini 3, DeepSeek) decide how long they think and take no budget, + * so this is a floor sized for them; a reply that still runs out is retried. + * The low floor also stays above Claude's 1024-token minimum budget. + */ +const REASONING_HEADROOM: Record = { low: 2_048, medium: 4_096, high: 8_192 }; + +/** + * The max_tokens to send so the reply keeps `replyTokens` after reasoning: + * after Claude's full budget, and after the headroom for every other model. + * Bounded by the model's own limit. */ export function outputTokenLimit( replyTokens: number, effort: ReasoningEffort, modelMaxTokens: number, ): number { - const withShare = Math.ceil(replyTokens / (1 - REASONING_SHARE[effort])); - return Math.min(modelMaxTokens, Math.max(withShare, replyTokens + MIN_REASONING_TOKENS)); + const afterBudget = Math.ceil(replyTokens / (1 - REASONING_SHARE[effort])); + return Math.min(modelMaxTokens, Math.max(afterBudget, replyTokens + REASONING_HEADROOM[effort])); } diff --git a/supabase/functions/ai-stream/index.ts b/supabase/functions/ai-stream/index.ts index ed7ccdcdf..a49a6e850 100644 --- a/supabase/functions/ai-stream/index.ts +++ b/supabase/functions/ai-stream/index.ts @@ -345,7 +345,7 @@ export async function handleRequest(req: Request): Promise { } // The text already reached the reader, so a cut-off answer cannot be - // retried. The client tells the user it stopped short. + // retried. The flag tells the user it stopped short. const replyTruncated = streamFinishReason === "length"; if (replyTruncated) { console.warn(`[AI_STREAM] Reply hit its ask cap (${REPLY_TOKEN_CAPS.ask} tokens, ${model}, effort ${reasoningEffort})`); @@ -359,20 +359,11 @@ export async function handleRequest(req: Request): Promise { generatedAt: new Date().toISOString(), followUpQuestions: [], uncertainties: accumulated.trim() ? [] : ["The model returned an empty streamed response."], + ...(replyTruncated ? { replyTruncated: true } : {}), }; if (!req.signal.aborted) { - try { - emit("final", { - output, - meta: { - requestedModel, - resolvedModel: model, - ...(replyTruncated ? { replyTruncated: true } : {}), - ...(debateMeta ? { debateContext: debateMeta } : {}), - }, - }); - } catch { /* client disconnected after provider completion */ } + try { emit("final", { output, meta: { requestedModel, resolvedModel: model, ...(debateMeta ? { debateContext: debateMeta } : {}) } }); } catch { /* client disconnected after provider completion */ } } await logUsage(user.id, "ask", OPENROUTER_PROVIDER, model, collectUsage(), Date.now() - startedAt); } catch (err) { From dda9bc010686d2a43886dafe63b70ecf6e4a08cc Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 22 Oct 2026 12:53:23 +0000 Subject: [PATCH 3/3] Fail an Ask stream cut off before any text, widen Sparring caps, pin the cap math An Ask stream that spent its whole cap on reasoning now fails so the client falls back to the structured endpoint, which retries with a larger cap. The Sparring speech and scorecard caps fit a full-length Hungarian reply. The cap formula uses integer percentages, and a table test pins it from outside. Co-Authored-By: Claude Opus 5.5 --- .../functions/_shared/output-caps.test.ts | 32 +++++++++++++++++++ supabase/functions/_shared/output-caps.ts | 12 +++---- supabase/functions/ai-stream/index.ts | 7 ++-- supabase/functions/ai-stream/index_test.ts | 19 +++++++++-- 4 files changed, 59 insertions(+), 11 deletions(-) create mode 100644 supabase/functions/_shared/output-caps.test.ts diff --git a/supabase/functions/_shared/output-caps.test.ts b/supabase/functions/_shared/output-caps.test.ts new file mode 100644 index 000000000..78f4f7d09 --- /dev/null +++ b/supabase/functions/_shared/output-caps.test.ts @@ -0,0 +1,32 @@ +// max_tokens covers reasoning and the reply together, so a wrong formula +// leaves a model no room to answer. These pin the formula from outside it. + +import { assertEquals } from "jsr:@std/assert@1.0.19"; +import { outputTokenLimit, REPLY_TOKEN_CAPS, TRUNCATION_RETRY_FACTOR } from "./output-caps.ts"; +import type { ReasoningEffort } from "./model-catalog.ts"; + +// OpenRouter's Claude budget: max(max_tokens * share, 1024). +// https://openrouter.ai/docs/use-cases/reasoning-tokens +const CLAUDE_SHARE: Record = { low: 0.2, medium: 0.5, high: 0.8 }; +const EFFORTS: ReasoningEffort[] = ["low", "medium", "high"]; + +Deno.test("max_tokens for a known reply and effort", () => { + assertEquals(outputTokenLimit(1_500, "low", 128_000), 3_548); + assertEquals(outputTokenLimit(1_500, "medium", 128_000), 5_596); + assertEquals(outputTokenLimit(1_500, "high", 128_000), 9_692); + assertEquals(outputTokenLimit(6_000, "high", 128_000), 30_000); + assertEquals(outputTokenLimit(6_000, "high", 20_000), 20_000); +}); + +Deno.test("every cap leaves its full reply after Claude's reasoning budget and after the floor for other models", () => { + for (const [cap, reply] of Object.entries(REPLY_TOKEN_CAPS)) { + for (const effort of EFFORTS) { + for (const replyTokens of [reply, reply * TRUNCATION_RETRY_FACTOR]) { + const limit = outputTokenLimit(replyTokens, effort, 128_000); + const claudeBudget = Math.max(Math.floor(limit * CLAUDE_SHARE[effort]), 1_024); + if (limit - claudeBudget < replyTokens) throw new Error(`${cap} at ${effort}: Claude keeps ${limit - claudeBudget} of ${replyTokens}`); + if (limit - replyTokens < 2_048) throw new Error(`${cap} at ${effort}: only ${limit - replyTokens} tokens for reasoning`); + } + } + } +}); diff --git a/supabase/functions/_shared/output-caps.ts b/supabase/functions/_shared/output-caps.ts index 2592a4121..c2f4be837 100644 --- a/supabase/functions/_shared/output-caps.ts +++ b/supabase/functions/_shared/output-caps.ts @@ -15,10 +15,10 @@ export const REPLY_TOKEN_CAPS = { crossfireTurn: 1_000, /** The turn-cited Crossfire debrief: six arrays of findings. */ crossfireDebrief: 2_000, - /** One Sparring opponent speech. */ - sparringSpeech: 1_500, + /** One Sparring opponent speech: up to 7 minutes, 6,000 characters. */ + sparringSpeech: 2_500, /** The Sparring scorecard: four categories, moments, areas, verdict. */ - sparringScore: 2_000, + sparringScore: 3_000, /** A full case, any length setting. */ generate: 6_000, attack: 4_000, @@ -37,12 +37,12 @@ export type ReplyCap = keyof typeof REPLY_TOKEN_CAPS; export const TRUNCATION_RETRY_FACTOR = 2; /** - * The share of max_tokens a budget-based model (Claude) may spend on + * The percentage of max_tokens a budget-based model (Claude) may spend on * reasoning at each effort. OpenRouter sets its budget to * max(max_tokens * share, 1024). * https://openrouter.ai/docs/use-cases/reasoning-tokens */ -const REASONING_SHARE: Record = { low: 0.2, medium: 0.5, high: 0.8 }; +const REASONING_SHARE_PERCENT: Record = { low: 20, medium: 50, high: 80 }; /** * Reasoning room kept on top of the reply at each effort. Effort-based models @@ -62,6 +62,6 @@ export function outputTokenLimit( effort: ReasoningEffort, modelMaxTokens: number, ): number { - const afterBudget = Math.ceil(replyTokens / (1 - REASONING_SHARE[effort])); + const afterBudget = Math.ceil((replyTokens * 100) / (100 - REASONING_SHARE_PERCENT[effort])); return Math.min(modelMaxTokens, Math.max(afterBudget, replyTokens + REASONING_HEADROOM[effort])); } diff --git a/supabase/functions/ai-stream/index.ts b/supabase/functions/ai-stream/index.ts index 23388b572..c8eabf8ef 100644 --- a/supabase/functions/ai-stream/index.ts +++ b/supabase/functions/ai-stream/index.ts @@ -344,11 +344,14 @@ export async function handleRequest(req: Request): Promise { throw new Error("OpenRouter stream ended before completion"); } - // The text already reached the reader, so a cut-off answer cannot be - // retried. The flag tells the user it stopped short. + // A cut-off answer that already reached the reader cannot be retried; + // the flag tells the user it stopped short. One that spent the whole + // cap on reasoning sent nothing, so it fails and the client asks the + // structured endpoint, which retries with a larger cap. const replyTruncated = streamFinishReason === "length"; if (replyTruncated) { console.warn(`[AI_STREAM] Reply hit its ask cap (${REPLY_TOKEN_CAPS.ask} tokens, ${model}, effort ${reasoningEffort})`); + if (!accumulated.trim()) throw new Error("OpenRouter reply was cut off before any text"); } const output = { diff --git a/supabase/functions/ai-stream/index_test.ts b/supabase/functions/ai-stream/index_test.ts index 59dbe5dad..e0e0f6636 100644 --- a/supabase/functions/ai-stream/index_test.ts +++ b/supabase/functions/ai-stream/index_test.ts @@ -14,7 +14,7 @@ for (const [key, value] of Object.entries({ const { handleRequest } = await import("./index.ts"); -type Scenario = "complete" | "audit-error" | "truncated" | "provider-error" | "provider-error-after-finish" | "aborted" | "aborted-after-finish"; +type Scenario = "complete" | "audit-error" | "truncated" | "cut-off" | "cut-off-empty" | "provider-error" | "provider-error-after-finish" | "aborted" | "aborted-after-finish"; const usage = { prompt_tokens: 12, @@ -51,6 +51,15 @@ function framesFor(scenario: Exclude): string { usageFrame, ].join(""); } + // Stopped at max_tokens: after some text, or after reasoning only. + if (scenario === "cut-off" || scenario === "cut-off-empty") { + return [ + ...(scenario === "cut-off" ? [`data: ${JSON.stringify({ choices: [{ delta: { content: "Partial" } }] })}\n\n`] : []), + `data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: "length" }] })}\n\n`, + usageFrame, + "data: [DONE]\n\n", + ].join(""); + } if (scenario === "provider-error-after-finish") { return [ usageFrame, @@ -145,7 +154,7 @@ async function runScenario(scenario: Scenario) { } Deno.test("OpenRouter stream accounting bills only completed work", async () => { - for (const scenario of ["complete", "audit-error", "truncated", "provider-error", "provider-error-after-finish", "aborted", "aborted-after-finish"] as const) { + for (const scenario of ["complete", "audit-error", "cut-off", "truncated", "cut-off-empty", "provider-error", "provider-error-after-finish", "aborted", "aborted-after-finish"] as const) { const result = await runScenario(scenario); if (result.providerCalls !== 1) throw new Error(`${scenario}: expected one provider call, got ${result.providerCalls}`); if (result.rows.length !== 1) throw new Error(`${scenario}: expected one audit insert, got ${result.rows.length}`); @@ -160,10 +169,14 @@ Deno.test("OpenRouter stream accounting bills only completed work", async () => throw new Error(`${scenario}: raw provider usage was lost: ${JSON.stringify(row)}`); } - if (scenario === "complete" || scenario === "audit-error") { + if (scenario === "complete" || scenario === "audit-error" || scenario === "cut-off") { if (row.status !== "success" || row.cost_usd !== usage.cost || !result.body.includes("event: final")) { throw new Error(`complete stream was not billed: ${JSON.stringify(row)}\n${result.body}`); } + // A partial answer is kept and flagged; a full one is not. + if (result.body.includes('"replyTruncated":true') !== (scenario === "cut-off")) { + throw new Error(`${scenario}: wrong cut-off flag\n${result.body}`); + } continue; }