From 76f077eb01cf9a2f7c85aab7b9a8aa40f7c97da1 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 22 Oct 2026 11:15:26 +0000 Subject: [PATCH] Trim AI prompts and cache Anthropic system prompts Research prompts list each source once: a page read in full is no longer repeated as a search snippet, a URL several queries found appears once, and at most 12 snippets go in. Fetched pages, including links pasted into Ask, keep 2,000 characters instead of 8,000. Non-streamed Ask keeps the last 6 earlier messages, and only user and assistant turns. Anthropic models get a cache breakpoint on the system prompt. The limits live in supabase/functions/_shared/prompt-budget.ts. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 2 + e2e/full-stack/fake-openrouter.mjs | 25 +++++++--- supabase/functions/_shared/prompt-budget.ts | 50 +++++++++++++++++++ supabase/functions/_shared/research.test.ts | 26 +++++++++- supabase/functions/_shared/research.ts | 38 ++++++++++---- supabase/functions/ai-stream/handler_test.ts | 11 +++- supabase/functions/ai-stream/index.ts | 8 +-- supabase/functions/ai/features/ask.ts | 9 ++-- .../functions/ai/handler-research.test.ts | 16 ++++++ supabase/functions/ai/provider.ts | 3 +- 10 files changed, 157 insertions(+), 31 deletions(-) create mode 100644 supabase/functions/_shared/prompt-budget.ts diff --git a/docs/architecture.md b/docs/architecture.md index 30b508c00..544632f25 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -92,6 +92,8 @@ Shared research, source ranking, URL validation, and prompt construction live un The import endpoint validates public HTTP URLs, resolves redirect targets, blocks private networks, and enforces response timeout, content type, and size limits. Research paths bound source count and content before prompt construction. +[`supabase/functions/_shared/prompt-budget.ts`](../supabase/functions/_shared/prompt-budget.ts) holds the limits that set how many input tokens a request pays for: characters kept per fetched page, search snippets per research prompt, earlier Ask messages, and the Ask context cap. A research prompt lists each source once: a page read in full is not repeated as a search snippet, and a URL that several queries found appears a single time. For `anthropic/*` models the system prompt carries a `cache_control` breakpoint, so a repeat of the same prefix within five minutes is billed at the cache-read price; a prompt below the model's minimum cacheable length is sent uncached at the normal price. + Every AI request in a Current Debate carries a `debateContext` of IDs and preferences, never text: the debate, the side the request prepares, the Library source IDs, and the saved results to read. [`src/lib/debate/context.ts`](../src/lib/debate/context.ts) chooses the default with the rules in [`supabase/functions/_shared/debate-context.ts`](../supabase/functions/_shared/debate-context.ts): the debate's active sources, pinned results first, then up to three of the newest results of the types that feature uses, for the requested side. A compact line above each tool's primary button shows that selection, and Review changes it for one request, or for a whole Crossfire or Sparring round, where Review appears only before the round starts; pins persist in `debate_context_pins`. [`supabase/functions/_shared/debate-context-loader.ts`](../supabase/functions/_shared/debate-context-loader.ts) loads the motion, settings, instructions, sources, and result digests with every query filtered by the caller and the one debate, before quota or provider work. The prompt renders that material in delimited blocks followed by rules that it cannot override. Live sparring stays transcript-only: the scorecard sees only the motion and settings. Crossfire omits free-form instructions. Whole results are dropped when their budget is spent, and a Crossfire round that would pass its 100,000-character prompt budget drops its saved results. In both cases `meta.debateContext` reports what reached the model and what was set aside. A debate deleted on another device returns 404, and the browser reloads its debate list. One-off requests send no debate context. The retired per-user custom system prompt is no longer sent or honored; operator prompts in `global_prompts` remain. Jev scores relevance, importance, and evidence strength after the existing deterministic quality floor, while failures preserve the prior order. Generate also records a bounded source-use snapshot in the saved case output. Semantic counts and the latest case-use flag live in the existing source `metadata` JSON, so the Library can default to Smart ordering without a schema migration. Ordinary Ask remains stateless while a conversation is in progress. The browser sends only the newest eight messages within a UTF-8 byte budget. Inside a debate, each finished exchange is recorded into one chat for that debate; in one-off mode, pressing Save persists the chat and its messages; Workspace threads use their separate durable server-side context path. Legacy local-only sources are not sent to the authenticated AI endpoint because they have no persisted UUID to authorize and load. diff --git a/e2e/full-stack/fake-openrouter.mjs b/e2e/full-stack/fake-openrouter.mjs index 2c036e568..e28c4fc1a 100644 --- a/e2e/full-stack/fake-openrouter.mjs +++ b/e2e/full-stack/fake-openrouter.mjs @@ -176,16 +176,28 @@ const routes = [ ["ask", "You are a debate coach chat bot", ask], ]; +// A cacheable system prompt arrives as text parts instead of a plain string. +function messageText(message) { + const content = message?.content; + if (Array.isArray(content)) return content.map((part) => part?.text ?? "").join(""); + return String(content ?? ""); +} + function classify(body) { const messages = Array.isArray(body.messages) ? body.messages : []; - const system = String(messages.find((message) => message.role === "system")?.content ?? ""); + const systemMessage = messages.find((message) => message.role === "system"); + const system = messageText(systemMessage); const user = String(messages.at(-1)?.content ?? ""); + const shape = { + messageCount: messages.length, + cacheBreakpoint: Array.isArray(systemMessage?.content) && systemMessage.content.some((part) => part?.cache_control), + }; if (body.stream === true && system.includes("Answer in clear Markdown that can stream smoothly")) { - return { feature: "ask-stream", system, user }; + return { feature: "ask-stream", system, user, ...shape }; } const route = routes.find(([, marker]) => system.includes(marker)); - if (!route) return { feature: null, system, user }; - return { feature: route[0], system, user, output: route[2](user, system) }; + if (!route) return { feature: null, system, user, ...shape }; + return { feature: route[0], system, user, ...shape, output: route[2](user, system) }; } function jevAnswers(questions) { @@ -358,8 +370,9 @@ createServer(async (req, res) => { } const body = JSON.parse(await readBody(req)); - const { feature, system, user, output } = classify(body); - requests.push({ feature, model: body.model, stream: body.stream === true, system, user }); + const { output, ...record } = classify(body); + const { feature } = record; + requests.push({ ...record, model: body.model, stream: body.stream === true }); if (!feature) return json(res, 500, { error: { message: "Fake OpenRouter does not recognize this prompt" } }); const failure = failures.find((item) => item.count > 0 && (!item.feature || item.feature === feature)); diff --git a/supabase/functions/_shared/prompt-budget.ts b/supabase/functions/_shared/prompt-budget.ts new file mode 100644 index 000000000..c07fc645f --- /dev/null +++ b/supabase/functions/_shared/prompt-budget.ts @@ -0,0 +1,50 @@ +// Prompt size and caching limits for the AI Edge Functions. Every number here +// sets how many input tokens a request pays for, so raise them deliberately. + +/** Characters kept from each fetched web page: research reads and links pasted into Ask. */ +export const MAX_FETCHED_PAGE_CHARS = 2_000; + +/** Search snippets rendered into a research prompt, besides the pages read in full. */ +export const MAX_RESEARCH_SNIPPETS = 12; + +/** Earlier chat messages an Ask request sends to the model. */ +export const MAX_ASK_HISTORY_MESSAGES = 6; + +/** Characters kept from each earlier Ask message. */ +export const MAX_ASK_HISTORY_MESSAGE_CHARS = 6_000; + +/** Characters of pasted-page and research context a non-streamed Ask request may carry. */ +export const MAX_ASK_CONTEXT_CHARS = 32_000; + +/** + * Mark the system prompt cacheable for Anthropic models. OpenRouter passes the + * breakpoint to Anthropic, which bills a repeat of the same prefix within five + * minutes at a tenth of the input price. Prompts below the model's minimum + * cacheable length are sent uncached at the normal price. + */ +export const ANTHROPIC_PROMPT_CACHING = true; + +type ChatMessage = { role: string; content: string }; +type CacheableMessage = ChatMessage | { + role: string; + content: Array<{ type: "text"; text: string; cache_control: { type: "ephemeral" } }>; +}; + +/** Adds an Anthropic cache breakpoint to the system message; other models get the messages unchanged. */ +export function withPromptCaching(model: string, messages: ChatMessage[]): CacheableMessage[] { + if (!ANTHROPIC_PROMPT_CACHING || !model.startsWith("anthropic/")) return messages; + return messages.map((message) => message.role === "system" + ? { role: "system", content: [{ type: "text", text: message.content, cache_control: { type: "ephemeral" } }] } + : message); +} + +/** The last earlier messages of an Ask conversation, each cut to the per-message cap. */ +export function boundAskHistory(history: unknown): ChatMessage[] { + if (!Array.isArray(history)) return []; + return history.slice(-MAX_ASK_HISTORY_MESSAGES).flatMap((item) => { + if (!item || typeof item !== "object") return []; + const { role, content } = item as Record; + if ((role !== "user" && role !== "assistant") || typeof content !== "string" || !content) return []; + return [{ role, content: content.slice(0, MAX_ASK_HISTORY_MESSAGE_CHARS) }]; + }); +} diff --git a/supabase/functions/_shared/research.test.ts b/supabase/functions/_shared/research.test.ts index 717268ec6..f1bc728d7 100644 --- a/supabase/functions/_shared/research.test.ts +++ b/supabase/functions/_shared/research.test.ts @@ -12,6 +12,7 @@ import { MAX_PROVIDER_RESPONSE_BYTES, type ResearchContext, } from "./research.ts"; +import { MAX_FETCHED_PAGE_CHARS } from "./prompt-budget.ts"; function ctxWith(overrides: Partial): ResearchContext { return { ...emptyResearchContext(), ...overrides }; @@ -19,11 +20,32 @@ function ctxWith(overrides: Partial): ResearchContext { Deno.test("empty research does not render a research context block", () => { const rendered = buildResearchContext(ctxWith({ - searches: [{ query: "empty search", results: "" }], + searches: [{ query: "empty search", results: [] }], })); if (rendered !== "") throw new Error(`empty research rendered unexpected context: ${rendered}`); }); +Deno.test("research context lists each source once and caps page text", () => { + const ranked = (url: string, query: string, qualityScore: number) => ({ + url, query, title: `Title ${qualityScore}`, content: "snippet", sourceType: "news" as const, qualityScore, qualityLabel: "strong" as const, qualityReasons: [], + }); + const read = ranked("https://news.example/read", "first", 90); + const shared = ranked("https://news.example/shared", "first", 80); + const rendered = buildResearchContext(ctxWith({ + searches: [ + { query: "first", results: [read, shared] }, + { query: "second", results: [{ ...shared, query: "second" }, ranked("https://news.example/other", "second", 60)] }, + ], + rankedResults: [read, shared], + readContent: [{ url: read.url, title: "Read", content: "p".repeat(MAX_FETCHED_PAGE_CHARS + 500), result: read }], + })); + const count = (needle: string) => rendered.split(needle).length - 1; + if (count(read.url) !== 1) throw new Error("a page read in full was repeated as a search snippet"); + if (count(shared.url) !== 1) throw new Error("a URL found by two searches was listed twice"); + if (count("https://news.example/other") !== 1) throw new Error("a distinct search result was dropped"); + if (count("p".repeat(MAX_FETCHED_PAGE_CHARS + 1)) !== 0) throw new Error("page text exceeded the configured cap"); +}); + Deno.test("factoid citations backed by strong or moderate fetched sources stay verified", () => { const research = ctxWith({ readContent: [ @@ -242,7 +264,7 @@ Deno.test("Exa contents sends the documented maxCharacters field", async () => { try { await exaGetContents("https://example.com/report"); const text = requestBody?.text as { maxCharacters?: number; max_characters?: number } | undefined; - if (text?.maxCharacters !== 8000) throw new Error("contents maxCharacters was not sent"); + if (text?.maxCharacters !== MAX_FETCHED_PAGE_CHARS) throw new Error("contents maxCharacters was not sent"); if (text?.max_characters !== undefined) throw new Error("legacy contents casing leaked into request"); } finally { globalThis.fetch = originalFetch; diff --git a/supabase/functions/_shared/research.ts b/supabase/functions/_shared/research.ts index 9dc695470..fef52000c 100644 --- a/supabase/functions/_shared/research.ts +++ b/supabase/functions/_shared/research.ts @@ -18,6 +18,7 @@ import { export type { ExaResult, RankedSearchResult, ResearchSource }; import type { ReasoningEffort } from "./model-catalog.ts"; +import { MAX_FETCHED_PAGE_CHARS, MAX_RESEARCH_SNIPPETS } from "./prompt-budget.ts"; export type Provider = "openrouter"; @@ -35,7 +36,7 @@ export type Source = { }; export interface ResearchContext { - searches: Array<{ query: string; results: string }>; + searches: Array<{ query: string; results: RankedSearchResult[] }>; rankedResults: RankedSearchResult[]; readContent: Array<{ url: string; title: string; content: string; result?: RankedSearchResult }>; verifiedUrls: Set; @@ -298,8 +299,8 @@ export async function exaSearch(search: string | ExaSearchRequest, signal?: Abor } } -// exaGetContents: extract full text from a specific URL using Exa's /contents endpoint -export async function exaGetContents(url: string, signal?: AbortSignal): Promise { +// exaGetContents: extract a page's text, up to maxChars, using Exa's /contents endpoint +export async function exaGetContents(url: string, signal?: AbortSignal, maxChars = MAX_FETCHED_PAGE_CHARS): Promise { const apiKey = Deno.env.get("EXA_API_KEY"); if (!apiKey) { console.warn("[EXA] EXA_API_KEY not configured"); @@ -310,7 +311,7 @@ export async function exaGetContents(url: string, signal?: AbortSignal): Promise const { response, data } = await fetchJsonWithTimeout<{ results?: unknown }>("https://api.exa.ai/contents", { method: "POST", headers: { "Content-Type": "application/json", "x-api-key": apiKey }, - body: JSON.stringify({ urls: [url], text: { maxCharacters: 8000 } }), + body: JSON.stringify({ urls: [url], text: { maxCharacters: maxChars } }), }, 30000, signal); throwIfAborted(signal); if (!response.ok) { @@ -318,7 +319,7 @@ export async function exaGetContents(url: string, signal?: AbortSignal): Promise return ""; } if (!data?.results || !Array.isArray(data.results) || data.results.length === 0) return ""; - return (data.results[0]?.text || "").slice(0, 8000); + return (data.results[0]?.text || "").slice(0, maxChars); } catch (err) { if (signal?.aborted) throw err; console.warn(`[EXA] Contents fetch failed urlLength=${url.length}`); @@ -464,7 +465,7 @@ async function deepResearchInner( throwIfAborted(signal); const ranked = rankSearchResults(results.map((result) => ({ ...result, query })), claim, preferredSourceType); const visibleRanked = preferredSourceType === "any" ? ranked : ranked.filter(matchesPreferredSourceType); - context.searches.push({ query, results: formatRankedSearchResults(visibleRanked) }); + context.searches.push({ query, results: visibleRanked }); console.log(`[RESEARCH] Seed queryLength=${query.length}: ${results.length} results`); return results.map((result) => ({ ...result, query })); })); @@ -484,7 +485,7 @@ async function deepResearchInner( throwIfAborted(signal); const ranked = rankSearchResults(results.map((result) => ({ ...result, query })), claim, preferredSourceType); const visibleRanked = preferredSourceType === "any" ? ranked : ranked.filter(matchesPreferredSourceType); - context.searches.push({ query, results: formatRankedSearchResults(visibleRanked) }); + context.searches.push({ query, results: visibleRanked }); allResults.push(...results.map((result) => ({ ...result, query }))); return results.length; }); @@ -574,9 +575,24 @@ export function buildResearchContext(research: ResearchContext): string { let ctx = "=== WEB RESEARCH RESULTS ===\n\n"; ctx += "IMPORTANT: Only cite sources marked 'strong' or 'moderate'. Sources marked 'weak' are fallback context only, do NOT cite them as evidence. If only weak sources are available, state that evidence is limited.\n\n"; - // Add search results - for (const search of research.searches) { - ctx += `--- Search: "${search.query}" ---\n${search.results}\n\n`; + // Each source appears once. A page read in full is not repeated as a search + // snippet, and a URL several queries found is listed a single time. + const listed = new Set(research.readContent.map((page) => page.url)); + const snippets = research.searches + .flatMap((search) => search.results) + .sort((a, b) => b.qualityScore - a.qualityScore) + .filter((result) => { + if (!result.url || listed.has(result.url)) return false; + listed.add(result.url); + return true; + }) + .slice(0, MAX_RESEARCH_SNIPPETS); + + if (research.searches.length > 0) { + ctx += `Searches run: ${research.searches.map((search) => `"${search.query}"`).join("; ")}\n\n`; + } + if (snippets.length > 0) { + ctx += `--- Search results ---\n${formatRankedSearchResults(snippets)}\n\n`; } // Add full page content @@ -585,7 +601,7 @@ export function buildResearchContext(research: ResearchContext): string { for (const page of research.readContent) { const quality = page.result ? `${page.result.sourceType}, ${page.result.qualityLabel}, ${page.result.qualityScore}/100` : "verified"; const warning = page.result?.qualityLabel === "weak" ? " [WEAK, do not cite as evidence]" : ""; - ctx += `--- From: ${page.title} (${page.url}) ---\nSource quality: ${quality}${warning}\n${page.content.slice(0, 4000)}\n\n`; + ctx += `--- From: ${page.title} (${page.url}) ---\nSource quality: ${quality}${warning}\n${page.content.slice(0, MAX_FETCHED_PAGE_CHARS)}\n\n`; } } diff --git a/supabase/functions/ai-stream/handler_test.ts b/supabase/functions/ai-stream/handler_test.ts index 54e0c9ba5..e0430bc8b 100644 --- a/supabase/functions/ai-stream/handler_test.ts +++ b/supabase/functions/ai-stream/handler_test.ts @@ -155,7 +155,10 @@ Deno.test("streaming Ask forwards the shared prompt and returns provider SSE", a assertIncludes(body, '"requestedModel":"auto-fast"'); assertIncludes(body, '"resolvedModel":"anthropic/claude-haiku-5.5"'); - const messages = providerRequest?.messages as Array<{ role: string; content: string }> | undefined; + const messages = providerRequest?.messages as Array<{ + role: string; + content: string | Array<{ text: string; cache_control?: unknown }>; + }> | undefined; if (providerRequest?.model !== "anthropic/claude-haiku-5.5") { throw new Error(`unexpected OpenRouter model: ${String(providerRequest?.model)}`); } @@ -170,7 +173,11 @@ Deno.test("streaming Ask forwards the shared prompt and returns provider SSE", a if ("sort" in preferences || "order" in preferences) { throw new Error(`sort/order disables OpenRouter balanced routing: ${JSON.stringify(preferences)}`); } - const system = messages?.find((message) => message.role === "system")?.content ?? ""; + const systemContent = messages?.find((message) => message.role === "system")?.content; + if (!Array.isArray(systemContent) || !systemContent.some((part) => part.cache_control)) { + throw new Error("Anthropic system prompt was sent without a cache breakpoint"); + } + const system = systemContent.map((part) => part.text).join(""); assertIncludes(system, "Answer in clear Markdown that can stream smoothly token by token."); assertIncludes(system, "Follow any explicit output-language instruction."); assertIncludes(system, "Otherwise match the language used by the user."); diff --git a/supabase/functions/ai-stream/index.ts b/supabase/functions/ai-stream/index.ts index f595da149..7fbf56efe 100644 --- a/supabase/functions/ai-stream/index.ts +++ b/supabase/functions/ai-stream/index.ts @@ -22,6 +22,7 @@ import { HttpError, jsonError, requireUser } from "../_shared/auth.ts"; import { getCorsHeaders } from "../_shared/cors.ts"; import { checkRateLimit, logUsage } from "../_shared/entitlements.ts"; import { OPENROUTER_API_URL } from "../_shared/research.ts"; +import { MAX_ASK_HISTORY_MESSAGE_CHARS, MAX_ASK_HISTORY_MESSAGES, withPromptCaching } from "../_shared/prompt-budget.ts"; import { AUTO_PRO, DEFAULT_MODEL, @@ -51,7 +52,6 @@ const supabaseServiceKey = Deno.env.get("SUPABASE_SERVICE_ROLE_KEY")!; const supabase = createClient(supabaseUrl, supabaseServiceKey); const MAX_INPUT_LENGTH = 50000; -const MAX_HISTORY_ITEMS = 8; const OPENROUTER_PROVIDER = "openrouter" as const; function jsonResponse(body: Record, status: number, corsHeaders: Record) { @@ -73,10 +73,10 @@ function sanitizeText(value: unknown, limit: number) { function normalizeHistory(value: unknown): ChatMessage[] { if (!Array.isArray(value)) return []; - return value.slice(-MAX_HISTORY_ITEMS).flatMap((item) => { + return value.slice(-MAX_ASK_HISTORY_MESSAGES).flatMap((item) => { if (!item || typeof item !== "object") return []; const role = (item as Record).role; - const content = sanitizeText((item as Record).content, 6000); + const content = sanitizeText((item as Record).content, MAX_ASK_HISTORY_MESSAGE_CHARS); if ((role === "user" || role === "assistant") && content) return [{ role, content }]; return []; }); @@ -264,7 +264,7 @@ export async function handleRequest(req: Request): Promise { const payload: Record = { model, - messages, + messages: withPromptCaching(model, messages), max_tokens: Math.min(modelEntry.maxTokens, MAX_REQUEST_OUTPUT_TOKENS), reasoning: { effort: reasoningEffort, exclude: true }, // Omitting sort and order keeps OpenRouter's balanced provider load balancing. diff --git a/supabase/functions/ai/features/ask.ts b/supabase/functions/ai/features/ask.ts index 9211efbfb..ef3f0c229 100644 --- a/supabase/functions/ai/features/ask.ts +++ b/supabase/functions/ai/features/ask.ts @@ -15,15 +15,14 @@ import { type ResearchSource, } from "../../_shared/research.ts"; import { enforceAskControls } from "../../_shared/control-enforcement.ts"; +import { boundAskHistory, MAX_ASK_CONTEXT_CHARS } from "../../_shared/prompt-budget.ts"; import { hasResearch, type FeatureContext } from "./types.ts"; -const MAX_ASK_CONTEXT_LENGTH = 32000; - // ASK: Optionally run web search and include URL context from chat export async function handleAsk(ctx: FeatureContext): Promise { const feature = "ask" as const; const { userId, provider, model, reasoningEffort, workSignal, researchDeps, startTime, researchUsageForLog, recordMainResult, buildSystemPromptFor, input: sanitizedInput } = ctx; - const history = (sanitizedInput.history as Array<{ role: string; content: string }>) || []; + const history = boundAskHistory(sanitizedInput.history); const message = String(sanitizedInput.message || ""); const mode = String(sanitizedInput.mode || "general"); const citationMode = Boolean(sanitizedInput.citationMode); @@ -44,7 +43,7 @@ export async function handleAsk(ctx: FeatureContext): Promise { const contents = await Promise.all(Array.from(urls).slice(0, 3).map(async (u) => { const content = await exaGetContents(u, workSignal); return content.trim() - ? `--- Content from ${u} ---\n${content.slice(0, 8000)}\n--- End of ${u} ---` + ? `--- Content from ${u} ---\n${content}\n--- End of ${u} ---` : ""; })); const usableContents = contents.filter(Boolean); @@ -79,7 +78,7 @@ export async function handleAsk(ctx: FeatureContext): Promise { researchAvailable: Boolean(webSearchContext || urlContext), }; const initialBasePrompt = buildAskBasePrompt(askPromptInput); - const contextBudget = Math.max(0, MAX_ASK_CONTEXT_LENGTH - initialBasePrompt.length); + const contextBudget = Math.max(0, MAX_ASK_CONTEXT_CHARS - initialBasePrompt.length); const webSearchBudget = Math.max(0, contextBudget - urlContext.length); const trimmedWebSearch = truncateTextToBudget(webSearchContext, webSearchBudget); webSearchContext = trimmedWebSearch.text; diff --git a/supabase/functions/ai/handler-research.test.ts b/supabase/functions/ai/handler-research.test.ts index d9ab9383c..0fbc2db65 100644 --- a/supabase/functions/ai/handler-research.test.ts +++ b/supabase/functions/ai/handler-research.test.ts @@ -19,6 +19,7 @@ for (const [key, value] of Object.entries(environment)) Deno.env.set(key, value) const { createRequestBudgetSignal, handleRequest } = await import("./index.ts"); const { handleRequest: handleStreamRequest } = await import("../ai-stream/index.ts"); const { MAX_REQUEST_OUTPUT_TOKENS, MODEL_CATALOG } = await import("../_shared/model-catalog.ts"); +const { MAX_ASK_HISTORY_MESSAGES } = await import("../_shared/prompt-budget.ts"); Deno.test("AI request work shares one deadline signal", () => { const caller = new AbortController(); @@ -394,6 +395,7 @@ Deno.test("Ask marks only surviving URL and research context as available", asyn return { system: String(messages.find((message) => message.role === "system")?.content ?? ""), user: String(messages.at(-1)?.content ?? ""), + messages, body: await response.json(), }; }; @@ -405,6 +407,20 @@ Deno.test("Ask marks only surviving URL and research context as available", asyn throw new Error("empty search context leaked into the Ask provider prompt"); } + // Only the latest earlier messages reach the model, and only as chat turns. + const history = Array.from({ length: 10 }, (_, index) => ({ + role: index === 5 ? "system" : index % 2 === 0 ? "user" : "assistant", + content: `Earlier turn ${index}`, + })); + const longChat = await runAsk({ message: "And the counterargument?", mode: "general", history }); + const earlier = longChat.messages.slice(1, -1); + if (earlier.length !== MAX_ASK_HISTORY_MESSAGES - 1 || earlier.some((message) => message.role === "system")) { + throw new Error(`Ask sent unexpected history: ${JSON.stringify(earlier)}`); + } + if (earlier[0].content !== "Earlier turn 4" || earlier.at(-1)?.content !== "Earlier turn 9") { + throw new Error("Ask history did not keep the latest messages"); + } + scenario = "empty-url"; const emptyUrl = await runAsk({ message: "Read https://example.com/empty", mode: "general", citationMode: false }); if (!emptyUrl.user.includes("Research Context: NONE") || emptyUrl.user.includes("WEB PAGE CONTEXT:")) { diff --git a/supabase/functions/ai/provider.ts b/supabase/functions/ai/provider.ts index 52e01bcda..8c40c09ba 100644 --- a/supabase/functions/ai/provider.ts +++ b/supabase/functions/ai/provider.ts @@ -22,6 +22,7 @@ import { type Provider, readResponseText, } from "../_shared/research.ts"; +import { withPromptCaching } from "../_shared/prompt-budget.ts"; export const OPENROUTER_PROVIDER: Provider = "openrouter"; @@ -234,7 +235,7 @@ export async function callOpenRouter( const modelEntry = modelForRequest(model); const payload: Record = { model, - messages, + messages: withPromptCaching(model, messages), max_tokens: Math.min(modelEntry.maxTokens, MAX_REQUEST_OUTPUT_TOKENS), response_format: { type: "json_object" }, reasoning: { effort: reasoningEffort, exclude: true },