Trim AI prompts and cache Anthropic system prompts

docs/architecture.md

@@ -94,6 +94,8 @@ Research makes two small model calls of its own: Ask decides whether a question
 
 The import endpoint validates public HTTP URLs, resolves redirect targets, blocks private networks, and enforces response timeout, content type, and size limits. Research paths bound source count and content before prompt construction.
 
+[`supabase/functions/_shared/prompt-budget.ts`](../supabase/functions/_shared/prompt-budget.ts) holds the limits that set how many input tokens a request pays for: characters kept per fetched page, search snippets per research prompt, earlier Ask messages, and the Ask context cap. A research prompt lists each source once: a page read in full is not repeated as a search snippet, and a URL that several queries found appears a single time. For `anthropic/*` models the system prompt carries a `cache_control` breakpoint, so a repeat of the same prefix within five minutes is billed at the cache-read price; a prompt below the model's minimum cacheable length is sent uncached at the normal price.
+
 Every AI request in a Current Debate carries a `debateContext` of IDs and preferences, never text: the debate, the side the request prepares, the Library source IDs, and the saved results to read. [`src/lib/debate/context.ts`](../src/lib/debate/context.ts) chooses the default with the rules in [`supabase/functions/_shared/debate-context.ts`](../supabase/functions/_shared/debate-context.ts): the debate's active sources, pinned results first, then up to three of the newest results of the types that feature uses, for the requested side. A compact line above each tool's primary button shows that selection, and Review changes it for one request, or for a whole Crossfire or Sparring round, where Review appears only before the round starts; pins persist in `debate_context_pins`. [`supabase/functions/_shared/debate-context-loader.ts`](../supabase/functions/_shared/debate-context-loader.ts) loads the motion, settings, instructions, sources, and result digests with every query filtered by the caller and the one debate, before quota or provider work. The prompt renders that material in delimited blocks followed by rules that it cannot override. Live sparring stays transcript-only: the scorecard sees only the motion and settings. Crossfire omits free-form instructions. Whole results are dropped when their budget is spent, and a Crossfire round that would pass its 100,000-character prompt budget drops its saved results. In both cases `meta.debateContext` reports what reached the model and what was set aside. A debate deleted on another device returns 404, and the browser reloads its debate list. One-off requests send no debate context. The retired per-user custom system prompt is no longer sent or honored; operator prompts in `global_prompts` remain. Jev scores relevance, importance, and evidence strength after the existing deterministic quality floor, while failures preserve the prior order. Generate also records a bounded source-use snapshot in the saved case output. Semantic counts and the latest case-use flag live in the existing source `metadata` JSON, so the Library can default to Smart ordering without a schema migration.
 
 Ordinary Ask remains stateless while a conversation is in progress. The browser sends only the newest eight messages within a UTF-8 byte budget. Inside a debate, each finished exchange is recorded into one chat for that debate; in one-off mode, pressing Save persists the chat and its messages; Workspace threads use their separate durable server-side context path. Legacy local-only sources are not sent to the authenticated AI endpoint because they have no persisted UUID to authorize and load.

e2e/full-stack/fake-openrouter.mjs

@@ -176,16 +176,28 @@ const routes = [
   ["ask", "You are a debate coach chat bot", ask],
 ];
 
+// A cacheable system prompt arrives as text parts instead of a plain string.
+function messageText(message) {
+  const content = message?.content;
+  if (Array.isArray(content)) return content.map((part) => part?.text ?? "").join("");
+  return String(content ?? "");
+}
+
 function classify(body) {
   const messages = Array.isArray(body.messages) ? body.messages : [];
-  const system = String(messages.find((message) => message.role === "system")?.content ?? "");
+  const systemMessage = messages.find((message) => message.role === "system");
+  const system = messageText(systemMessage);
   const user = String(messages.at(-1)?.content ?? "");
+  const shape = {
+    messageCount: messages.length,
+    cacheBreakpoint: Array.isArray(systemMessage?.content) && systemMessage.content.some((part) => part?.cache_control),
+  };
   if (body.stream === true && system.includes("Answer in clear Markdown that can stream smoothly")) {
-    return { feature: "ask-stream", system, user };
+    return { feature: "ask-stream", system, user, ...shape };
   }
   const route = routes.find(([, marker]) => system.includes(marker));
-  if (!route) return { feature: null, system, user };
-  return { feature: route[0], system, user, output: route[2](user, system) };
+  if (!route) return { feature: null, system, user, ...shape };
+  return { feature: route[0], system, user, ...shape, output: route[2](user, system) };
 }
 
 function jevAnswers(questions) {
@@ -358,8 +370,9 @@ createServer(async (req, res) => {
   }
 
   const body = JSON.parse(await readBody(req));
-  const { feature, system, user, output } = classify(body);
-  requests.push({ feature, model: body.model, stream: body.stream === true, system, user });
+  const { output, ...record } = classify(body);
+  const { feature } = record;
+  requests.push({ ...record, model: body.model, stream: body.stream === true });
   if (!feature) return json(res, 500, { error: { message: "Fake OpenRouter does not recognize this prompt" } });
 
   const failure = failures.find((item) => item.count > 0 && (!item.feature || item.feature === feature));

supabase/functions/_shared/prompt-budget.ts

@@ -0,0 +1,50 @@
+// Prompt size and caching limits for the AI Edge Functions. Every number here
+// sets how many input tokens a request pays for, so raise them deliberately.
+
+/** Characters kept from each fetched web page: research reads and links pasted into Ask. */
+export const MAX_FETCHED_PAGE_CHARS = 2_000;
+
+/** Search snippets rendered into a research prompt, besides the pages read in full. */
+export const MAX_RESEARCH_SNIPPETS = 12;
+
+/** Earlier chat messages an Ask request sends to the model. */
+export const MAX_ASK_HISTORY_MESSAGES = 6;
+
+/** Characters kept from each earlier Ask message. */
+export const MAX_ASK_HISTORY_MESSAGE_CHARS = 6_000;
+
+/** Characters of pasted-page and research context a non-streamed Ask request may carry. */
+export const MAX_ASK_CONTEXT_CHARS = 32_000;
+
+/**
+ * Mark the system prompt cacheable for Anthropic models. OpenRouter passes the
+ * breakpoint to Anthropic, which bills a repeat of the same prefix within five
+ * minutes at a tenth of the input price. Prompts below the model's minimum
+ * cacheable length are sent uncached at the normal price.
+ */
+export const ANTHROPIC_PROMPT_CACHING = true;
+
+type ChatMessage = { role: string; content: string };
+type CacheableMessage = ChatMessage | {
+  role: string;
+  content: Array<{ type: "text"; text: string; cache_control: { type: "ephemeral" } }>;
+};
+
+/** Adds an Anthropic cache breakpoint to the system message; other models get the messages unchanged. */
+export function withPromptCaching(model: string, messages: ChatMessage[]): CacheableMessage[] {
+  if (!ANTHROPIC_PROMPT_CACHING || !model.startsWith("anthropic/")) return messages;
+  return messages.map((message) => message.role === "system"
+    ? { role: "system", content: [{ type: "text", text: message.content, cache_control: { type: "ephemeral" } }] }
+    : message);
+}
+
+/** The last earlier messages of an Ask conversation, each cut to the per-message cap. */
+export function boundAskHistory(history: unknown): ChatMessage[] {
+  if (!Array.isArray(history)) return [];
+  return history.slice(-MAX_ASK_HISTORY_MESSAGES).flatMap((item) => {
+    if (!item || typeof item !== "object") return [];
+    const { role, content } = item as Record<string, unknown>;
+    if ((role !== "user" && role !== "assistant") || typeof content !== "string" || !content) return [];
+    return [{ role, content: content.slice(0, MAX_ASK_HISTORY_MESSAGE_CHARS) }];
+  });
+}

supabase/functions/_shared/research.test.ts

@@ -12,18 +12,40 @@ import {
   MAX_PROVIDER_RESPONSE_BYTES,
   type ResearchContext,
 } from "./research.ts";
+import { MAX_FETCHED_PAGE_CHARS } from "./prompt-budget.ts";
 
 function ctxWith(overrides: Partial<ResearchContext>): ResearchContext {
   return { ...emptyResearchContext(), ...overrides };
 }
 
 Deno.test("empty research does not render a research context block", () => {
   const rendered = buildResearchContext(ctxWith({
-    searches: [{ query: "empty search", results: "" }],
+    searches: [{ query: "empty search", results: [] }],
   }));
   if (rendered !== "") throw new Error(`empty research rendered unexpected context: ${rendered}`);
 });
 
+Deno.test("research context lists each source once and caps page text", () => {
+  const ranked = (url: string, query: string, qualityScore: number) => ({
+    url, query, title: `Title ${qualityScore}`, content: "snippet", sourceType: "news" as const, qualityScore, qualityLabel: "strong" as const, qualityReasons: [],
+  });
+  const read = ranked("https://news.example/read", "first", 90);
+  const shared = ranked("https://news.example/shared", "first", 80);
+  const rendered = buildResearchContext(ctxWith({
+    searches: [
+      { query: "first", results: [read, shared] },
+      { query: "second", results: [{ ...shared, query: "second" }, ranked("https://news.example/other", "second", 60)] },
+    ],
+    rankedResults: [read, shared],
+    readContent: [{ url: read.url, title: "Read", content: "p".repeat(MAX_FETCHED_PAGE_CHARS + 500), result: read }],
+  }));
+  const count = (needle: string) => rendered.split(needle).length - 1;
+  if (count(read.url) !== 1) throw new Error("a page read in full was repeated as a search snippet");
+  if (count(shared.url) !== 1) throw new Error("a URL found by two searches was listed twice");
+  if (count("https://news.example/other") !== 1) throw new Error("a distinct search result was dropped");
+  if (count("p".repeat(MAX_FETCHED_PAGE_CHARS + 1)) !== 0) throw new Error("page text exceeded the configured cap");
+});
+
 Deno.test("factoid citations backed by strong or moderate fetched sources stay verified", () => {
   const research = ctxWith({
     readContent: [
@@ -242,7 +264,7 @@ Deno.test("Exa contents sends the documented maxCharacters field", async () => {
   try {
     await exaGetContents("https://example.com/report");
     const text = requestBody?.text as { maxCharacters?: number; max_characters?: number } | undefined;
-    if (text?.maxCharacters !== 8000) throw new Error("contents maxCharacters was not sent");
+    if (text?.maxCharacters !== MAX_FETCHED_PAGE_CHARS) throw new Error("contents maxCharacters was not sent");
     if (text?.max_characters !== undefined) throw new Error("legacy contents casing leaked into request");
   } finally {
     globalThis.fetch = originalFetch;

supabase/functions/_shared/research.ts

@@ -18,6 +18,7 @@ import {
 export type { ExaResult, RankedSearchResult, ResearchSource };
 
 import type { ReasoningEffort } from "./model-catalog.ts";
+import { MAX_FETCHED_PAGE_CHARS, MAX_RESEARCH_SNIPPETS } from "./prompt-budget.ts";
 
 export type Provider = "openrouter";
 
@@ -35,7 +36,7 @@ export type Source = {
 };
 
 export interface ResearchContext {
-  searches: Array<{ query: string; results: string }>;
+  searches: Array<{ query: string; results: RankedSearchResult[] }>;
   rankedResults: RankedSearchResult[];
   readContent: Array<{ url: string; title: string; content: string; result?: RankedSearchResult }>;
   verifiedUrls: Set<string>;
@@ -298,8 +299,8 @@ export async function exaSearch(search: string | ExaSearchRequest, signal?: Abor
   }
 }
 
-// exaGetContents: extract full text from a specific URL using Exa's /contents endpoint
-export async function exaGetContents(url: string, signal?: AbortSignal): Promise<string> {
+// exaGetContents: extract a page's text, up to maxChars, using Exa's /contents endpoint
+export async function exaGetContents(url: string, signal?: AbortSignal, maxChars = MAX_FETCHED_PAGE_CHARS): Promise<string> {
   const apiKey = Deno.env.get("EXA_API_KEY");
   if (!apiKey) {
     console.warn("[EXA] EXA_API_KEY not configured");
@@ -310,15 +311,15 @@ export async function exaGetContents(url: string, signal?: AbortSignal): Promise
     const { response, data } = await fetchJsonWithTimeout<{ results?: unknown }>("https://api.exa.ai/contents", {
       method: "POST",
       headers: { "Content-Type": "application/json", "x-api-key": apiKey },
-      body: JSON.stringify({ urls: [url], text: { maxCharacters: 8000 } }),
+      body: JSON.stringify({ urls: [url], text: { maxCharacters: maxChars } }),
     }, 30000, signal);
     throwIfAborted(signal);
     if (!response.ok) {
       console.warn(`[EXA] Contents fetch failed (${response.status}) urlLength=${url.length}`);
       return "";
     }
     if (!data?.results || !Array.isArray(data.results) || data.results.length === 0) return "";
-    return (data.results[0]?.text || "").slice(0, 8000);
+    return (data.results[0]?.text || "").slice(0, maxChars);
   } catch (err) {
     if (signal?.aborted) throw err;
     console.warn(`[EXA] Contents fetch failed urlLength=${url.length}`);
@@ -464,7 +465,7 @@ async function deepResearchInner(
     throwIfAborted(signal);
     const ranked = rankSearchResults(results.map((result) => ({ ...result, query })), claim, preferredSourceType);
     const visibleRanked = preferredSourceType === "any" ? ranked : ranked.filter(matchesPreferredSourceType);
-    context.searches.push({ query, results: formatRankedSearchResults(visibleRanked) });
+    context.searches.push({ query, results: visibleRanked });
     console.log(`[RESEARCH] Seed queryLength=${query.length}: ${results.length} results`);
     return results.map((result) => ({ ...result, query }));
   }));
@@ -484,7 +485,7 @@ async function deepResearchInner(
       throwIfAborted(signal);
       const ranked = rankSearchResults(results.map((result) => ({ ...result, query })), claim, preferredSourceType);
       const visibleRanked = preferredSourceType === "any" ? ranked : ranked.filter(matchesPreferredSourceType);
-      context.searches.push({ query, results: formatRankedSearchResults(visibleRanked) });
+      context.searches.push({ query, results: visibleRanked });
       allResults.push(...results.map((result) => ({ ...result, query })));
       return results.length;
     });
@@ -574,9 +575,24 @@ export function buildResearchContext(research: ResearchContext): string {
   let ctx = "=== WEB RESEARCH RESULTS ===\n\n";
   ctx += "IMPORTANT: Only cite sources marked 'strong' or 'moderate'. Sources marked 'weak' are fallback context only, do NOT cite them as evidence. If only weak sources are available, state that evidence is limited.\n\n";
 
-  // Add search results
-  for (const search of research.searches) {
-    ctx += `--- Search: "${search.query}" ---\n${search.results}\n\n`;
+  // Each source appears once. A page read in full is not repeated as a search
+  // snippet, and a URL several queries found is listed a single time.
+  const listed = new Set(research.readContent.map((page) => page.url));
+  const snippets = research.searches
+    .flatMap((search) => search.results)
+    .sort((a, b) => b.qualityScore - a.qualityScore)
+    .filter((result) => {
+      if (!result.url || listed.has(result.url)) return false;
+      listed.add(result.url);
+      return true;
+    })
+    .slice(0, MAX_RESEARCH_SNIPPETS);
+
+  if (research.searches.length > 0) {
+    ctx += `Searches run: ${research.searches.map((search) => `"${search.query}"`).join("; ")}\n\n`;
+  }
+  if (snippets.length > 0) {
+    ctx += `--- Search results ---\n${formatRankedSearchResults(snippets)}\n\n`;
   }
 
   // Add full page content
@@ -585,7 +601,7 @@ export function buildResearchContext(research: ResearchContext): string {
     for (const page of research.readContent) {
       const quality = page.result ? `${page.result.sourceType}, ${page.result.qualityLabel}, ${page.result.qualityScore}/100` : "verified";
       const warning = page.result?.qualityLabel === "weak" ? " [WEAK, do not cite as evidence]" : "";
-      ctx += `--- From: ${page.title} (${page.url}) ---\nSource quality: ${quality}${warning}\n${page.content.slice(0, 4000)}\n\n`;
+      ctx += `--- From: ${page.title} (${page.url}) ---\nSource quality: ${quality}${warning}\n${page.content.slice(0, MAX_FETCHED_PAGE_CHARS)}\n\n`;
     }
   }
 

supabase/functions/ai-stream/handler_test.ts

@@ -155,7 +155,10 @@ Deno.test("streaming Ask forwards the shared prompt and returns provider SSE", a
     assertIncludes(body, '"requestedModel":"auto-fast"');
     assertIncludes(body, '"resolvedModel":"anthropic/claude-haiku-5.5"');
 
-    const messages = providerRequest?.messages as Array<{ role: string; content: string }> | undefined;
+    const messages = providerRequest?.messages as Array<{
+      role: string;
+      content: string | Array<{ text: string; cache_control?: unknown }>;
+    }> | undefined;
     if (providerRequest?.model !== "anthropic/claude-haiku-5.5") {
       throw new Error(`unexpected OpenRouter model: ${String(providerRequest?.model)}`);
     }
@@ -170,7 +173,11 @@ Deno.test("streaming Ask forwards the shared prompt and returns provider SSE", a
     if ("sort" in preferences || "order" in preferences) {
       throw new Error(`sort/order disables OpenRouter balanced routing: ${JSON.stringify(preferences)}`);
     }
-    const system = messages?.find((message) => message.role === "system")?.content ?? "";
+    const systemContent = messages?.find((message) => message.role === "system")?.content;
+    if (!Array.isArray(systemContent) || !systemContent.some((part) => part.cache_control)) {
+      throw new Error("Anthropic system prompt was sent without a cache breakpoint");
+    }
+    const system = systemContent.map((part) => part.text).join("");
     assertIncludes(system, "Answer in clear Markdown that can stream smoothly token by token.");
     assertIncludes(system, "Follow any explicit output-language instruction.");
     assertIncludes(system, "Otherwise match the language used by the user.");

supabase/functions/ai-stream/index.ts

@@ -22,6 +22,7 @@ import { HttpError, jsonError, requireUser } from "../_shared/auth.ts";
 import { getCorsHeaders } from "../_shared/cors.ts";
 import { checkRateLimit, logUsage } from "../_shared/entitlements.ts";
 import { OPENROUTER_API_URL } from "../_shared/research.ts";
+import { MAX_ASK_HISTORY_MESSAGE_CHARS, MAX_ASK_HISTORY_MESSAGES, withPromptCaching } from "../_shared/prompt-budget.ts";
 import {
   AUTO_PRO,
   DEFAULT_MODEL,
@@ -51,7 +52,6 @@ const supabaseServiceKey = Deno.env.get("SUPABASE_SERVICE_ROLE_KEY")!;
 const supabase = createClient(supabaseUrl, supabaseServiceKey);
 
 const MAX_INPUT_LENGTH = 50000;
-const MAX_HISTORY_ITEMS = 8;
 const OPENROUTER_PROVIDER = "openrouter" as const;
 
 function jsonResponse(body: Record<string, unknown>, status: number, corsHeaders: Record<string, string>) {
@@ -73,10 +73,10 @@ function sanitizeText(value: unknown, limit: number) {
 
 function normalizeHistory(value: unknown): ChatMessage[] {
   if (!Array.isArray(value)) return [];
-  return value.slice(-MAX_HISTORY_ITEMS).flatMap((item) => {
+  return value.slice(-MAX_ASK_HISTORY_MESSAGES).flatMap((item) => {
     if (!item || typeof item !== "object") return [];
     const role = (item as Record<string, unknown>).role;
-    const content = sanitizeText((item as Record<string, unknown>).content, 6000);
+    const content = sanitizeText((item as Record<string, unknown>).content, MAX_ASK_HISTORY_MESSAGE_CHARS);
     if ((role === "user" || role === "assistant") && content) return [{ role, content }];
     return [];
   });
@@ -264,7 +264,7 @@ export async function handleRequest(req: Request): Promise<Response> {
 
         const payload: Record<string, unknown> = {
           model,
-          messages,
+          messages: withPromptCaching(model, messages),
           max_tokens: Math.min(modelEntry.maxTokens, MAX_REQUEST_OUTPUT_TOKENS),
           reasoning: { effort: reasoningEffort, exclude: true },
           // Omitting sort and order keeps OpenRouter's balanced provider load balancing.