From fbbca717893f30f53bf7269cc8f6d53153cf69d0 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 22 Oct 2026 11:14:44 +0000 Subject: [PATCH 1/3] Log the real cost of BYOK model calls and Exa research OpenRouter reports cost: 0 for calls billed to our own provider key (every Anthropic model), with the real charge in cost_details.upstream_inference_cost. The calculator now adds it, and a BYOK response without it stays unpriced. Exa searches and page reads now report costDollars.total into the request's research usage, so the parent ai_requests row includes them and raw_usage keeps one entry per Exa call. A migration recomputes the old $0 rows from their saved raw usage. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 2 +- supabase/functions/_shared/cost.test.ts | 64 +++++++++++++- supabase/functions/_shared/cost.ts | 44 +++++++++- supabase/functions/_shared/research.test.ts | 56 ++++++++++++ supabase/functions/_shared/research.ts | 33 +++++-- supabase/functions/ai/features/ask.ts | 2 +- supabase/functions/ai/index.ts | 8 +- ...1022120000_backfill_byok_request_costs.sql | 86 +++++++++++++++++++ 8 files changed, 283 insertions(+), 12 deletions(-) create mode 100644 supabase/migrations/20261022120000_backfill_byok_request_costs.sql diff --git a/docs/architecture.md b/docs/architecture.md index 30b508c00..4db295e02 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -132,7 +132,7 @@ Supabase Auth owns user sessions. Protected routes provide a client navigation b Checkout and portal functions create Stripe sessions after authenticated ownership and redirect validation. The webhook updates subscription state and records event processing for idempotency. AI handlers record the provider-reported usage and charge for each request. -All model requests use the single OpenRouter backend route and `OPENROUTER_API_KEY`. The shared ten-model catalog, its defaults (Claude Haiku 5.5 for Basic, Claude Opus 5.5 for Pro), Pro-only entries, retired-ID successors, and request controls live in [`supabase/functions/_shared/model-catalog.ts`](../supabase/functions/_shared/model-catalog.ts). OpenRouter usage remains raw in the audit row, and the reported charge is unknown when the provider does not return a complete cost. The opponent's voice is the one exception to the OpenRouter route: the `voice` function calls ElevenLabs with `ELEVENLABS_API_KEY`. ElevenLabs reports billed characters, not dollars, so [`supabase/functions/_shared/cost.ts`](../supabase/functions/_shared/cost.ts) prices those rows from a dated list-price snapshot; promotions and plan credits are not modelled, so a voice row can overstate the charge. +All model requests use the single OpenRouter backend route and `OPENROUTER_API_KEY`. The shared ten-model catalog, its defaults (Claude Haiku 5.5 for Basic, Claude Opus 5.5 for Pro), Pro-only entries, retired-ID successors, and request controls live in [`supabase/functions/_shared/model-catalog.ts`](../supabase/functions/_shared/model-catalog.ts). OpenRouter usage remains raw in the audit row, and the reported charge is unknown when the provider does not return a complete cost. For a call billed to our own provider key (BYOK, which the Anthropic models use), OpenRouter's `cost` is only its fee, so the logged cost adds `cost_details.upstream_inference_cost`. Exa searches and page reads report their charge as `costDollars.total`; it joins the request's row, with one `raw_usage.calls` entry per Exa call. The opponent's voice is the one exception to the OpenRouter route: the `voice` function calls ElevenLabs with `ELEVENLABS_API_KEY`. ElevenLabs reports billed characters, not dollars, so [`supabase/functions/_shared/cost.ts`](../supabase/functions/_shared/cost.ts) prices those rows from a dated list-price snapshot; promotions and plan credits are not modelled, so a voice row can overstate the charge. `Auto Fast` and `Auto Pro` are virtual selections, not provider model IDs. The Edge Functions resolve them through TypeSafe Jev on OpenRouter before calling the concrete writing model. Auto Fast is available to Basic accounts, routes only among the Basic models (Claude Haiku 5.5, GPT-6 Luna, Muse Spark 1.3 Contributor), and normally resolves to Claude Haiku 5.5. Auto Pro remains Pro-only and normally resolves to Claude Opus 5.5. Provider calls, usage rows, and response metadata record the concrete resolved model, while response metadata also retains the requested automatic selection. A Jev router call uses the same OpenRouter key and does not consume a second daily quota unit. diff --git a/supabase/functions/_shared/cost.test.ts b/supabase/functions/_shared/cost.test.ts index cc7b0de90..2c0e999d4 100644 --- a/supabase/functions/_shared/cost.test.ts +++ b/supabase/functions/_shared/cost.test.ts @@ -3,6 +3,7 @@ import { markIncompleteUsage, mergeNormalizedUsage, normalizeElevenLabsUsage, + normalizeExaUsage, normalizeOpenRouterUsage, selectObservedUsage, } from "./cost.ts"; @@ -15,7 +16,68 @@ Deno.test("OpenRouter usage uses its reported account cost, including zero", () }); if (result.costUsd !== 0) throw new Error(`zero cost was discarded: ${result.costUsd}`); if (result.inputTokens !== 1000 || result.outputTokens !== 500) throw new Error("token counts lost"); - if (result.costCalculationVersion !== "openrouter-usage-v1") throw new Error("wrong calculation version"); + if (result.costCalculationVersion !== "openrouter-usage-v2") throw new Error("wrong calculation version"); +}); + +// Shape copied from a staging Claude Haiku 5.5 row that v1 logged as $0. +const BYOK_USAGE = { + prompt_tokens: 1527, + completion_tokens: 671, + cost: 0, + is_byok: true, + cost_details: { + upstream_inference_cost: 0.0004882, + upstream_inference_prompt_cost: 0.0001527, + upstream_inference_completions_cost: 0.0003355, + }, +}; + +Deno.test("BYOK OpenRouter usage bills the upstream provider charge plus OpenRouter's fee", () => { + const free = calculateRequestCost({ + provider: "openrouter", + model: "anthropic/claude-haiku-5.5", + usage: normalizeOpenRouterUsage(BYOK_USAGE), + }); + if (free.costUsd !== 0.0004882) throw new Error(`BYOK upstream charge was not billed: ${free.costUsd}`); + + const withFee = normalizeOpenRouterUsage({ ...BYOK_USAGE, cost: 0.0001 }); + if (Math.abs((withFee.reportedCostUsd ?? -1) - 0.0005882) > 1e-12) { + throw new Error(`BYOK fee was not added to the upstream charge: ${withFee.reportedCostUsd}`); + } +}); + +Deno.test("BYOK OpenRouter usage without an upstream charge fails closed instead of logging $0", () => { + const { cost_details: _details, ...withoutUpstream } = BYOK_USAGE; + const result = calculateRequestCost({ + provider: "openrouter", + model: "anthropic/claude-haiku-5.5", + usage: normalizeOpenRouterUsage(withoutUpstream), + }); + if (result.costUsd !== null) throw new Error(`missing BYOK upstream cost was logged as ${result.costUsd}`); +}); + +Deno.test("Exa charges merge into the request row next to model usage", () => { + const search = normalizeExaUsage("search", { requestId: "s1", costDollars: { total: 0.007, search: { neural: 0.007 } } }); + const read = normalizeExaUsage("contents", { requestId: "c1", costDollars: { total: 0.001 } }); + const merged = mergeNormalizedUsage([normalizeOpenRouterUsage(BYOK_USAGE), search, read]); + const result = calculateRequestCost({ provider: "openrouter", model: "anthropic/claude-haiku-5.5", usage: merged }); + if (Math.abs((result.costUsd ?? -1) - 0.0084882) > 1e-12) throw new Error(`Exa cost missing from total: ${result.costUsd}`); + if (result.inputTokens !== 1527) throw new Error("Exa usage changed token totals"); + const calls = (result.rawUsage as { calls: Array> }).calls; + if (calls[1]?.provider !== "exa" || calls[1]?.endpoint !== "search" || calls[2]?.request_id !== "c1") { + throw new Error("per-call Exa audit entries were lost"); + } +}); + +Deno.test("an Exa response without a charge makes the request cost unknown", () => { + const unpriced = normalizeExaUsage("search", { requestId: "s1", results: [] }); + if (!unpriced.incomplete) throw new Error("missing Exa charge was not marked incomplete"); + const result = calculateRequestCost({ + provider: "openrouter", + model: "anthropic/claude-haiku-5.5", + usage: mergeNormalizedUsage([normalizeOpenRouterUsage(BYOK_USAGE), unpriced]), + }); + if (result.costUsd !== null) throw new Error(`unpriced Exa call was treated as free: ${result.costUsd}`); }); Deno.test("OpenRouter usage without a reported cost fails closed", () => { diff --git a/supabase/functions/_shared/cost.ts b/supabase/functions/_shared/cost.ts index ca051b645..16026232f 100644 --- a/supabase/functions/_shared/cost.ts +++ b/supabase/functions/_shared/cost.ts @@ -2,7 +2,11 @@ // costUsd = null, never an invented estimate. Raw provider usage is preserved // so every billing-relevant field stays auditable. -export const OPENROUTER_COST_CALCULATION_VERSION = "openrouter-usage-v1"; +// v2 adds the upstream charge OpenRouter reports for BYOK calls, which v1 +// logged as $0, and the Exa charges merged into a request's research. +export const OPENROUTER_COST_CALCULATION_VERSION = "openrouter-usage-v2"; + +export const EXA_PROVIDER = "exa"; export const ELEVENLABS_PROVIDER = "elevenlabs"; export const ELEVENLABS_COST_CALCULATION_VERSION = "elevenlabs-characters-v1"; @@ -67,6 +71,20 @@ function reportedCostOrUndefined(value: unknown): number | undefined { return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined; } +/** + * OpenRouter's `cost` is what it charged the account. With a provider key + * attached (BYOK, `is_byok: true`) that is only its fee, often 0, and the + * provider bills our own key for `cost_details.upstream_inference_cost`. The + * real cost is the sum. A BYOK response without the upstream figure stays + * unpriced rather than logging the fee alone as the whole bill. + */ +function openRouterCostOrUndefined(usage: Record): number | undefined { + const fee = reportedCostOrUndefined(usage.cost); + if (usage.is_byok !== true || fee === undefined) return fee; + const upstream = reportedCostOrUndefined(asRecord(usage.cost_details)?.upstream_inference_cost); + return upstream === undefined ? undefined : fee + upstream; +} + export function normalizeOpenRouterUsage(usage: Record): NormalizedUsage { const promptDetails = asRecord(usage.prompt_tokens_details); const completionDetails = asRecord(usage.completion_tokens_details); @@ -76,11 +94,33 @@ export function normalizeOpenRouterUsage(usage: Record): Normal cacheWriteTokens: intOrUndefined(promptDetails?.cache_write_tokens), outputTokens: intOrUndefined(usage.completion_tokens), reasoningOutputTokens: intOrUndefined(completionDetails?.reasoning_tokens), - reportedCostUsd: reportedCostOrUndefined(usage.cost), + reportedCostUsd: openRouterCostOrUndefined(usage), raw: usage, }; } +/** + * Exa reports each call's charge as `costDollars.total`. The usage joins the + * request's research usage, so the parent ai_requests row carries the cost + * and `raw_usage.calls` keeps one entry per Exa call. A successful response + * without a charge is incomplete: the call was billed, the amount is unknown. + */ +export function normalizeExaUsage(endpoint: "search" | "contents", data: unknown): NormalizedUsage { + const body = asRecord(data); + const costDollars = asRecord(body?.costDollars); + const total = reportedCostOrUndefined(costDollars?.total); + return { + incomplete: total === undefined || undefined, + reportedCostUsd: total, + raw: { + provider: EXA_PROVIDER, + endpoint, + request_id: typeof body?.requestId === "string" ? body.requestId : null, + cost_dollars: costDollars ?? null, + }, + }; +} + /** * ElevenLabs bills per input character and reports the count in the * `character-cost` response header. Without a usable header, the characters diff --git a/supabase/functions/_shared/research.test.ts b/supabase/functions/_shared/research.test.ts index 717268ec6..baba6e96a 100644 --- a/supabase/functions/_shared/research.test.ts +++ b/supabase/functions/_shared/research.test.ts @@ -12,6 +12,7 @@ import { MAX_PROVIDER_RESPONSE_BYTES, type ResearchContext, } from "./research.ts"; +import type { NormalizedUsage } from "./cost.ts"; function ctxWith(overrides: Partial): ResearchContext { return { ...emptyResearchContext(), ...overrides }; @@ -293,6 +294,61 @@ Deno.test("filtered research retries without domain filters when Exa returns no } }); +Deno.test("research reports the charge of every billed Exa call and none for failed ones", async () => { + const originalFetch = globalThis.fetch; + const previousApiKey = Deno.env.get("EXA_API_KEY"); + Deno.env.set("EXA_API_KEY", "test-key"); + const json = (body: unknown, status = 200) => Promise.resolve(new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json" }, + })); + globalThis.fetch = ((input: Request | URL | string, init?: RequestInit) => { + const body = JSON.parse(String(init?.body)) as Record; + if (String(input).endsWith("/contents")) { + const [url] = body.urls as string[]; + return url.includes("gsa.gov") + ? json({ requestId: "read-ok", results: [{ text: "page" }], costDollars: { total: 0.001 } }) + : json({ error: "upstream" }, 500); + } + // The domain-filtered search finds nothing, so research pays for a fallback. + return Array.isArray(body.includeDomains) + ? json({ requestId: "filtered", results: [], costDollars: { total: 0.005 } }) + : json({ + requestId: "fallback", + results: [ + { title: "Official guidance", url: "https://gsa.gov/policy", text: "Official government procurement policy with detailed methodology." }, + { title: "Agency data", url: "https://data.gov/procurement", text: "Official government procurement data and supporting methodology." }, + ], + costDollars: { total: 0.007 }, + }); + }) as typeof fetch; + + const usages: NormalizedUsage[] = []; + try { + await deepResearch("federal procurement reform", "United States", { + provider: "openrouter", + model: "test-model", + reasoningEffort: "low", + callModel: async () => ({ output: { queries: [] } }), + onExaUsage: (usage) => usages.push(usage), + }, { + maxFollowUpQueries: 0, + maxUrlsToRead: 2, + maxSeedQueries: 1, + overallTimeoutMs: 1000, + preferredSourceType: "government", + }); + const ids = usages.map((usage) => usage.raw?.request_id).sort(); + if (ids.join(",") !== "fallback,filtered,read-ok") throw new Error(`unexpected billed Exa calls: ${ids.join(",")}`); + const total = usages.reduce((sum, usage) => sum + (usage.reportedCostUsd ?? 0), 0); + if (Math.abs(total - 0.013) > 1e-12) throw new Error(`Exa charges did not add up: ${total}`); + } finally { + globalThis.fetch = originalFetch; + if (previousApiKey === undefined) Deno.env.delete("EXA_API_KEY"); + else Deno.env.set("EXA_API_KEY", previousApiKey); + } +}); + Deno.test("JSON body timeout cancels a read that stalled after headers", async () => { const originalFetch = globalThis.fetch; let bodyStartedResolve!: () => void; diff --git a/supabase/functions/_shared/research.ts b/supabase/functions/_shared/research.ts index 9dc695470..d257ac1d5 100644 --- a/supabase/functions/_shared/research.ts +++ b/supabase/functions/_shared/research.ts @@ -18,6 +18,7 @@ import { export type { ExaResult, RankedSearchResult, ResearchSource }; import type { ReasoningEffort } from "./model-catalog.ts"; +import { normalizeExaUsage, type NormalizedUsage } from "./cost.ts"; export type Provider = "openrouter"; @@ -72,6 +73,8 @@ export interface ResearchDeps { signal?: AbortSignal; /** Called synchronously when the overall research timeout fires. */ onTimeout?: () => void; + /** Receives the charge of every paid Exa call made for the request. */ + onExaUsage?: ExaUsageSink; /** Injected model caller; returns the parsed JSON output of a provider call. */ callModel: (args: { provider: Provider; @@ -258,8 +261,26 @@ export async function fetchJsonWithTimeout( // ============== EXA FUNCTIONS ============== +export type ExaUsageSink = (usage: NormalizedUsage) => void; + +// A response Exa answered successfully was billed. Like research model calls, +// work that finishes after the request's signal aborted is not reported: the +// timeout already marked the request's research accounting incomplete. +function reportExaUsage( + endpoint: "search" | "contents", + data: unknown, + signal: AbortSignal | undefined, + onUsage: ExaUsageSink | undefined, +): void { + if (onUsage && !signal?.aborted) onUsage(normalizeExaUsage(endpoint, data)); +} + // exaSearch: web search using Exa's neural search (type "auto" = balanced relevance & speed) -export async function exaSearch(search: string | ExaSearchRequest, signal?: AbortSignal): Promise { +export async function exaSearch( + search: string | ExaSearchRequest, + signal?: AbortSignal, + onUsage?: ExaUsageSink, +): Promise { const request = typeof search === "string" ? { query: search } : search; const query = request.query; const apiKey = Deno.env.get("EXA_API_KEY"); @@ -279,6 +300,7 @@ export async function exaSearch(search: string | ExaSearchRequest, signal?: Abor contents: { text: { maxCharacters: 10000 } }, }), }, 30000, signal); + if (response.ok) reportExaUsage("search", data, signal, onUsage); throwIfAborted(signal); if (!response.ok) { console.warn(`[EXA] Search failed (${response.status}) queryLength=${query.length}`); @@ -299,7 +321,7 @@ export async function exaSearch(search: string | ExaSearchRequest, signal?: Abor } // exaGetContents: extract full text from a specific URL using Exa's /contents endpoint -export async function exaGetContents(url: string, signal?: AbortSignal): Promise { +export async function exaGetContents(url: string, signal?: AbortSignal, onUsage?: ExaUsageSink): Promise { const apiKey = Deno.env.get("EXA_API_KEY"); if (!apiKey) { console.warn("[EXA] EXA_API_KEY not configured"); @@ -312,6 +334,7 @@ export async function exaGetContents(url: string, signal?: AbortSignal): Promise headers: { "Content-Type": "application/json", "x-api-key": apiKey }, body: JSON.stringify({ urls: [url], text: { maxCharacters: 8000 } }), }, 30000, signal); + if (response.ok) reportExaUsage("contents", data, signal, onUsage); throwIfAborted(signal); if (!response.ok) { console.warn(`[EXA] Contents fetch failed (${response.status}) urlLength=${url.length}`); @@ -448,14 +471,14 @@ async function deepResearchInner( ); const searchWithFallback = async (request: ExaSearchRequest) => { - const filtered = await exaSearch(request, signal); + const filtered = await exaSearch(request, signal, deps.onExaUsage); if (filtered.length > 0 || (!request.includeDomains && !request.category)) return filtered; const { includeDomains: _includeDomains, category: _category, ...fallbackRequest } = request; - return exaSearch(fallbackRequest, signal); + return exaSearch(fallbackRequest, signal, deps.onExaUsage); }; const seedResults = await Promise.all(seedRequests.map(async (request) => { @@ -503,7 +526,7 @@ async function deepResearchInner( const readPromises = urlsToRead.map(async (result) => { const url = result.url || ""; - const content = await exaGetContents(url, signal); + const content = await exaGetContents(url, signal, deps.onExaUsage); throwIfAborted(signal); if (content) { context.readContent.push({ url, title: result.title || sourceTitleFromUrl(url), content, result }); diff --git a/supabase/functions/ai/features/ask.ts b/supabase/functions/ai/features/ask.ts index 9211efbfb..860a4e7c0 100644 --- a/supabase/functions/ai/features/ask.ts +++ b/supabase/functions/ai/features/ask.ts @@ -42,7 +42,7 @@ export async function handleAsk(ctx: FeatureContext): Promise { let urlContext = ""; if (urls.size > 0) { const contents = await Promise.all(Array.from(urls).slice(0, 3).map(async (u) => { - const content = await exaGetContents(u, workSignal); + const content = await exaGetContents(u, workSignal, researchDeps.onExaUsage); return content.trim() ? `--- Content from ${u} ---\n${content.slice(0, 8000)}\n--- End of ${u} ---` : ""; diff --git a/supabase/functions/ai/index.ts b/supabase/functions/ai/index.ts index 786ccb8f2..9249ce657 100644 --- a/supabase/functions/ai/index.ts +++ b/supabase/functions/ai/index.ts @@ -624,8 +624,9 @@ export async function handleRequest(req: Request): Promise { }); const startTime = Date.now(); - // Usage from research sub-calls (search decision, follow-up queries) is - // collected here so the parent request's row accounts for every token. + // Usage from research sub-calls (search decision, follow-up queries, Exa + // searches and page reads) is collected here so the parent request's row + // accounts for every token and every paid call. const researchUsage: NormalizedUsage[] = []; let researchAccountingIncomplete = false; const researchDeps: ResearchDeps = { @@ -636,6 +637,9 @@ export async function handleRequest(req: Request): Promise { onTimeout: () => { researchAccountingIncomplete = true; }, + onExaUsage: (usage) => { + researchUsage.push(usage); + }, callModel: async (args) => { try { const result = await callAI(args.provider, args.model, args.systemPrompt, args.messages, args.reasoningEffort, args.signal); diff --git a/supabase/migrations/20261022120000_backfill_byok_request_costs.sql b/supabase/migrations/20261022120000_backfill_byok_request_costs.sql new file mode 100644 index 000000000..7c02a74e4 --- /dev/null +++ b/supabase/migrations/20261022120000_backfill_byok_request_costs.sql @@ -0,0 +1,86 @@ +-- Repair ai_requests rows that logged a BYOK model call as $0. +-- +-- With a provider key attached (BYOK), OpenRouter's usage.cost is only its +-- own fee, usually 0, and the provider bills our key for +-- usage.cost_details.upstream_inference_cost. Calculator v1 +-- (openrouter-usage-v1) stored the fee alone, so every Anthropic call logged +-- $0. v2 adds the upstream charge; this recomputes the old rows the same way +-- from the raw usage each row kept. +-- +-- Only v1 rows that were priced and contain a BYOK call change. A row is +-- left alone unless every call in it has a usable cost (and, for BYOK calls, +-- a usable upstream charge). Repaired rows get their own calculation version, +-- so the update cannot apply twice and stays auditable. raw_usage is not +-- touched. + +with leaves as ( + -- One row per provider call. raw_usage is either a single OpenRouter usage + -- object or {calls: [...]} for a merged request, and a repair retry can + -- nest one more {calls: [...]} level. Anything deeper has no top-level + -- cost and keeps its row out of the update. + select r.id, leaf.c + from public.ai_requests r + cross join lateral ( + select e + from jsonb_array_elements( + case when jsonb_typeof(r.raw_usage -> 'calls') = 'array' + then r.raw_usage -> 'calls' else jsonb_build_array(r.raw_usage) end + ) as top(e) + where jsonb_typeof(e -> 'calls') is distinct from 'array' + union all + select n + from jsonb_array_elements( + case when jsonb_typeof(r.raw_usage -> 'calls') = 'array' + then r.raw_usage -> 'calls' else jsonb_build_array(r.raw_usage) end + ) as top(e) + cross join lateral jsonb_array_elements(e -> 'calls') as nested(n) + where jsonb_typeof(e -> 'calls') = 'array' + ) as leaf(c) + where r.cost_calculation_version = 'openrouter-usage-v1' + and r.cost_usd is not null + and jsonb_typeof(r.raw_usage) = 'object' +), +checked as ( + -- Validity is decided on jsonb values, without casts, so a malformed row is + -- skipped instead of failing the migration. + select + id, + c, + coalesce(c -> 'is_byok' = 'true'::jsonb, false) as byok, + coalesce( + jsonb_typeof(c -> 'cost') = 'number' + and c -> 'cost' >= '0'::jsonb + and ( + c -> 'is_byok' is distinct from 'true'::jsonb + or ( + jsonb_typeof(c -> 'cost_details' -> 'upstream_inference_cost') = 'number' + and c -> 'cost_details' -> 'upstream_inference_cost' >= '0'::jsonb + ) + ), + false + ) as ok + from leaves +), +recomputed as ( + select + id, + sum( + case + when not ok then null + when byok then (c ->> 'cost')::numeric + (c -> 'cost_details' ->> 'upstream_inference_cost')::numeric + else (c ->> 'cost')::numeric + end + ) as cost, + bool_and(ok) as complete, + bool_or(byok) as has_byok + from checked + group by id +) +update public.ai_requests as a +set + cost_usd = recomputed.cost, + cost_calculation_version = 'openrouter-usage-v1-byok-backfill' +from recomputed +where a.id = recomputed.id + and recomputed.complete + and recomputed.has_byok; From 737d31af8624c1fe166cb994c7660ec44e08e144 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 22 Oct 2026 11:45:30 +0000 Subject: [PATCH 2/3] Make an unsettled Exa call leave the request cost unknown A sent Exa call that is cancelled, times out, or returns an unreadable body may still be billed. It now reports an incomplete usage entry instead of nothing, so the row's cost is unknown rather than quietly too low. Error statuses are still not billed. Co-Authored-By: Claude Opus 5.5 --- supabase/functions/_shared/research.test.ts | 15 +++++-- supabase/functions/_shared/research.ts | 45 +++++++++++++++------ 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/supabase/functions/_shared/research.test.ts b/supabase/functions/_shared/research.test.ts index baba6e96a..58ec81863 100644 --- a/supabase/functions/_shared/research.test.ts +++ b/supabase/functions/_shared/research.test.ts @@ -294,7 +294,7 @@ Deno.test("filtered research retries without domain filters when Exa returns no } }); -Deno.test("research reports the charge of every billed Exa call and none for failed ones", async () => { +Deno.test("research reports every billed Exa call, none for error statuses, and unknown for unsettled ones", async () => { const originalFetch = globalThis.fetch; const previousApiKey = Deno.env.get("EXA_API_KEY"); Deno.env.set("EXA_API_KEY", "test-key"); @@ -306,6 +306,7 @@ Deno.test("research reports the charge of every billed Exa call and none for fai const body = JSON.parse(String(init?.body)) as Record; if (String(input).endsWith("/contents")) { const [url] = body.urls as string[]; + if (url.includes("energy.gov")) return Promise.reject(new TypeError("connection reset")); return url.includes("gsa.gov") ? json({ requestId: "read-ok", results: [{ text: "page" }], costDollars: { total: 0.001 } }) : json({ error: "upstream" }, 500); @@ -318,6 +319,7 @@ Deno.test("research reports the charge of every billed Exa call and none for fai results: [ { title: "Official guidance", url: "https://gsa.gov/policy", text: "Official government procurement policy with detailed methodology." }, { title: "Agency data", url: "https://data.gov/procurement", text: "Official government procurement data and supporting methodology." }, + { title: "Energy procurement", url: "https://energy.gov/procurement", text: "Official government procurement rules and supporting methodology." }, ], costDollars: { total: 0.007 }, }); @@ -333,15 +335,20 @@ Deno.test("research reports the charge of every billed Exa call and none for fai onExaUsage: (usage) => usages.push(usage), }, { maxFollowUpQueries: 0, - maxUrlsToRead: 2, + maxUrlsToRead: 3, maxSeedQueries: 1, overallTimeoutMs: 1000, preferredSourceType: "government", }); - const ids = usages.map((usage) => usage.raw?.request_id).sort(); + const billed = usages.filter((usage) => !usage.incomplete); + const ids = billed.map((usage) => usage.raw?.request_id).sort(); if (ids.join(",") !== "fallback,filtered,read-ok") throw new Error(`unexpected billed Exa calls: ${ids.join(",")}`); - const total = usages.reduce((sum, usage) => sum + (usage.reportedCostUsd ?? 0), 0); + const total = billed.reduce((sum, usage) => sum + (usage.reportedCostUsd ?? 0), 0); if (Math.abs(total - 0.013) > 1e-12) throw new Error(`Exa charges did not add up: ${total}`); + const unsettled = usages.filter((usage) => usage.incomplete); + if (unsettled.length !== 1 || unsettled[0].raw?.endpoint !== "contents") { + throw new Error(`the reset page read was not reported as unknown: ${JSON.stringify(unsettled)}`); + } } finally { globalThis.fetch = originalFetch; if (previousApiKey === undefined) Deno.env.delete("EXA_API_KEY"); diff --git a/supabase/functions/_shared/research.ts b/supabase/functions/_shared/research.ts index d257ac1d5..44c66b605 100644 --- a/supabase/functions/_shared/research.ts +++ b/supabase/functions/_shared/research.ts @@ -263,16 +263,31 @@ export async function fetchJsonWithTimeout( export type ExaUsageSink = (usage: NormalizedUsage) => void; -// A response Exa answered successfully was billed. Like research model calls, -// work that finishes after the request's signal aborted is not reported: the -// timeout already marked the request's research accounting incomplete. -function reportExaUsage( - endpoint: "search" | "contents", - data: unknown, - signal: AbortSignal | undefined, - onUsage: ExaUsageSink | undefined, -): void { - if (onUsage && !signal?.aborted) onUsage(normalizeExaUsage(endpoint, data)); +// Reports each sent Exa call once. A successful response was billed at the +// charge it reports; an error status was not. A call that was sent but never +// settled (cancelled, timed out, unreadable body) may still have been billed, +// so it makes the request's cost unknown instead of quietly lowering it. +function exaUsageReporter(endpoint: "search" | "contents", onUsage: ExaUsageSink | undefined) { + let sent = false; + let reported = false; + const report = (usage: NormalizedUsage) => { + if (reported || !onUsage) return; + reported = true; + onUsage(usage); + }; + return { + sent: () => { sent = true; }, + response: (ok: boolean, data: unknown) => { + if (!ok) { + reported = true; + return; + } + report(normalizeExaUsage(endpoint, data)); + }, + unsettled: () => { + if (sent) report({ incomplete: true, raw: { provider: "exa", endpoint, unsettled: true } }); + }, + }; } // exaSearch: web search using Exa's neural search (type "auto" = balanced relevance & speed) @@ -288,8 +303,10 @@ export async function exaSearch( console.warn("[EXA] EXA_API_KEY not configured"); return []; } + const usage = exaUsageReporter("search", onUsage); try { throwIfAborted(signal); + usage.sent(); const { response, data } = await fetchJsonWithTimeout<{ results?: unknown }>("https://api.exa.ai/search", { method: "POST", headers: { "Content-Type": "application/json", "x-api-key": apiKey }, @@ -300,7 +317,7 @@ export async function exaSearch( contents: { text: { maxCharacters: 10000 } }, }), }, 30000, signal); - if (response.ok) reportExaUsage("search", data, signal, onUsage); + usage.response(response.ok, data); throwIfAborted(signal); if (!response.ok) { console.warn(`[EXA] Search failed (${response.status}) queryLength=${query.length}`); @@ -314,6 +331,7 @@ export async function exaSearch( publishedDate: r.publishedDate, })); } catch (err) { + usage.unsettled(); if (signal?.aborted) throw err; console.warn(`[EXA] Search failed queryLength=${query.length}`); return []; @@ -327,14 +345,16 @@ export async function exaGetContents(url: string, signal?: AbortSignal, onUsage? console.warn("[EXA] EXA_API_KEY not configured"); return ""; } + const usage = exaUsageReporter("contents", onUsage); try { throwIfAborted(signal); + usage.sent(); const { response, data } = await fetchJsonWithTimeout<{ results?: unknown }>("https://api.exa.ai/contents", { method: "POST", headers: { "Content-Type": "application/json", "x-api-key": apiKey }, body: JSON.stringify({ urls: [url], text: { maxCharacters: 8000 } }), }, 30000, signal); - if (response.ok) reportExaUsage("contents", data, signal, onUsage); + usage.response(response.ok, data); throwIfAborted(signal); if (!response.ok) { console.warn(`[EXA] Contents fetch failed (${response.status}) urlLength=${url.length}`); @@ -343,6 +363,7 @@ export async function exaGetContents(url: string, signal?: AbortSignal, onUsage? if (!data?.results || !Array.isArray(data.results) || data.results.length === 0) return ""; return (data.results[0]?.text || "").slice(0, 8000); } catch (err) { + usage.unsettled(); if (signal?.aborted) throw err; console.warn(`[EXA] Contents fetch failed urlLength=${url.length}`); return ""; From b8d906e43f8e5b58f89e2c08cdfcf32309ac9cd1 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 22 Oct 2026 12:57:43 +0000 Subject: [PATCH 3/3] Note that the BYOK backfill must run after the new functions are live --- .../migrations/20261022120000_backfill_byok_request_costs.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/supabase/migrations/20261022120000_backfill_byok_request_costs.sql b/supabase/migrations/20261022120000_backfill_byok_request_costs.sql index 7c02a74e4..d5ad6f29e 100644 --- a/supabase/migrations/20261022120000_backfill_byok_request_costs.sql +++ b/supabase/migrations/20261022120000_backfill_byok_request_costs.sql @@ -12,6 +12,10 @@ -- a usable upstream charge). Repaired rows get their own calculation version, -- so the update cannot apply twice and stays auditable. raw_usage is not -- touched. +-- +-- Deploy order: deploy the new ai and ai-stream functions (each bundles its +-- own copy of _shared/cost.ts) before applying this migration. It runs once, +-- so v1 rows that the old functions write after it has run stay at $0. with leaves as ( -- One row per provider call. raw_usage is either a single OpenRouter usage