diff --git a/.claude/skills/verify-debatelab/SKILL.md b/.claude/skills/verify-debatelab/SKILL.md index 59a876a39..b56f7f983 100644 --- a/.claude/skills/verify-debatelab/SKILL.md +++ b/.claude/skills/verify-debatelab/SKILL.md @@ -1,6 +1,6 @@ --- name: verify-debatelab -description: Launch and drive the real DebateLab web app (React client + local Supabase Auth/Postgres/Edge Functions, with either a fake model provider or real OpenRouter pinned to the cheap meta/muse-spark-1.3-contributor model plus real Exa research) to prove a user-facing feature works, with screenshots, ARIA snapshots, provider calls, and database rows as evidence. Use when asked to verify, demo, reproduce, or screenshot DebateLab behavior in a browser, or before calling a DebateLab UI/Edge Function change done. +description: Launch and drive the real DebateLab web app (React client + local Supabase Auth/Postgres/Edge Functions, with either a fake model provider or real OpenRouter pinned to Claude Haiku 5.5 on the Anthropic BYOK key plus real Exa research) to prove a user-facing feature works, with screenshots, ARIA snapshots, provider calls, and database rows as evidence. Use when asked to verify, demo, reproduce, or screenshot DebateLab behavior in a browser, or before calling a DebateLab UI/Edge Function change done. --- # Verify DebateLab @@ -8,7 +8,7 @@ description: Launch and drive the real DebateLab web app (React client + local S DebateLab is a web app. The surface a user touches is the browser client at `/login` and `/app/*`, backed by Supabase Auth, Postgres/RLS, and Deno Edge Functions. This skill runs all of that locally and drives it with Playwright. It has two modes: - **Fake (default, free).** `e2e/full-stack/fake-openrouter.mjs` answers every model call with canned text. Proves wiring, rows, and UI, not model behavior. -- **Live (`up --live`, costs money).** The same container becomes a recording proxy to real OpenRouter. It rewrites every chat call to `meta/muse-spark-1.3-contributor` (override with `VERIFY_LIVE_MODEL`), records OpenRouter's reply model as proof, and stops forwarding after `--max-calls` (default 400). Edge Functions also get a real Exa key, so web research and verified citations are real. +- **Live (`up --live`, costs money).** The same container becomes a recording proxy to real OpenRouter. It rewrites every chat call to `anthropic/claude-haiku-5.5` pinned to the Anthropic provider with fallbacks off, so it bills the Anthropic BYOK key and not OpenRouter credits (override with `VERIFY_LIVE_MODEL`, which must be an `anthropic/` model), records OpenRouter's reply model as proof, and stops forwarding after `--max-calls` (default 400). Edge Functions also get a real Exa key, so web research and verified citations are real. Everything goes through one helper, run from the repository root (`git rev-parse --show-toplevel`): @@ -42,7 +42,7 @@ STRIPE_TEST_SECRET_KEY=sk_test_... # optional; live keys are refused STRIPE_TEST_PRICE_ID=price_... # required with the Stripe key ``` -It prints nothing about the values. Ask Adam before the first live run: it spends money and sends prompts to a contributor model that may use them for training, so never put real user data in a live run. Costs: Muse Spark 1.3 contributor is $0.10 per million prompt tokens and $0.20 per million completion tokens, so a full `drive all` is cents of model spend. Exa bills per search and is usually the larger part. Jev decisions go to `/api/alpha/decisions` with Jev's own model and are not pinned. `doctor` shows `live calls used N/max`; once the cap is hit the proxy answers 429 and tests fail rather than spend more. +It prints nothing about the values. Ask Adam before the first live run: it spends money (Exa, Jev, and the Anthropic key), so never put real user data in a live run. Model calls bill the Anthropic BYOK key; Exa bills per search. Jev decisions go to `/api/alpha/decisions` with Jev's own model and are not pinned. `doctor` shows `live calls used N/max`; once the cap is hit the proxy answers 429 and tests fail rather than spend more. ## Launch @@ -123,7 +123,7 @@ $H e2e cache ls # recorded agent flo - `e2e` runs `doctor` first and refuses a broken stack, like `drive`. It turns e2e telemetry off and prints `EVIDENCE `, which holds `summary.md`, `report.json`, and per-attempt screenshots, traces, and failure videos. - The suite is tool journeys (tag `journey`) and a page sweep (tag `pages`) of every student and public page in English, Spanish and Hungarian (not `/app/admin` or `/share/:code`). Every test fails on an uncaught error, `console.error` (React warnings included), a Supabase 5xx, horizontal overflow, or a request to a hosted project: `agentic/support/capture-errors.js` records them in the page. A failure that names `errors the page logged` is a real app error to fix, not test noise. -- The testing agent has its own OpenRouter models: `inclusionai/ling-3.0-flash-vl` acts (test steps on a cache miss, and explorations) and `meta/muse-spark-1.3-contributor` judges. `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` override them; the model is not part of the replay key. The key comes from `OPENROUTER_API_KEY`, `LIVE_OPENROUTER_API_KEY`, or the live keys file, and is needed in fake mode too. A recorded `agent.act` replays with no model call; every judgment and every exploration step is a call. Without a key, agent tests skip. An OpenRouter error such as `[Meta] The requested model was not found` is infrastructure: rerun with `--last-failed`. The Muse Spark judge cannot act. +- The testing agent has its own OpenRouter models: `anthropic/claude-haiku-5.5` acts (test steps on a cache miss, and explorations) and `anthropic/claude-sonnet-5.5` judges, both pinned to the Anthropic provider (BYOK) with fallbacks off. Haiku is not reliable enough to judge screens: it reads content below the fold as cut off. `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` override them but must be `anthropic/` models; the model is not part of the replay key. The key comes from `OPENROUTER_API_KEY`, `LIVE_OPENROUTER_API_KEY`, or the live keys file, and is needed in fake mode too. A recorded `agent.act` replays with no model call; every judgment and every exploration step is a call. Without a key, agent tests skip. An OpenRouter error naming Anthropic (an outage, or the key out of credit) is infrastructure: rerun with `--last-failed`. - `--session student` starts an exploration signed in as a fresh demo user, minted by `agentic/sessions.e2e.ts`. `--agent` picks a persona: `first-timer`, `coach`, `edge-input`, or `hungarian`. `--target mobile` explores the phone-sized viewport. - Explorations report `issue` (functional) and `warning` (cosmetic) findings, and exit 1 when there is an issue. Treat a finding as a lead: the stack serves Vite dev, so a route that is still compiling can look dead, and fake mode answers every motion with free-transport text. Prove a finding with a repro test tagged `repro` before you report it as a bug. - Several explorations can run at once (a bug bash), because each gets its own evidence directory. They share this one stack, so give each a different area. diff --git a/.claude/skills/verify-debatelab/bin/verify.sh b/.claude/skills/verify-debatelab/bin/verify.sh index 57070704b..f755e2ebe 100755 --- a/.claude/skills/verify-debatelab/bin/verify.sh +++ b/.claude/skills/verify-debatelab/bin/verify.sh @@ -40,8 +40,11 @@ STATE="$STATE_DIR/stack.env" USERS="$STATE_DIR/users.tsv" ARTIFACTS="$VERIFY_ROOT/artifacts" # evidence; never removed by this script PROJECT_ID="$(basename "$REPO")" # supabase/config.toml has no project_id -# Adam's standing choice for live verification: the cheap contributor tier. -LIVE_MODEL="${VERIFY_LIVE_MODEL:-meta/muse-spark-1.3-contributor}" +# Live verification runs on Claude Haiku through DebateLab's Anthropic key +# (BYOK in OpenRouter). Other families bill OpenRouter credits, so they are +# refused; the proxy pins the rewritten call to the Anthropic provider. +LIVE_MODEL="${VERIFY_LIVE_MODEL:-anthropic/claude-haiku-5.5}" +case "$LIVE_MODEL" in anthropic/*) ;; *) printf 'FAIL VERIFY_LIVE_MODEL must be an anthropic/ model (Anthropic BYOK), not %s\n' "$LIVE_MODEL" >&2; exit 1 ;; esac LIVE_KEYS_FILE="${VERIFY_LIVE_KEYS_FILE:-$HOME/.config/debatelab/live-verify.env}" say() { printf '%s\n' "$*"; } @@ -299,7 +302,7 @@ cmd_e2e() { [ -n "$key" ] && export OPENROUTER_API_KEY="$key" fi if [ -n "${OPENROUTER_API_KEY:-}" ]; then - say "info testing agent: OpenRouter actor ${E2E_AGENT_MODEL:-inclusionai/ling-3.0-flash-vl}, judge ${E2E_JUDGE_MODEL:-meta/muse-spark-1.3-contributor} (replayed steps make no model call)" + say "info testing agent: OpenRouter actor ${E2E_AGENT_MODEL:-anthropic/claude-haiku-5.5}, judge ${E2E_JUDGE_MODEL:-anthropic/claude-sonnet-5.5}, Anthropic BYOK only (replayed steps make no model call)" else say "info no OpenRouter key: tests with agent steps will skip" fi diff --git a/.claude/skills/verify-debatelab/features/README.md b/.claude/skills/verify-debatelab/features/README.md index 03db23784..6270cb895 100644 --- a/.claude/skills/verify-debatelab/features/README.md +++ b/.claude/skills/verify-debatelab/features/README.md @@ -8,7 +8,7 @@ This directory is the maintained source for verifying DebateLab's user-facing be - The browser client is the `up` instance at `http://127.0.0.1:18081`. Never use the `debatelab` container on `:8080` or a plain `npm run dev`, because both reach production. - Each scripted test signs in with its own fresh demo-code user (a Pro account with a daily limit of 999) and deletes it afterwards. The fixture marks the onboarding tour as seen, and the user starts with no Current Debate. - Most tools work inside a Current Debate (see [current-debate](./current-debate.md)). Start one with `startDebate(page, motion)` first, or answer the motion dialog that opens after submit with `chooseForMotion(page, "create" | "once")`. The Library's `Add Source` stays disabled until a debate exists. -- In fake mode, model answers come from the fake provider and contain the marker `FAKE-PROVIDER`. In live mode (`up --live`), every chat call goes to real OpenRouter pinned to `meta/muse-spark-1.3-contributor`, research uses real Exa, and answers are real model text. +- In fake mode, model answers come from the fake provider and contain the marker `FAKE-PROVIDER`. In live mode (`up --live`), every chat call goes to real OpenRouter pinned to `anthropic/claude-haiku-5.5` on the Anthropic BYOK key, research uses real Exa, and answers are real model text. - The feature files quote fake-mode text such as `FAKE-PROVIDER case thesis`. Recipes never assert it: they read the `ai` response body with `aiResponse()` and check that its fields are on screen, so the same recipe passes in both modes. ## Driving conventions diff --git a/.claude/skills/verify-debatelab/features/model-defaults.md b/.claude/skills/verify-debatelab/features/model-defaults.md index 9f6683732..f685f775d 100644 --- a/.claude/skills/verify-debatelab/features/model-defaults.md +++ b/.claude/skills/verify-debatelab/features/model-defaults.md @@ -20,5 +20,5 @@ Every AI tool has a model selector. Basic accounts can pick Claude Haiku 5.5, GP ## Gotchas -- Fake mode records the requested model but answers with fake text, and live mode rewrites every chat call to the Muse Spark contributor pin. Neither proves that OpenRouter accepts the Claude 5.5 IDs; that needs a direct OpenRouter call. +- Fake mode records the requested model but answers with fake text, and live mode rewrites every chat call to the Claude Haiku 5.5 pin. Neither proves that OpenRouter accepts the Sonnet and Opus 5.5 IDs; that needs a direct OpenRouter call. - Existing saved choices of a model still in the catalog are kept. Only retired IDs and unsaved settings move to the new defaults. Saved Opus 5.5 settings from the old Pro default are moved to Sonnet 5.5 once, by the `20261022120000_opus_default_to_sonnet.sql` data migration, not by the client. diff --git a/.superset/db-gate.sh b/.superset/db-gate.sh index 279b42034..139b1964c 100755 --- a/.superset/db-gate.sh +++ b/.superset/db-gate.sh @@ -30,9 +30,11 @@ FAKE_PROVIDER=debatelab-fake-openrouter # the fake provider. It needs LIVE_OPENROUTER_API_KEY and LIVE_EXA_API_KEY. # LIVE_MODEL=as-requested runs e2e/model-matrix instead: one case per catalog model. LIVE_PROVIDERS="${LIVE_PROVIDERS:-0}" -# Muse Spark 1.3 Contributor is the cheapest model that supports every request -# parameter the app sends. It is a test-only model: it is not in the app catalog. -LIVE_MODEL="${LIVE_MODEL:-meta/muse-spark-1.3-contributor}" # empty also means default +# Claude Haiku runs on DebateLab's Anthropic key (BYOK in OpenRouter), so the +# live gate spends no OpenRouter credits. Other families would, so they are +# refused; as-requested (the model matrix) is the one deliberate exception. +LIVE_MODEL="${LIVE_MODEL:-anthropic/claude-haiku-5.5}" # empty also means default +case "$LIVE_MODEL" in anthropic/*|as-requested) ;; *) echo "LIVE_MODEL must be an anthropic/ model (Anthropic BYOK) or as-requested, not $LIVE_MODEL" >&2; exit 1 ;; esac if [ "$LIVE_PROVIDERS" = 1 ] && { [ -z "${LIVE_OPENROUTER_API_KEY:-}" ] || [ -z "${LIVE_EXA_API_KEY:-}" ]; }; then echo "LIVE_PROVIDERS=1 requires LIVE_OPENROUTER_API_KEY and LIVE_EXA_API_KEY" >&2 exit 1 diff --git a/AGENTS.md b/AGENTS.md index c30364961..1471baf2d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -96,14 +96,14 @@ To prove that a user-facing change works, drive the real app with the `verify-de ```bash H=.claude/skills/verify-debatelab/bin/verify.sh $H up # fake model provider, free -$H up --live # real AI: OpenRouter pinned to meta/muse-spark-1.3-contributor, real Exa +$H up --live # real AI: OpenRouter pinned to anthropic/claude-haiku-5.5 (Anthropic BYOK), real Exa $H doctor # proves the stack is local, ours, and (live) pinned $H drive all # every recipe; or recipes/, e2e/; add --project desktop-chromium $H e2e run # agentic tests in agentic/ (TesterArmy e2e); `e2e explore "" --session student --agent ` hunts for bugs $H down # always, including after failures ``` -- Use `--live` when the change touches prompts, model output handling, web research, or citations. The fake mode cannot prove those. URL import fetches pages directly and works the same in both modes. Every chat call is rewritten to the Muse Spark contributor model, so a full run costs cents plus Exa searches, and the proxy stops after 400 upstream calls (`--max-calls N`). Only use made-up debate content: the contributor model may keep prompts. +- Use `--live` when the change touches prompts, model output handling, web research, or citations. The fake mode cannot prove those. URL import fetches pages directly and works the same in both modes. Every chat call is rewritten to Claude Haiku 5.5 pinned to the Anthropic provider, so model calls bill the Anthropic BYOK key, not OpenRouter credits; Exa searches still cost, and the proxy stops after 400 upstream calls (`--max-calls N`). Only use made-up debate content. Never point a default, an Auto pool, or the testing agent at a non-`anthropic/` model or enable `allow_fallbacks`: `src/test/ai/anthropic-only-guard.test.ts` fails on it. - Live keys come from `~/.config/debatelab/live-verify.env` (mode 600, `LIVE_OPENROUTER_API_KEY` and `LIVE_EXA_API_KEY`). On Adam's VPS it holds copies of the Hermes keys in `~/.hermes/.env`. Never print them. - The stack holds the same lock as `./.superset/db-gate.sh`, so only one worktree can run it at a time. - Most tools work inside a Current Debate. Recipes start one with `startDebate` or answer the motion dialog with `chooseForMotion`; see the skill's Drive section. diff --git a/agentic/support/anthropic-cache.ts b/agentic/support/anthropic-cache.ts new file mode 100644 index 000000000..27ce8293e --- /dev/null +++ b/agentic/support/anthropic-cache.ts @@ -0,0 +1,46 @@ +// The testing agent marks Anthropic prompt-cache breakpoints on the system +// prompt and on the newest message. @openrouter/ai-sdk-provider sends a tool +// message as one OpenRouter message per tool result and copies the message's +// breakpoint onto each, so an actor turn with parallel tool calls asks +// Anthropic for more than its limit of four breakpoints and the call fails +// ("A maximum of 4 blocks with cache_control may be provided"). Moving the +// breakpoint onto the last tool result keeps one per message, which caches +// the same prefix. + +type Options = Record | undefined>; +type Part = { providerOptions?: Options } & Record; +type Message = { role: string; content: unknown; providerOptions?: Options }; +type CallOptions = { prompt: Message[] } & Record; + +export function oneBreakpointPerToolMessage(message: Message): Message { + if (message.role !== "tool" || !Array.isArray(message.content) || message.content.length < 2) return message; + const anthropic = message.providerOptions?.anthropic; + const breakpoint = anthropic?.cacheControl ?? anthropic?.cache_control; + if (!anthropic || breakpoint === undefined) return message; + const { cacheControl: _camel, cache_control: _snake, ...rest } = anthropic; + const parts = message.content as Part[]; + const last = parts.length - 1; + return { + ...message, + providerOptions: { ...message.providerOptions, anthropic: rest }, + content: parts.map((part, index) => + index === last + ? { ...part, providerOptions: { ...part.providerOptions, anthropic: { ...part.providerOptions?.anthropic, cacheControl: breakpoint } } } + : part, + ), + }; +} + +/** The same model, with every call's prompt passed through oneBreakpointPerToolMessage. */ +export function withAnthropicCacheLimit(model: M): M { + const fix = (options: CallOptions): CallOptions => ({ ...options, prompt: options.prompt.map(oneBreakpointPerToolMessage) }); + return new Proxy(model, { + get(target, property) { + const value: unknown = Reflect.get(target, property); + if ((property === "doGenerate" || property === "doStream") && typeof value === "function") { + return (options: CallOptions) => value.call(target, fix(options)); + } + return typeof value === "function" ? value.bind(target) : value; + }, + }); +} diff --git a/docs/architecture.md b/docs/architecture.md index 5ef48d404..81951378e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -152,7 +152,7 @@ All model requests use the single OpenRouter backend route and `OPENROUTER_API_K Every model call asks for JSON. Calls whose shape the server checks in code (the Crossfire next turn and debrief, the Ask search decision, and research follow-up queries) send a strict JSON Schema, so OpenRouter routes them only to endpoints that enforce it; the other features use plain JSON mode. [`supabase/functions/ai/provider.ts`](../supabase/functions/ai/provider.ts) fixes common slips in a reply locally (markdown fences, prose around the object, trailing commas, raw line breaks in strings) before it pays for one repair retry, and a reply the provider cut off at its cap is retried once with twice the budget, not with the repair prompt. -`Auto Fast` and `Auto Pro` are virtual selections, not provider model IDs. The Edge Functions resolve them through TypeSafe Jev on OpenRouter before calling the concrete writing model. Auto Fast is available to Basic accounts, routes only among the Basic models (Claude Haiku 5.5, GPT-6 Luna, Muse Spark 1.3 Contributor), and normally resolves to Claude Haiku 5.5. Auto Pro remains Pro-only, routes between Claude Sonnet 5.5 and GPT-6.1 Sol, and normally resolves to Claude Sonnet 5.5; it never routes or falls back to Opus. Provider calls, usage rows, and response metadata record the concrete resolved model, while response metadata also retains the requested automatic selection. A Jev router call uses the same OpenRouter key and does not consume a second daily quota unit. +`Auto Fast` and `Auto Pro` are virtual selections, not provider model IDs. Auto routes only to Claude, which runs on DebateLab's Anthropic key (BYOK in OpenRouter); other families bill OpenRouter credits, so they run only when a student picks one by hand. Auto Fast is available to Basic accounts and resolves to Claude Haiku 5.5. Auto Pro remains Pro-only and resolves to Claude Sonnet 5.5; it never routes or falls back to Opus. With one candidate per pool the Edge Functions skip the TypeSafe Jev routing call; a pool with more than one candidate (all `anthropic/`, which `src/test/ai/anthropic-only-guard.test.ts` enforces) is resolved through Jev on OpenRouter before calling the concrete writing model. Provider calls, usage rows, and response metadata record the concrete resolved model, while response metadata also retains the requested automatic selection. A Jev router call uses the same OpenRouter key and does not consume a second daily quota unit. The daily AI quota counts units, not requests. `checkRateLimit` in [`supabase/functions/_shared/entitlements.ts`](../supabase/functions/_shared/entitlements.ts) reserves the requested model's weight through `consume_ai_quota` (Claude Opus 5.5 counts as 5, Sonnet as 2, cheap models as 1; Auto Pro counts as its most expensive candidate because the quota is reserved before routing). The same call refuses the request with `weekly_budget_reached` once the user's weekly AI spend reaches the weekly budget ($2 by default, calendar week from Monday 00:00 UTC). Weekly spend is the sum of `ai_requests.cap_cost_usd`: each request's full logged charge, including Claude calls billed to our own BYOK Anthropic key, plus Exa and ElevenLabs charges. A BYOK response without the upstream charge counts OpenRouter's fee, the part that is known. When part of a request's cost is unknown, the known part counts; rows with no known cap spend, and rows logged before the column existed, do not count. Cost is logged after the provider answers, so the cap can be passed by about one request. The weights, the budget, and the tier defaults live in [`supabase/functions/_shared/usage-limits.ts`](../supabase/functions/_shared/usage-limits.ts). Voice, import, share, and Crossfire signal quotas keep calling `consume_daily_quota`, one unit per call, and are not gated by the weekly budget (voice spend still counts toward it). diff --git a/docs/testing.md b/docs/testing.md index ffbf05e79..811a19c5c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -219,7 +219,7 @@ To check one feature by hand, or keep evidence for a change, use the same stack LIVE_PROVIDERS=1 LIVE_OPENROUTER_API_KEY=... LIVE_EXA_API_KEY=... ./.superset/db-gate.sh ``` -Live mode uses the same locked stack but skips pgTAP and the fake-provider specs. It runs `e2e/live/` on desktop Chromium against real OpenRouter, Jev, and Exa. Imported web pages are fetched live too. The fake provider becomes a recording proxy. It forwards every chat call to OpenRouter with the model pinned to `LIVE_MODEL` (default `meta/muse-spark-1.3-contributor`; CI reads the `LIVE_MODEL` repository variable, which is unset), passes Jev decisions through unchanged, and refuses calls past `MAX_UPSTREAM_CALLS` (200). The default is the cheapest model that supports every request parameter the app sends, including `response_format` and reasoning effort. It is also the Basic catalog model "Muse Spark 1.3", so a live run that selects it cannot be told apart from the pin by the upstream model alone; recipes check the requested model instead. Meta is its only provider, so an outage there fails the gate instead of failing over. +Live mode uses the same locked stack but skips pgTAP and the fake-provider specs. It runs `e2e/live/` on desktop Chromium against real OpenRouter, Jev, and Exa. Imported web pages are fetched live too. The fake provider becomes a recording proxy. It forwards every chat call to OpenRouter with the model pinned to `LIVE_MODEL` (default `anthropic/claude-haiku-5.5`; CI reads the `LIVE_MODEL` repository variable, which is unset), passes Jev decisions through unchanged, and refuses calls past `MAX_UPSTREAM_CALLS` (200). Every rewritten call leaves pinned to the Anthropic provider with `allow_fallbacks: false`, so it bills DebateLab's Anthropic key (BYOK in OpenRouter) and never OpenRouter credits. The gate, `verify.sh`, and the proxy refuse a `LIVE_MODEL` outside `anthropic/`, except `as-requested` for the model matrix. The default is also the Basic catalog default, so a live run that selects it cannot be told apart from the pin by the upstream model alone; recipes check the requested model instead. Anthropic is its only provider, so an outage or an exhausted key fails the gate instead of failing over. The specs cover web-researched case generation, verified Factoids citations, Ask with web search, URL import, streamed Ask, Attack, Brainstorm, a judged sparring round, and Crossfire with live Jev signals. Outputs vary, so tests assert structure rather than text: @@ -267,7 +267,7 @@ How it runs: - Always through `verify.sh e2e `. The wrapper runs `doctor`, loads the stack environment, turns e2e telemetry off, and takes the testing agent's key from `OPENROUTER_API_KEY`, `LIVE_OPENROUTER_API_KEY`, or `~/.config/debatelab/live-verify.env`. It writes the report (`summary.md`, `report.json`), screenshots, traces, and failure videos to `.verify/artifacts/-e2e--/`. The config refuses a non-loopback URL and never starts a dev server, because a plain `npm run dev` reaches production. - Each signed-in test signs in a fresh demo-code user through the Login page and deletes it afterwards. Public-page tests use the `visitorTest` fixture with the same checks. - There are two targets, `desktop` (1280 by 800) and `mobile` (393 by 851). e2e does not emulate touch. The app lays out by viewport width and never reads the user agent, so the viewport is what matters; the Playwright Pixel 5 project still owns touch behavior. -- The testing agent has two models on OpenRouter, separate from the app's provider mode, and a fake-mode stack still needs both. The actor (`agent.act` and explorations) is `inclusionai/ling-3.0-flash-vl` ($0.021 and $0.062 per million input and output tokens on Novita, with DeepInfra as fallback). Recorded flows replay without it, so it runs only on a cache miss and in explorations. A small model is enough to click through these flows: with the cache off it passed the actor-heavy specs for about a fifth of GPT-6 Luna's price, while Gemma 3 12B, cheaper still, failed most of them. It sometimes calls `check` on a switch without `checked: false` and then reports the step done, so an unrecorded step that turns something off can fail; record such steps and commit the entry. OpenRouter withdrew the free Qwen tier the suite used before, which now fails with `This model is unavailable for free`. The judge (`agent.assert`, `waitFor`, `extract`) is `meta/muse-spark-1.3-contributor` ($0.10 and $0.20 per million input and output tokens), the same tier as the app's live pin. It answers every judgment on every run. A full run of both viewports makes about 250 judgments and costs about ten cents. It takes about 45 minutes on one machine with three workers, which is why CI splits it into four jobs. The judge cannot act, because Meta's endpoint accepts only `tool_choice: "auto"`. `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` override the two. The judge's tier may keep prompts, so use made-up debate content only. A transient OpenRouter error such as `[Meta] The requested model was not found` is infrastructure: rerun with `--last-failed`. The agent's context explains that fake AI answers are canned free-transport text. +- The testing agent has two models on OpenRouter, separate from the app's provider mode, and a fake-mode stack still needs both. The actor (`agent.act` and explorations) and the judge (`agent.assert`, `waitFor`, `extract`) are `anthropic/claude-haiku-5.5` and `anthropic/claude-sonnet-5.5`, both pinned to the Anthropic provider with `allow_fallbacks: false`, so they bill DebateLab's Anthropic key (BYOK in OpenRouter) rather than OpenRouter credits. Recorded flows replay without the actor, so it runs only on a cache miss and in explorations; the model is not part of the replay key. The judge answers every judgment on every run: a full run of both viewports makes about 250 judgments. It is Sonnet because a Haiku judge failed about one in eight page checks on screens Sonnet passes, mostly by reading content below the fold as cut off. It takes about 45 minutes on one machine with three workers, which is why CI splits it into four jobs. `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` override the two, but `e2e.config.ts` refuses anything outside `anthropic/` (`src/test/ai/anthropic-only-guard.test.ts` checks it). Use made-up debate content only. An OpenRouter error naming Anthropic (an outage, or the key out of credit) is infrastructure: rerun with `--last-failed`. The agent's context explains that fake AI answers are canned free-transport text. - Replay. When a later check passes, an `agent.act` flow is recorded in [`agentic/replay/`](../agentic/replay/). Later runs replay it with no model call and verify that its effect happened, until the app changes and the agent takes over and records again. The model is not part of the cache key, so a flow recorded with one actor replays under another. A step that ends on a passing toast or other transient state can hand off to the agent on replay (`end-mismatch` in the report); that costs a model call, not a failure. Commit new and changed entries with the test that made them. Local runs read and write the cache; CI only reads it. Changing an instruction, its params, the test or target name, the agent's `context`, or e2e's minor version re-records. - Without a model key or the stack environment, agent tests skip, so a bare `npx e2e run` proves nothing. `failOnSkippedFailure` fails a run in which a test skipped after a failure. diff --git a/e2e.config.ts b/e2e.config.ts index cdbe7c66b..dd7d16189 100644 --- a/e2e.config.ts +++ b/e2e.config.ts @@ -1,6 +1,7 @@ import type { E2EConfig } from "e2e"; import { web } from "@e2e-dev/web"; import { openrouter } from "@openrouter/ai-sdk-provider"; +import { withAnthropicCacheLimit } from "./agentic/support/anthropic-cache"; // Agentic browser tests (TesterArmy e2e) against the verify-debatelab stack. // Run them through `.claude/skills/verify-debatelab/bin/verify.sh e2e`, which @@ -18,15 +19,27 @@ const live = process.env.VERIFY_MODE === "live"; // The testing agent's models, separate from the app's own. Both need tool // calling and image input. The actor drives agent.act and explorations; its // recorded flows replay with no model call, so it runs only on a cache miss -// and in explorations. Ling 3.0 Flash VL is about a fifth of GPT-6 Luna's -// price and drives these flows fine; Gemma 3 12B, cheaper still, could not. -// OpenRouter withdrew the free Qwen tier used before. -// The judge answers every agent.assert, waitFor and extract on every run, so it -// must be cheap and reliable: the Muse Spark contributor tier, which like the -// app's live pin may keep prompts (made-up content only). +// and in explorations. The judge answers every agent.assert, waitFor and +// extract on every run. +// The actor runs on Claude Haiku and the judge on Claude Sonnet: with Haiku +// judging, about one in eight page checks failed on screens Sonnet passes +// (content below the fold read as "cut off", verdicts that contradict their +// own reasoning). Both are pinned to the Anthropic provider with fallbacks off, +// so they bill DebateLab's Anthropic key (BYOK in OpenRouter), never OpenRouter +// credits. Any other family would bill credits, so a non-Claude override fails +// here instead (src/test/ai/anthropic-only-guard.test.ts checks this file). // `||`, not `??`: an unset Actions variable arrives as an empty string. -const model = openrouter(process.env.E2E_AGENT_MODEL || "inclusionai/ling-3.0-flash-vl"); -const judge = openrouter(process.env.E2E_JUDGE_MODEL || "meta/muse-spark-1.3-contributor"); +const E2E_ACTOR_DEFAULT = "anthropic/claude-haiku-5.5"; +const E2E_JUDGE_DEFAULT = "anthropic/claude-sonnet-5.5"; +function claudeOnly(id: string) { + if (!id.startsWith("anthropic/")) { + throw new Error(`The testing agent runs only on Claude through the Anthropic BYOK key; ${id} would bill OpenRouter credits`); + } + // Anthropic allows four prompt-cache breakpoints; see agentic/support/anthropic-cache.ts. + return withAnthropicCacheLimit(openrouter(id, { provider: { only: ["anthropic"], allow_fallbacks: false } })); +} +const model = claudeOnly(process.env.E2E_AGENT_MODEL || E2E_ACTOR_DEFAULT); +const judge = claudeOnly(process.env.E2E_JUDGE_MODEL || E2E_JUDGE_DEFAULT); // `context` reaches every model call, judges included: what the app calls things. const context = [ @@ -49,8 +62,7 @@ const rules = [ "Never delete the account or all data, sign out, or open billing checkout unless the instruction asks for it.", ]; -// The judge cannot act: Meta's endpoint accepts only tool_choice "auto", and -// the act loop requires a tool call. Explorations act with the actor too. +// The judge only judges; the actor drives agent.act and explorations. const agent = (persona?: string) => ({ model, judge, diff --git a/e2e/full-stack/fake-openrouter.mjs b/e2e/full-stack/fake-openrouter.mjs index b61101bdd..e7d64c0d0 100644 --- a/e2e/full-stack/fake-openrouter.mjs +++ b/e2e/full-stack/fake-openrouter.mjs @@ -14,17 +14,25 @@ const port = Number(process.env.PORT ?? 8787); const upstream = process.env.UPSTREAM; const liveModel = process.env.LIVE_MODEL; const chatModel = (requested) => (liveModel === "as-requested" ? requested : liveModel); -// Claude requests pin OpenRouter to the Anthropic provider. A rewritten -// model is served elsewhere, so it keeps only the provider options that -// still apply to it. +// Claude runs only on DebateLab's Anthropic key (BYOK in OpenRouter): every +// Claude call leaves pinned to the Anthropic provider with fallbacks off, and +// any `models` fallback list is dropped. A rewrite to another family (only +// as-requested reaches one) keeps just the options that still apply to it. +const ANTHROPIC_ONLY = { only: ["anthropic"], allow_fallbacks: false }; const liveChatBody = (body) => { const model = chatModel(body.model); - if (model === body.model || !body.provider || typeof body.provider !== "object") return { ...body, model }; - const { only: _only, allow_fallbacks: _allowFallbacks, ...provider } = body.provider; - return { ...body, model, provider }; + const { models: _models, ...rest } = body; + const provider = body.provider && typeof body.provider === "object" ? body.provider : {}; + if (model.startsWith("anthropic/")) return { ...rest, model, provider: { ...provider, ...ANTHROPIC_ONLY } }; + if (model === body.model || !body.provider || typeof body.provider !== "object") return { ...rest, model }; + const { only: _only, allow_fallbacks: _allowFallbacks, ...others } = provider; + return { ...rest, model, provider: others }; }; const maxUpstreamCalls = Number(process.env.MAX_UPSTREAM_CALLS ?? 200); if (upstream && !liveModel) throw new Error("LIVE_MODEL is required with UPSTREAM"); +if (upstream && liveModel !== "as-requested" && !liveModel.startsWith("anthropic/")) { + throw new Error(`LIVE_MODEL must be an anthropic/ model (Anthropic BYOK) or as-requested, not ${liveModel}`); +} const requests = []; // Subscription state the fake Stripe returns, set by tests through /__subscription. const subscriptions = new Map(); diff --git a/scripts/prompt-examples.ts b/scripts/prompt-examples.ts index e532c575c..97eb4bd5b 100644 --- a/scripts/prompt-examples.ts +++ b/scripts/prompt-examples.ts @@ -12,7 +12,7 @@ import { PROMPT_EXAMPLES } from "../supabase/functions/_shared/prompts/prompt-examples.ts"; import { openRouterProviderOptions } from "../supabase/functions/_shared/provider-routing.ts"; -const [functionsDir, outPath, model = "openai/gpt-6-luna", only] = Deno.args; +const [functionsDir, outPath, model = "anthropic/claude-haiku-5.5", only] = Deno.args; const selected = only ? new Set(only.split(",")) : null; if (!functionsDir || !outPath) { console.error("usage: prompt-examples.ts [model]"); diff --git a/src/test/ai/anthropic-cache-breakpoints.test.ts b/src/test/ai/anthropic-cache-breakpoints.test.ts new file mode 100644 index 000000000..9172e95bb --- /dev/null +++ b/src/test/ai/anthropic-cache-breakpoints.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { oneBreakpointPerToolMessage, withAnthropicCacheLimit } from "../../../agentic/support/anthropic-cache"; + +const breakpoint = { type: "ephemeral" }; +const result = (id: string) => ({ type: "tool-result", toolCallId: id, toolName: "click", output: { type: "text", value: "ok" } }); + +describe("Anthropic cache breakpoints on the testing agent's tool messages", () => { + it("keeps one breakpoint, on the last result, when a turn made parallel tool calls", () => { + const message = { role: "tool", content: [result("a"), result("b"), result("c")], providerOptions: { anthropic: { cacheControl: breakpoint } } }; + const fixed = oneBreakpointPerToolMessage(message); + expect(fixed.providerOptions?.anthropic).toEqual({}); + const parts = fixed.content as Array<{ providerOptions?: { anthropic?: { cacheControl?: unknown } } }>; + expect(parts.map((part) => part.providerOptions?.anthropic?.cacheControl)).toEqual([undefined, undefined, breakpoint]); + }); + + it("leaves single results, other roles and unmarked messages alone", () => { + const single = { role: "tool", content: [result("a")], providerOptions: { anthropic: { cacheControl: breakpoint } } }; + const user = { role: "user", content: [{ type: "text", text: "hi" }], providerOptions: { anthropic: { cacheControl: breakpoint } } }; + const unmarked = { role: "tool", content: [result("a"), result("b")] }; + for (const message of [single, user, unmarked]) expect(oneBreakpointPerToolMessage(message)).toBe(message); + }); + + it("wraps a model without changing its id or provider", async () => { + const seen: unknown[] = []; + const model = { + modelId: "anthropic/claude-haiku-5.5", + provider: "openrouter.chat", + async doGenerate(options: { prompt: unknown[] }) { seen.push(options.prompt); return "ok"; }, + }; + const wrapped = withAnthropicCacheLimit(model); + expect(wrapped.modelId).toBe("anthropic/claude-haiku-5.5"); + expect(wrapped.provider).toBe("openrouter.chat"); + await wrapped.doGenerate({ prompt: [{ role: "tool", content: [result("a"), result("b")], providerOptions: { anthropic: { cacheControl: breakpoint } } }] }); + const [prompt] = seen as Array }>>; + expect(prompt[0].content[0].providerOptions).toBeUndefined(); + }); +}); diff --git a/src/test/ai/anthropic-only-guard.test.ts b/src/test/ai/anthropic-only-guard.test.ts new file mode 100644 index 000000000..52cb72c74 --- /dev/null +++ b/src/test/ai/anthropic-only-guard.test.ts @@ -0,0 +1,103 @@ +// DebateLab's model spend must land on the Anthropic key (BYOK in OpenRouter), +// which Adam's Claude plan covers. Anything else bills OpenRouter credits. This +// guard fails when a default, the Auto pools, the test harness, or a request +// payload could send a call to a non-Anthropic model or let OpenRouter fall +// back to another provider or model. A student picking a non-Claude model by +// hand is the one deliberate exception; the model matrix (LIVE_MODEL=as-requested) +// is the one deliberate test exception. +import { describe, expect, it } from 'vitest'; +import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { + BACKGROUND_MODEL, + DEFAULT_MODEL, + DEFAULT_PRO_MODEL, +} from '../../../supabase/functions/_shared/model-catalog'; +import { openRouterProviderOptions } from '../../../supabase/functions/_shared/provider-routing'; + +const root = resolve(__dirname, '../../..'); +const read = (path: string) => readFileSync(resolve(root, path), 'utf8'); +const isClaude = (id: string) => id.startsWith('anthropic/'); + +function sourceFiles(dir: string): string[] { + return readdirSync(resolve(root, dir)).flatMap((name) => { + const path = join(dir, name); + if (name === 'node_modules') return []; + if (statSync(resolve(root, path)).isDirectory()) return sourceFiles(path); + return /\.(ts|tsx|mjs|js|sh)$/.test(name) ? [path] : []; + }); +} + +describe('Anthropic BYOK only', () => { + it('app defaults and the research background model are Claude', () => { + for (const id of [DEFAULT_MODEL, DEFAULT_PRO_MODEL, BACKGROUND_MODEL]) expect(isClaude(id), id).toBe(true); + }); + + it('Claude is pinned to the Anthropic provider with fallbacks off', () => { + for (const id of [DEFAULT_MODEL, DEFAULT_PRO_MODEL, BACKGROUND_MODEL]) { + const options = openRouterProviderOptions(id); + expect(options.only).toEqual(['anthropic']); + expect(options.allow_fallbacks).toBe(false); + } + }); + + it('Auto Fast and Auto Pro route only among Claude models', () => { + const router = read('supabase/functions/_shared/auto-model-router.ts'); + for (const pool of ['FAST_CANDIDATES', 'PRO_CANDIDATES']) { + const body = new RegExp(`export const ${pool} = \\{([\\s\\S]*?)\\} as const;`).exec(router)?.[1]; + expect(body, `${pool} not found`).toBeTruthy(); + const ids = [...body!.matchAll(/^\s*"([^"]+)":/gm)].map((match) => match[1]); + expect(ids.length).toBeGreaterThan(0); + for (const id of ids) expect(isClaude(id), `${pool} has ${id}`).toBe(true); + } + expect(router).toMatch(/fallbackFor[\s\S]*?"anthropic\/claude-haiku-5\.5" : "anthropic\/claude-sonnet-5\.5"/); + }); + + it('the agentic testing agent runs on Claude pinned to Anthropic: Haiku acts, Sonnet judges', () => { + const config = read('e2e.config.ts'); + expect(config).toContain('const E2E_ACTOR_DEFAULT = "anthropic/claude-haiku-5.5";'); + expect(config).toContain('const E2E_JUDGE_DEFAULT = "anthropic/claude-sonnet-5.5";'); + expect(config).toContain('claudeOnly(process.env.E2E_AGENT_MODEL || E2E_ACTOR_DEFAULT)'); + expect(config).toContain('claudeOnly(process.env.E2E_JUDGE_MODEL || E2E_JUDGE_DEFAULT)'); + expect(config).toContain('provider: { only: ["anthropic"], allow_fallbacks: false }'); + expect(config).toContain('if (!id.startsWith("anthropic/"))'); + // No other model ID literal can slip in as a default. + const vendors = /"((?:anthropic|openai|google|meta|meta-llama|deepseek|z-ai|x-ai|mistralai|qwen|inclusionai|moonshotai|minimax|nousresearch|cohere|amazon|microsoft)\/[^"]+)"/g; + const ids = [...config.matchAll(vendors)].map((match) => match[1]); + expect(ids).toContain('anthropic/claude-haiku-5.5'); + for (const id of ids) expect(isClaude(id), id).toBe(true); + }); + + it('live verification and the live gate default to Claude and refuse other families', () => { + const verify = read('.claude/skills/verify-debatelab/bin/verify.sh'); + expect(verify).toMatch(/^LIVE_MODEL="\$\{VERIFY_LIVE_MODEL:-anthropic\/[^}]+\}"$/m); + expect(verify).toMatch(/case "\$LIVE_MODEL" in anthropic\/\*\) ;; \*\)/); + const gate = read('.superset/db-gate.sh'); + expect(gate).toMatch(/^LIVE_MODEL="\$\{LIVE_MODEL:-anthropic\/[^}]+\}"/m); + expect(gate).toMatch(/case "\$LIVE_MODEL" in anthropic\/\*\|as-requested\) ;; \*\)/); + const proxy = read('e2e/full-stack/fake-openrouter.mjs'); + expect(proxy).toContain('const ANTHROPIC_ONLY = { only: ["anthropic"], allow_fallbacks: false };'); + expect(proxy).toContain('liveModel !== "as-requested" && !liveModel.startsWith("anthropic/")'); + }); + + it('no request payload enables provider fallbacks or a non-Claude models list', () => { + const files = [ + ...sourceFiles('supabase/functions'), + ...sourceFiles('scripts'), + ...sourceFiles('agentic'), + ...sourceFiles('e2e'), + ...sourceFiles('.claude/skills/verify-debatelab'), + ...sourceFiles('src/lib'), + 'e2e.config.ts', + ].filter((path) => !/(\.test|_test|\.spec)\.(ts|tsx|mjs)$/.test(path)); + expect(files.length).toBeGreaterThan(50); + for (const path of files) { + const text = read(path); + expect(text, `${path} enables allow_fallbacks`).not.toMatch(/["']?allow_fallbacks["']?\s*:\s*true/); + for (const match of text.matchAll(/["']?\bmodels["']?\s*:\s*\[([^\]]*)\]/g)) { + const ids = [...match[1].matchAll(/["']([^"']+)["']/g)].map((id) => id[1]); + for (const id of ids) expect(isClaude(id), `${path} lists fallback model ${id}`).toBe(true); + } + } + }); +}); diff --git a/supabase/functions/_shared/auto-model-router.test.ts b/supabase/functions/_shared/auto-model-router.test.ts index 2882041a9..3c7870682 100644 --- a/supabase/functions/_shared/auto-model-router.test.ts +++ b/supabase/functions/_shared/auto-model-router.test.ts @@ -1,7 +1,11 @@ import { AUTO_FAST, AUTO_PRO } from "./model-catalog.ts"; import { + FAST_CANDIDATES, + PRO_CANDIDATES, resolveModelSelection, + routeAmongCandidates, type AutoModelRouterJev, + type ResolveModelSelectionArgs, } from "./auto-model-router.ts"; import type { JevDecisionResult } from "./jev.ts"; @@ -29,74 +33,49 @@ function jevReturning(result: JevDecisionResult, observed?: { calls: number; sta }; } -Deno.test("Auto Fast normally resolves to Claude Haiku for a Basic user", async () => { - const resolved = await resolveModelSelection({ - requestedModel: AUTO_FAST, - feature: "generate", - input: { motion: "This house would subsidize public transit", language: "en" }, - canAccessProModels: false, - callJevImpl: jevReturning(choiceResult("anthropic/claude-haiku-5.5")), - }); - if (resolved.requestedModel !== AUTO_FAST || resolved.resolvedModel !== "anthropic/claude-haiku-5.5" || resolved.usedFallback) { - throw new Error(`unexpected Auto Fast resolution: ${JSON.stringify(resolved)}`); +const HAIKU = "anthropic/claude-haiku-5.5"; +const SONNET = "anthropic/claude-sonnet-5.5"; +// A two-model pool to exercise the Jev routing path, which the live Auto pools +// (one Claude model each) skip. +const TEST_POOL = { [HAIKU]: "default", [SONNET]: "harder work" }; + +function poolArgs(overrides: Partial = {}): ResolveModelSelectionArgs { + return { requestedModel: AUTO_PRO, feature: "generate", input: { motion: "Should cities ban cars?" }, canAccessProModels: true, ...overrides }; +} + +Deno.test("Auto pools route only to Claude, so Auto never bills OpenRouter credits", () => { + for (const id of [...Object.keys(FAST_CANDIDATES), ...Object.keys(PRO_CANDIDATES)]) { + if (!id.startsWith("anthropic/")) throw new Error(`non-Anthropic Auto candidate: ${id}`); } + if (Object.hasOwn(PRO_CANDIDATES, "anthropic/claude-opus-5.5")) throw new Error("Auto Pro must not route to Opus"); }); -Deno.test("Auto Fast can route long-context work to Muse Spark", async () => { +Deno.test("Auto Fast resolves to Claude Haiku for a Basic user without a Jev call", async () => { + const observed = { calls: 0 }; const resolved = await resolveModelSelection({ requestedModel: AUTO_FAST, - feature: "attack", - input: { caseText: "A very long multilingual case file requiring broad context handling" }, + feature: "generate", + input: { motion: "This house would subsidize public transit", language: "en" }, canAccessProModels: false, - callJevImpl: jevReturning(choiceResult("meta/muse-spark-1.3-contributor")), + callJevImpl: jevReturning(choiceResult("meta/muse-spark-1.3-contributor"), observed), }); - if (resolved.resolvedModel !== "meta/muse-spark-1.3-contributor" || resolved.usedFallback) { - throw new Error(`long-context route was not accepted: ${JSON.stringify(resolved)}`); + if (resolved.requestedModel !== AUTO_FAST || resolved.resolvedModel !== HAIKU || resolved.usedFallback || resolved.jev) { + throw new Error(`unexpected Auto Fast resolution: ${JSON.stringify(resolved)}`); } + if (observed.calls !== 0) throw new Error("single-candidate Auto Fast called Jev"); }); -Deno.test("Auto Fast falls back to Claude Haiku when Jev is uncertain or unavailable", async () => { - const uncertain = await resolveModelSelection({ - requestedModel: AUTO_FAST, - feature: "ask", - input: { message: "Help me" }, - canAccessProModels: false, - callJevImpl: jevReturning(choiceResult("openai/gpt-6-luna", 0.54)), - }); - if (uncertain.resolvedModel !== "anthropic/claude-haiku-5.5" || !uncertain.usedFallback) throw new Error("uncertain route did not fall back"); - - const unavailable = await resolveModelSelection({ - requestedModel: AUTO_FAST, - feature: "ask", - input: { message: "Help me" }, - canAccessProModels: false, - callJevImpl: () => Promise.reject(Object.assign(new Error("provider unavailable"), { usage: { inputTokens: 9, outputTokens: 0 } })), - }); - if (unavailable.resolvedModel !== "anthropic/claude-haiku-5.5" || !unavailable.usedFallback) throw new Error("failed route did not fall back"); - if (unavailable.jevUsage?.inputTokens !== 9) throw new Error("failed route usage was discarded"); -}); - -Deno.test("Auto Fast ignores a Jev label outside its candidate set", async () => { - const resolved = await resolveModelSelection({ - requestedModel: AUTO_FAST, - feature: "generate", - input: { motion: "Should cities ban cars?" }, - canAccessProModels: false, - callJevImpl: jevReturning(choiceResult("anthropic/claude-opus-5.5")), - }); - if (resolved.resolvedModel !== "anthropic/claude-haiku-5.5" || !resolved.usedFallback) throw new Error("unsupported label escaped candidate gate"); +Deno.test("Auto Pro resolves to Claude Sonnet without a Jev call and never to Opus", async () => { + const observed = { calls: 0 }; + const resolved = await resolveModelSelection(poolArgs({ callJevImpl: jevReturning(choiceResult("anthropic/claude-opus-5.5"), observed) })); + if (resolved.resolvedModel !== SONNET || resolved.usedFallback) throw new Error(`unexpected Auto Pro resolution: ${JSON.stringify(resolved)}`); + if (observed.calls !== 0) throw new Error("single-candidate Auto Pro called Jev"); }); Deno.test("Auto Pro rejects Basic before calling Jev", async () => { const observed = { calls: 0 }; try { - await resolveModelSelection({ - requestedModel: AUTO_PRO, - feature: "generate", - input: { motion: "Should cities ban cars?" }, - canAccessProModels: false, - callJevImpl: jevReturning(choiceResult("openai/gpt-6.1-sol"), observed), - }); + await resolveModelSelection(poolArgs({ canAccessProModels: false, callJevImpl: jevReturning(choiceResult(SONNET), observed) })); } catch (error) { if (!(error instanceof Error) || !error.message.includes("Pro")) throw error; if (observed.calls !== 0) throw new Error("Jev ran before Auto Pro entitlement rejection"); @@ -105,54 +84,6 @@ Deno.test("Auto Pro rejects Basic before calling Jev", async () => { throw new Error("Basic Auto Pro request was accepted"); }); -Deno.test("Auto Pro accepts only its approved smart-model pool", async () => { - for (const model of ["anthropic/claude-sonnet-5.5", "openai/gpt-6.1-sol"]) { - const resolved = await resolveModelSelection({ - requestedModel: AUTO_PRO, - feature: "generate", - input: { motion: "Complex international law motion" }, - canAccessProModels: true, - callJevImpl: jevReturning(choiceResult(model)), - }); - if (resolved.resolvedModel !== model || resolved.usedFallback) throw new Error(`approved Pro route rejected: ${model}`); - } - - const outsidePool = await resolveModelSelection({ - requestedModel: AUTO_PRO, - feature: "generate", - input: { motion: "Complex international law motion" }, - canAccessProModels: true, - callJevImpl: jevReturning(choiceResult("z-ai/glm-5.3-flash")), - }); - if (outsidePool.resolvedModel !== "anthropic/claude-sonnet-5.5" || !outsidePool.usedFallback) { - throw new Error("Auto Pro accepted a model outside its pool"); - } -}); - -Deno.test("Auto Pro never routes or falls back to Opus", async () => { - const routedToOpus = await resolveModelSelection({ - requestedModel: AUTO_PRO, - feature: "generate", - input: { motion: "Complex international law motion" }, - canAccessProModels: true, - callJevImpl: jevReturning(choiceResult("anthropic/claude-opus-5.5")), - }); - if (routedToOpus.resolvedModel !== "anthropic/claude-sonnet-5.5" || !routedToOpus.usedFallback) { - throw new Error(`Auto Pro accepted Opus: ${JSON.stringify(routedToOpus)}`); - } - - const unavailable = await resolveModelSelection({ - requestedModel: AUTO_PRO, - feature: "generate", - input: { motion: "Complex international law motion" }, - canAccessProModels: true, - callJevImpl: () => Promise.reject(new Error("provider unavailable")), - }); - if (unavailable.resolvedModel !== "anthropic/claude-sonnet-5.5" || !unavailable.usedFallback) { - throw new Error(`Auto Pro fell back to the wrong model: ${JSON.stringify(unavailable)}`); - } -}); - Deno.test("manual model selections bypass Jev", async () => { const observed = { calls: 0 }; const resolved = await resolveModelSelection({ @@ -168,6 +99,29 @@ Deno.test("manual model selections bypass Jev", async () => { if (observed.calls !== 0) throw new Error("manual selection called Jev"); }); +Deno.test("a multi-model pool accepts a confident Jev choice from the pool", async () => { + const resolved = await routeAmongCandidates(poolArgs({ callJevImpl: jevReturning(choiceResult(SONNET)) }), TEST_POOL, HAIKU); + if (resolved.resolvedModel !== SONNET || resolved.usedFallback || !resolved.jev) throw new Error(`pool route rejected: ${JSON.stringify(resolved)}`); +}); + +Deno.test("a multi-model pool falls back when Jev is uncertain or unavailable", async () => { + const uncertain = await routeAmongCandidates(poolArgs({ callJevImpl: jevReturning(choiceResult(SONNET, 0.54)) }), TEST_POOL, HAIKU); + if (uncertain.resolvedModel !== HAIKU || !uncertain.usedFallback) throw new Error("uncertain route did not fall back"); + + const unavailable = await routeAmongCandidates(poolArgs({ + callJevImpl: () => Promise.reject(Object.assign(new Error("provider unavailable"), { usage: { inputTokens: 9, outputTokens: 0 } })), + }), TEST_POOL, HAIKU); + if (unavailable.resolvedModel !== HAIKU || !unavailable.usedFallback) throw new Error("failed route did not fall back"); + if (unavailable.jevUsage?.inputTokens !== 9) throw new Error("failed route usage was discarded"); +}); + +Deno.test("a multi-model pool ignores a Jev label outside the pool", async () => { + for (const label of ["anthropic/claude-opus-5.5", "meta/muse-spark-1.3-contributor", "openai/gpt-6.1-sol"]) { + const resolved = await routeAmongCandidates(poolArgs({ callJevImpl: jevReturning(choiceResult(label)) }), TEST_POOL, HAIKU); + if (resolved.resolvedModel !== HAIKU || !resolved.usedFallback) throw new Error(`label outside the pool escaped: ${label}`); + } +}); + Deno.test("router bounds state and propagates caller cancellation", async () => { const observed = { calls: 0, state: undefined as unknown }; const controller = new AbortController(); @@ -180,14 +134,12 @@ Deno.test("router bounds state and propagates caller cancellation", async () => else args.signal?.addEventListener("abort", onAbort, { once: true }); }); }; - const pending = resolveModelSelection({ - requestedModel: AUTO_FAST, + const pending = routeAmongCandidates(poolArgs({ feature: "attack", input: { caseText: "x".repeat(20_000) }, - canAccessProModels: false, signal: controller.signal, callJevImpl, - }); + }), TEST_POOL, HAIKU); controller.abort(new DOMException("User cancelled routing", "AbortError")); try { await pending; diff --git a/supabase/functions/_shared/auto-model-router.ts b/supabase/functions/_shared/auto-model-router.ts index b81674ca1..819039154 100644 --- a/supabase/functions/_shared/auto-model-router.ts +++ b/supabase/functions/_shared/auto-model-router.ts @@ -17,20 +17,21 @@ import type { NormalizedUsage } from "./cost.ts"; const ROUTING_CONFIDENCE = 0.55; const MAX_ROUTING_STATE_CHARS = 8_000; -const FAST_CANDIDATES = { +// Auto only routes to Claude, which runs on DebateLab's Anthropic key (BYOK in +// OpenRouter). Every other family bills OpenRouter credits, so Auto never picks +// one; a student can still choose one by hand. With one candidate per pool the +// Jev routing call is skipped (it would bill OpenRouter credits for nothing). +// Add a candidate only if it is an anthropic/ model: auto-model-router.test.ts +// and src/test/ai/anthropic-only-guard.test.ts fail otherwise. +export const FAST_CANDIDATES = { "anthropic/claude-haiku-5.5": "Default for ordinary DebateLab work: cases, rebuttals, coaching, and structured output.", - "openai/gpt-6-luna": "Best for short latency-sensitive conversation and simple coaching.", - "meta/muse-spark-1.3-contributor": "Best for very long input, multilingual output, or broad context handling.", } as const; // Opus is opt-in only: Auto Pro never routes or falls back to it. -const PRO_CANDIDATES = { +export const PRO_CANDIDATES = { "anthropic/claude-sonnet-5.5": "Default for professional DebateLab work: nuanced rhetoric, adversarial critique, and long-form writing.", - "openai/gpt-6.1-sol": "Best for complex structured reasoning and reliable multi-part analysis.", } as const; -type AutoCandidate = keyof typeof FAST_CANDIDATES | keyof typeof PRO_CANDIDATES; - export type AutoModelRouterJev = (args: CallJevArgs) => Promise; export interface ResolveModelSelectionArgs { @@ -71,7 +72,7 @@ function routingState(args: ResolveModelSelectionArgs) { }; } -function fallbackFor(selection: typeof AUTO_FAST | typeof AUTO_PRO): AutoCandidate { +function fallbackFor(selection: typeof AUTO_FAST | typeof AUTO_PRO): string { return selection === AUTO_FAST ? "anthropic/claude-haiku-5.5" : "anthropic/claude-sonnet-5.5"; } @@ -88,9 +89,26 @@ export async function resolveModelSelection(args: ResolveModelSelectionArgs): Pr return { requestedModel, resolvedModel: requestedModel, modelEntry, usedFallback: false }; } if (requestedModel === AUTO_PRO && !canAccessProModels) throw new Error("Auto Pro requires Pro access"); + const candidates: Readonly> = requestedModel === AUTO_FAST ? FAST_CANDIDATES : PRO_CANDIDATES; + return await routeAmongCandidates(args, candidates, fallbackFor(requestedModel)); +} - const candidates = requestedModel === AUTO_FAST ? FAST_CANDIDATES : PRO_CANDIDATES; - const fallback = fallbackFor(requestedModel); +/** + * Picks one of `candidates` with Jev, or `fallback` when Jev is unsure, fails, + * or names something outside the pool. A single-candidate pool resolves to it + * without a Jev call. Exported for tests; callers use resolveModelSelection. + */ +export async function routeAmongCandidates( + args: ResolveModelSelectionArgs, + candidates: Readonly>, + fallback: string, +): Promise { + const { requestedModel } = args; + const fallbackEntry = getModelCatalogEntry(fallback); + if (!fallbackEntry) throw new Error(`Missing fallback model: ${fallback}`); + if (Object.keys(candidates).length === 1 && Object.hasOwn(candidates, fallback)) { + return { requestedModel, resolvedModel: fallback, modelEntry: fallbackEntry, usedFallback: false }; + } let jev: JevDecisionResult | undefined; try { jev = await (args.callJevImpl ?? callJev)({ @@ -106,10 +124,8 @@ export async function resolveModelSelection(args: ResolveModelSelectionArgs): Pr }); } catch (error) { if (isAbort(error, args.signal)) throw error; - const modelEntry = getModelCatalogEntry(fallback); - if (!modelEntry) throw new Error(`Missing fallback model: ${fallback}`); const jevUsage = getJevErrorUsage(error); - return { requestedModel, resolvedModel: fallback, modelEntry, usedFallback: true, ...(jevUsage ? { jevUsage } : {}) }; + return { requestedModel, resolvedModel: fallback, modelEntry: fallbackEntry, usedFallback: true, ...(jevUsage ? { jevUsage } : {}) }; } const answer = jev.answers.model; diff --git a/supabase/functions/ai/handler-research.test.ts b/supabase/functions/ai/handler-research.test.ts index 0b134df2e..cc3f315ae 100644 --- a/supabase/functions/ai/handler-research.test.ts +++ b/supabase/functions/ai/handler-research.test.ts @@ -770,7 +770,8 @@ Deno.test("AI handlers use strict OpenRouter routing and server-side reasoning t } if (openRouterBodies.length !== MODEL_CATALOG.length + 8) throw new Error(`unexpected OpenRouter call count: ${openRouterBodies.length}`); - if (decisionBodies.length !== 2) throw new Error(`unexpected Decisions call count: ${decisionBodies.length}`); + // Auto Fast and Auto Pro each have one Claude candidate, so neither calls Jev. + if (decisionBodies.length !== 0) throw new Error(`unexpected Decisions call count: ${decisionBodies.length}`); for (let index = 0; index < MODEL_CATALOG.length; index += 1) { const entry = MODEL_CATALOG[index]; const body = openRouterBodies[index];