diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7d69a998a..7abcfd9ee 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -93,6 +93,8 @@ jobs: - name: Database and real workspace journey env: + # ghcr.io rate-limits shared runners; the same images are on ECR. + SUPABASE_INTERNAL_IMAGE_REGISTRY: public.ecr.aws WORKSPACE_GATE_ARTIFACT_ROOT: ${{ runner.temp }}/workspace-gate run: ./.superset/db-gate.sh diff --git a/.github/workflows/live-e2e.yml b/.github/workflows/live-e2e.yml new file mode 100644 index 000000000..5ee031644 --- /dev/null +++ b/.github/workflows/live-e2e.yml @@ -0,0 +1,68 @@ +name: Live provider E2E + +# Real OpenRouter, Jev, and Exa behind the local stack. Runs on pull requests, +# nightly to catch provider drift without code changes, and on demand. +on: + pull_request: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: live-e2e-${{ github.ref }} + cancel-in-progress: true + +jobs: + live-e2e: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + + - name: Set up Supabase CLI + uses: supabase/setup-cli@46f7f98c7f948ad727d22c1e67fab04c223a0520 # v3.0.0 + with: + version: 2.116.0 + + - name: Install dependencies + run: npm ci + + - name: Install Chromium + run: npx playwright install --with-deps chromium + + # db-gate.sh fails before starting anything when either key is missing. + # Accepted risk (2026-09-24): pull-request code runs with these keys, so + # anyone who can push a branch here could read them. Keep them low-limit. + - name: Live provider journeys + env: + LIVE_PROVIDERS: 1 + # Repository variable override; empty falls back to db-gate.sh's default. + LIVE_MODEL: ${{ vars.LIVE_MODEL }} + LIVE_OPENROUTER_API_KEY: ${{ secrets.LIVE_OPENROUTER_API_KEY }} + LIVE_EXA_API_KEY: ${{ secrets.LIVE_EXA_API_KEY }} + SUPABASE_INTERNAL_IMAGE_REGISTRY: public.ecr.aws + WORKSPACE_GATE_ARTIFACT_ROOT: ${{ runner.temp }}/workspace-gate + run: ./.superset/db-gate.sh + + - name: Upload live failure artifacts + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: live-e2e-failure-${{ github.run_id }} + path: | + ${{ runner.temp }}/workspace-gate/ + test-results/ + playwright-report/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/model-matrix.yml b/.github/workflows/model-matrix.yml new file mode 100644 index 000000000..b92e1ecf7 --- /dev/null +++ b/.github/workflows/model-matrix.yml @@ -0,0 +1,75 @@ +name: Model matrix + +# Generates one case with every model in the catalog, unpinned, so a model that +# times out or returns invalid JSON fails here before users hit it. One run +# costs about $1, so it runs weekly and when model routing, prompts, or the ai +# function change. +on: + pull_request: + paths: + - supabase/functions/_shared/** + - supabase/functions/ai/** + - src/lib/ai/** + - e2e/model-matrix/** + - .github/workflows/model-matrix.yml + schedule: + - cron: "29 4 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: model-matrix-${{ github.ref }} + cancel-in-progress: true + +jobs: + model-matrix: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + + - name: Set up Supabase CLI + uses: supabase/setup-cli@46f7f98c7f948ad727d22c1e67fab04c223a0520 # v3.0.0 + with: + version: 2.116.0 + + - name: Install dependencies + run: npm ci + + - name: Install Chromium + run: npx playwright install --with-deps chromium + + # db-gate.sh fails before starting anything when either key is missing. + # Accepted risk (2026-09-24): pull-request code runs with these keys, so + # anyone who can push a branch here could read them. Keep them low-limit. + - name: Case generation with every catalog model + env: + LIVE_PROVIDERS: 1 + LIVE_MODEL: as-requested + LIVE_OPENROUTER_API_KEY: ${{ secrets.LIVE_OPENROUTER_API_KEY }} + LIVE_EXA_API_KEY: ${{ secrets.LIVE_EXA_API_KEY }} + SUPABASE_INTERNAL_IMAGE_REGISTRY: public.ecr.aws + WORKSPACE_GATE_ARTIFACT_ROOT: ${{ runner.temp }}/workspace-gate + run: ./.superset/db-gate.sh + + - name: Upload live failure artifacts + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: model-matrix-failure-${{ github.run_id }} + path: | + ${{ runner.temp }}/workspace-gate/ + test-results/ + playwright-report/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/prod-canary.yml b/.github/workflows/prod-canary.yml new file mode 100644 index 000000000..2b1a582e8 --- /dev/null +++ b/.github/workflows/prod-canary.yml @@ -0,0 +1,49 @@ +name: Production canary + +# Checks the live site after every Vercel production deployment and daily. +# Read-only on deployments. The daily and manual runs also make two real AI +# requests (a streamed Ask and a case) as the dedicated canary account. +on: + deployment_status: + schedule: + - cron: "41 6 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: production-canary + cancel-in-progress: false + +jobs: + canary: + if: >- + github.event_name != 'deployment_status' || + (github.event.deployment_status.state == 'success' && github.event.deployment.environment == 'Production') + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + + - name: Install dependencies + run: npm ci + + - name: Install Chromium + run: npx playwright install --with-deps chromium + + - name: Production canary + env: + PROD_SUPABASE_ANON_KEY: ${{ vars.PROD_SUPABASE_ANON_KEY }} + PROD_CANARY_EMAIL: ${{ secrets.PROD_CANARY_EMAIL }} + PROD_CANARY_PASSWORD: ${{ secrets.PROD_CANARY_PASSWORD }} + CANARY_AI_REQUESTS: ${{ github.event_name != 'deployment_status' }} + run: npx playwright test --config=playwright.production.config.ts diff --git a/.superset/db-gate.sh b/.superset/db-gate.sh index 8dd95a7af..ac98b5a0b 100755 --- a/.superset/db-gate.sh +++ b/.superset/db-gate.sh @@ -14,6 +14,17 @@ source "$DIR/lib/lock.sh" DB_LOCK="$HOME/.superset/debatelab-db.lock" FAKE_PROVIDER=debatelab-fake-openrouter +# LIVE_PROVIDERS=1 runs e2e/live against real OpenRouter, Jev, and Exa instead of +# the fake provider. It needs LIVE_OPENROUTER_API_KEY and LIVE_EXA_API_KEY. +# LIVE_MODEL=as-requested runs e2e/model-matrix instead: one case per catalog model. +LIVE_PROVIDERS="${LIVE_PROVIDERS:-0}" +# GLM Flash is inexpensive and has broad endpoint coverage, so OpenRouter's +# default balanced routing can fail over between providers during this gate. +LIVE_MODEL="${LIVE_MODEL:-z-ai/glm-5.3-flash}" # empty also means default +if [ "$LIVE_PROVIDERS" = 1 ] && { [ -z "${LIVE_OPENROUTER_API_KEY:-}" ] || [ -z "${LIVE_EXA_API_KEY:-}" ]; }; then + echo "LIVE_PROVIDERS=1 requires LIVE_OPENROUTER_API_KEY and LIVE_EXA_API_KEY" >&2 + exit 1 +fi stack_started=false runtime_dir="" owned_pids=() @@ -67,10 +78,11 @@ for attempt in 1 2 3; do if supabase start >>"$runtime_dir/supabase.log" 2>&1; then break; fi if [ "$attempt" -eq 3 ]; then exit 1; fi echo "supabase start failed (attempt $attempt); retrying after $((attempt * 30))s" + tail -n 3 "$runtime_dir/supabase.log" supabase stop --no-backup >/dev/null 2>&1 || true sleep $((attempt * 30)) done -npm run test:db +if [ "$LIVE_PROVIDERS" != 1 ]; then npm run test:db; fi status_json="$(supabase status -o json 2>/dev/null)" status_value() { @@ -90,11 +102,20 @@ fi LOCAL_SERVICE_ROLE_KEY="$service_role_key" \ LOCAL_MAILER_HOST="$mailer_host" \ node scripts/workspace-test-env.mjs .superset/workspace-test.env "$runtime_dir/function.env" +provider_mode=() +if [ "$LIVE_PROVIDERS" = 1 ]; then + # printf is a builtin, so the keys never appear in a process list or log. + grep -v '^OPENROUTER_API_KEY=' "$runtime_dir/function.env" >"$runtime_dir/function.env.next" + printf 'OPENROUTER_API_KEY=%s\nEXA_API_KEY=%s\n' "$LIVE_OPENROUTER_API_KEY" "$LIVE_EXA_API_KEY" >>"$runtime_dir/function.env.next" + mv -- "$runtime_dir/function.env.next" "$runtime_dir/function.env" + chmod 600 "$runtime_dir/function.env" + provider_mode=(-e UPSTREAM=https://openrouter.ai/api -e "LIVE_MODEL=$LIVE_MODEL") +fi # Edge Functions reach the fake provider by container name on the stack network. docker rm -f "$FAKE_PROVIDER" >/dev/null 2>&1 || true docker run -d --rm --name "$FAKE_PROVIDER" --network "supabase_network_$project_id" \ - -p 127.0.0.1:18090:8787 -v "$(pwd)/e2e/full-stack/fake-openrouter.mjs:/fake.mjs:ro" \ + "${provider_mode[@]}" -p 127.0.0.1:18090:8787 -v "$(pwd)/e2e/full-stack/fake-openrouter.mjs:/fake.mjs:ro" \ node:22-alpine node /fake.mjs >/dev/null setsid supabase functions serve --no-verify-jwt \ @@ -125,6 +146,13 @@ for function_name in workspace-invites ai ai-stream demo-code-login admin announ curl --fail --silent --output /dev/null --request OPTIONS --header "Origin: $app_origin" "$function_url" done +if [ "$LIVE_PROVIDERS" = 1 ] && [ "$LIVE_MODEL" = as-requested ]; then + test_selection=(e2e/model-matrix --project=desktop-chromium) +elif [ "$LIVE_PROVIDERS" = 1 ]; then + test_selection=(e2e/live --project=desktop-chromium) +else + test_selection=(e2e/workspace-collaboration.spec.ts e2e/full-stack) +fi LOCAL_SUPABASE_URL="$api_url" \ LOCAL_SUPABASE_ANON_KEY="$anon_key" \ LOCAL_SUPABASE_SERVICE_ROLE_KEY="$service_role_key" \ @@ -135,6 +163,7 @@ COLLABORATION_ALLOWED_ORIGINS="$app_origin" \ SUPABASE_URL="$api_url" \ SUPABASE_ANON_KEY="$anon_key" \ FAKE_OPENROUTER_URL=http://127.0.0.1:18090 \ +LIVE_MODEL="$LIVE_MODEL" \ DEMO_CODE_PEPPER="$(sed -n 's/^DEMO_CODE_PEPPER=//p' .superset/workspace-test.env)" \ PLAYWRIGHT_BASE_URL="$app_origin" \ - npx playwright test --config=playwright.workspace.config.ts + npx playwright test --config=playwright.workspace.config.ts "${test_selection[@]}" diff --git a/.superset/workspace-test.env b/.superset/workspace-test.env index 4ab3b38f0..703812916 100644 --- a/.superset/workspace-test.env +++ b/.superset/workspace-test.env @@ -9,3 +9,5 @@ OPENROUTER_API_KEY=local-fake-openrouter-key OPENROUTER_API_URL=http://debatelab-fake-openrouter:8787/api DEMO_CODE_PEPPER=local-demo-code-pepper-for-browser-gate SPARRING_INTEGRITY_SECRET=local-sparring-integrity-secret-for-browser-gate +STRIPE_SECRET_KEY=sk_test_local-full-stack-unused +STRIPE_WEBHOOK_SECRET=whsec_local-full-stack-test diff --git a/AGENTS.md b/AGENTS.md index 47588b50d..4145a0c1a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,7 +71,11 @@ CI=1 PORT=18080 PLAYWRIGHT_BASE_URL=http://127.0.0.1:18080 npm run test:e2e:webk - Start bug fixes with a failing test or repeatable browser reproduction when a suitable test layer exists. - Stub sessions prove rendering only. Do not call them authenticated backend proof. -- Never put staging or production credentials in jobs, logs, traces, screenshots, commits, comments, or docs. +- Never put staging or production credentials in jobs, logs, traces, screenshots, commits, comments, or docs. Two scoped exceptions reach jobs only through GitHub Actions secrets: + - the protected staging values; + - the dedicated production canary account (`PROD_CANARY_EMAIL` and `PROD_CANARY_PASSWORD`, a regular user capped at 10 AI requests a day), which Adam approved on 2026-09-24. + + Neither exception may appear in logs, traces, or screenshots. No other production credential may enter a job. - Never substitute the production project when local or staging access is unavailable. - Production deployments, data changes, migrations, Edge Function deployments, and secret changes require explicit authorization. - A local pass proves only the local commit. Check the current-head GitHub run before calling CI green. diff --git a/docs/operations.md b/docs/operations.md index d64d51bfe..4e39fd7dd 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -25,6 +25,8 @@ Pull requests also start isolated local Supabase for pgTAP and run desktop plus Non-PR runs upload `dist/` briefly and may build a `linux/amd64` and `linux/arm64` Docker image when registry credentials are present. A successful quality run does not by itself prove that an image was pushed. +[`.github/workflows/live-e2e.yml`](../.github/workflows/live-e2e.yml) runs the local stack against real providers on pull requests and nightly. [`.github/workflows/model-matrix.yml`](../.github/workflows/model-matrix.yml) generates a case with every catalog model weekly and when AI code changes. [`.github/workflows/prod-canary.yml`](../.github/workflows/prod-canary.yml) checks production after each Vercel production deployment and daily. See [testing.md](testing.md) for all three. Both Supabase-starting jobs pull images from `public.ecr.aws`, because ghcr.io rate-limits shared runners. + [`.github/workflows/staging-e2e.yml`](../.github/workflows/staging-e2e.yml) owns protected staging. It accepts eligible successful main-branch quality runs or a manual dispatch from `main`. It runs only when `STAGING_E2E_ENABLED=true` and all protected values exist. ## Production image @@ -111,6 +113,8 @@ Secret names may be documented; values may not. Server-side categories include: Never place server secrets in `VITE_*`, screenshots, traces, logs, comments, commits, or documentation. +GitHub Actions secrets: `LIVE_OPENROUTER_API_KEY` and `LIVE_EXA_API_KEY` for live provider tests, and `PROD_CANARY_EMAIL` and `PROD_CANARY_PASSWORD` for the production canary account. The public anon key is the `PROD_SUPABASE_ANON_KEY` repository variable. To rotate the canary password, set it with the Supabase Auth admin API and pipe it into `gh secret set` over stdin. + ## Rollback Choose rollback before changing production: diff --git a/docs/testing.md b/docs/testing.md index 4a2ab799c..234da1465 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -164,13 +164,63 @@ Each test signs in through the Login page with a fresh demo code and deletes tha - sign-in, reload, sign-out, a rejected code, and an expired token that refreshes before an AI request; - case generation grounded in a 250 KB Library source, provider failure with retry, a zero quota, and a saved case that a fresh browser loads from History; - Attack, Brainstorm, Factoids, streamed Ask (it fails if the client falls back to the non-streaming path), and Ask with citations; -- a judged Lincoln-Douglas sparring round and a Crossfire debrief; -- Library activation and deletion, output-language sync across devices, and flowchart saves; +- a judged Lincoln-Douglas sparring round and a Crossfire debrief after a Jev signal attaches to an answer; +- Library activation and deletion, output-language sync across devices, flowchart saves, and Delete All My Data (it must leave another account's rows intact); +- a promo code redeemed in Settings that restores AI access, and signed Stripe webhooks that grant and revoke Pro while a forged or replayed one changes nothing; - admin announcements seen by another user, denied admin access, and a public share link. Tests assert the rendered result, what the Edge Function sent upstream (read back from the fake provider), and the resulting database rows. They also fail on any uncaught page error or horizontal page overflow. The fake provider rejects prompts it does not recognize, so a prompt change must update it. -These journeys do not prove live provider behavior, web research (the local stack has no search key), Jev decisions (the fake returns 503 and callers use their fallbacks), Stripe, or Google export. +These journeys do not prove live provider behavior, web research (the local stack has no search key), real Jev judgement (the fake answers every Jev question with a fixed confident choice), Stripe Checkout and the billing portal (they call the Stripe API; the local key is a placeholder), the `checkout.session.completed` and `invoice.payment_succeeded` webhooks (they fetch the subscription from Stripe), or Google export. + +### Live provider journeys + +```bash +LIVE_PROVIDERS=1 LIVE_OPENROUTER_API_KEY=... LIVE_EXA_API_KEY=... ./.superset/db-gate.sh +``` + +Live mode uses the same locked stack but skips pgTAP and the fake-provider specs. It runs `e2e/live/` on desktop Chromium against real OpenRouter, Jev, and Exa. Imported web pages are fetched live too. The fake provider becomes a recording proxy. It forwards every chat call to OpenRouter with the model pinned to `LIVE_MODEL` (default `z-ai/glm-5.3-flash`; CI reads the `LIVE_MODEL` repository variable), passes Jev decisions through unchanged, and refuses calls past `MAX_UPSTREAM_CALLS` (200). The default is inexpensive and broadly hosted, so requests can use OpenRouter's balanced provider routing instead of depending on one endpoint. + +The specs cover web-researched case generation, verified Factoids citations, Ask with web search, URL import, streamed Ask, Attack, Brainstorm, a judged sparring round, and Crossfire with live Jev signals. Outputs vary, so tests assert structure rather than text: + +- the Edge Function response is 200 and has the expected shape; +- research returned `http(s)` sources and the UI rendered them; +- database rows were written; +- every provider call returned 200 and used the pinned model. + +A live failure means the pipeline broke with this model. It does not prove how production models behave, because each model has its own latency and JSON quality. + +`.github/workflows/live-e2e.yml` runs this on pull requests, nightly at 03:17 UTC, and on demand, using the `LIVE_OPENROUTER_API_KEY` and `LIVE_EXA_API_KEY` repository secrets. The gate exits before starting anything when either key is missing, so a misconfigured job fails instead of skipping. + +### Model matrix + +```bash +LIVE_PROVIDERS=1 LIVE_MODEL=as-requested LIVE_OPENROUTER_API_KEY=... LIVE_EXA_API_KEY=... ./.superset/db-gate.sh +``` + +`LIVE_MODEL=as-requested` switches the gate to `e2e/model-matrix/`. The proxy forwards whatever model the Edge Function chose, and the spec picks each model in `MODEL_CATALOG` from the Generate page's model menu and generates one researched case. A model that exhausts the `ai` function's shared 130-second request budget or returns invalid JSON fails its test. Each test prints the provider latency (`latency : `) so the run log shows how close each model gets to the budget. + +`.github/workflows/model-matrix.yml` runs this on Mondays at 04:29 UTC, on demand, and on pull requests that change `supabase/functions/_shared/`, `supabase/functions/ai/`, or `src/lib/ai/`. It is not a required check, because it runs only on those paths. One run costs about $1 in model and search usage. + +## Production canary + +```bash +PROD_SUPABASE_ANON_KEY=... PROD_CANARY_EMAIL=... PROD_CANARY_PASSWORD=... \ +npx playwright test --config=playwright.production.config.ts +``` + +`e2e/production/canary.spec.ts` checks `https://debate.pi-stack.org` directly: + +- the site loads and signed-out app routes redirect to Login; +- every browser-facing Edge Function answers a preflight that allows the site origin (`workspace-invites` is excluded until production enables Workspaces); +- `ai` returns 401, not a 5xx, to an unauthenticated call; +- the canary account signs in, opens Generate, and Settings shows its Pro plan (a row-level-security read), without spending an AI request; +- the dedicated canary account gets a streamed GPT-5.6 Luna answer from `ai-stream`; +- the canary account generates a three-contention case with GPT-5.6 Luna through `ai`. + +The last two checks are the only production writes, and they touch only the canary account: its `ai_requests` usage rows and whatever the Generate page saves for it. The dedicated canary account has Pro promo access capped at 10 requests a day. Captures are disabled because they could retain its session. + +`.github/workflows/prod-canary.yml` runs after every successful Vercel `Production` deployment, daily at 06:41 UTC, and on demand. Main can deploy more than 10 times a day, so deployment runs set `CANARY_AI_REQUESTS=false` and skip the two AI checks. The signed-in session check still runs on every deployment. Only the daily and manual runs make real AI requests: two per run, plus at most two retries. It reads the password and email from secrets and the public anon key from the `PROD_SUPABASE_ANON_KEY` repository variable. The local stack begins at port `54321` and is not parallel-safe across worktrees. The task that starts it must stop it before finishing or abandoning the run. Use `supabase stop --no-backup` unless another active task explicitly owns the same stack. @@ -232,6 +282,8 @@ A skipped staging workflow is not a pass. Missing credentials or provider access [`.github/workflows/staging-e2e.yml`](../.github/workflows/staging-e2e.yml) owns protected staging conditions. +[`.github/workflows/live-e2e.yml`](../.github/workflows/live-e2e.yml) runs the live provider journeys, and [`.github/workflows/prod-canary.yml`](../.github/workflows/prod-canary.yml) runs the production canary. Branch protection on `main` requires `quality`, `database-tests`, `browser-tests`, and `live-e2e`. It also requires a pull request, and administrators are not exempt. + Before calling CI green: 1. identify the current head commit; @@ -247,3 +299,5 @@ Before calling CI green: - Local browser pass: no current-head CI proof. - Credential-free E2E: no staging synchronization proof. - Staging pass: no authorization to deploy or mutate production. +- Live provider pass: no proof for the production models, only for the pinned test model. +- Canary pass: production was healthy at that moment; it covers one AI path, not every feature. diff --git a/e2e/full-stack/account-data.spec.ts b/e2e/full-stack/account-data.spec.ts index 1f5a2ea68..acff3e971 100644 --- a/e2e/full-stack/account-data.spec.ts +++ b/e2e/full-stack/account-data.spec.ts @@ -1,4 +1,4 @@ -import { admin, expect, MARK, providerCall, signIn, stackReady, test } from "./stack"; +import { admin, createDemoUser, deleteDemoUser, expect, MARK, providerCall, signIn, stackReady, test } from "./stack"; import type { Page } from "@playwright/test"; test.skip(!stackReady, "Local full stack is required (run ./.superset/db-gate.sh)"); @@ -37,6 +37,41 @@ test("only active Library sources ground AI requests, and deletions persist", as expect(data?.map((row) => row.title)).toEqual(["Active study"]); }); +test("Delete All My Data erases the account's saved work and nobody else's", async ({ signedIn: page, user }) => { + const other = await createDemoUser("bystander"); + try { + // A signed-in session creates the bystander's rows, so this also proves the erase is scoped by user. + const bystander = await (await page.context().browser()!.newContext()).newPage(); + await bystander.addInitScript(() => localStorage.setItem("debate-trainer-onboarding", JSON.stringify({ state: { hasCompletedOnboarding: true, isTourRunning: false }, version: 0 }))); + await signIn(bystander, other); + await bystander.goto("/app/library"); + await addSource(bystander, "Bystander study", "Kept after another account erases its data."); + await bystander.context().close(); + + await page.goto("/app/library"); + await addSource(page, "My study", "Erased with the account data."); + const userId = await user.userId(); + const otherId = await other.userId(); + + await page.goto("/app/settings"); + await page.getByRole("button", { name: "Delete All My Data" }).click(); + const dialog = page.getByRole("dialog"); + await dialog.getByLabel("Type your email to confirm").fill(user.email); + await dialog.getByRole("button", { name: "Delete All My Data" }).click(); + await expect(page.getByText("Data Deleted").first()).toBeVisible(); + + const { data: mine } = await admin.from("sources").select("id").eq("user_id", userId); + expect(mine).toEqual([]); + const { data: theirs } = await admin.from("sources").select("title").eq("user_id", otherId); + expect(theirs?.map((row) => row.title)).toEqual(["Bystander study"]); + + await page.goto("/app/library"); + await expect(page.getByRole("switch", { name: "Toggle My study" })).toHaveCount(0); + } finally { + await deleteDemoUser(other); + } +}); + test("the output language setting follows the account and reaches the model", async ({ signedIn: page, user, browser }) => { await page.goto("/app/settings"); await page.locator("div.space-y-2", { has: page.getByText("AI Output Language", { exact: true }) }).getByRole("combobox").click(); diff --git a/e2e/full-stack/billing.spec.ts b/e2e/full-stack/billing.spec.ts new file mode 100644 index 000000000..7d40b3ec4 --- /dev/null +++ b/e2e/full-stack/billing.spec.ts @@ -0,0 +1,134 @@ +import { createHmac, randomUUID } from "node:crypto"; +import type { Page } from "@playwright/test"; +import { admin, expect, fake, MARK, providerCall, stackReady, test } from "./stack"; + +test.skip(!stackReady, "Local full stack is required (run ./.superset/db-gate.sh)"); + +// Matches STRIPE_WEBHOOK_SECRET in .superset/workspace-test.env. +const webhookSecret = "whsec_local-full-stack-test"; + +// Demo sign-in grants lifetime Pro and new accounts get a trial, so each test first removes both. +async function expireAccess(userId: string) { + const { error } = await admin.from("user_settings") + .update({ subscription_tier: "expired", trial_ends_at: null, grace_period_ends_at: null, daily_limit_override: null }) + .eq("user_id", userId); + if (error) throw error; +} + +async function tier(userId: string) { + const { data } = await admin.from("user_settings").select("subscription_tier").eq("user_id", userId).single(); + return data?.subscription_tier; +} + +async function brainstorm(page: Page) { + await page.goto("/app/brainstorm"); + await page.getByLabel("Motion / Topic *").fill("Public transport should be free"); + await page.getByRole("button", { name: "Start Brainstorming" }).click(); +} + +function sendWebhook(event: object, signature = true) { + const payload = JSON.stringify(event); + const timestamp = Math.floor(Date.now() / 1000); + const digest = createHmac("sha256", signature ? webhookSecret : "whsec_forged").update(`${timestamp}.${payload}`).digest("hex"); + return fetch(`${process.env.LOCAL_SUPABASE_URL}/functions/v1/stripe-webhook`, { + method: "POST", + headers: { "content-type": "application/json", "stripe-signature": `t=${timestamp},v1=${digest}` }, + body: payload, + }); +} + +function subscriptionEvent(type: string, subscriptionId: string, status: string) { + const now = Math.floor(Date.now() / 1000); + return { + id: `evt_${randomUUID()}`, + object: "event", + type, + data: { + object: { + id: subscriptionId, + object: "subscription", + status, + cancel_at_period_end: false, + current_period_start: now, + current_period_end: now + 30 * 86_400, + cancel_at: null, + canceled_at: null, + ended_at: null, + }, + }, + }; +} + +test("a promo code redeemed in Settings restores AI access", async ({ signedIn: page, user }) => { + const userId = await user.userId(); + await expireAccess(userId); + const code = `E2E${randomUUID().slice(0, 8).toUpperCase()}`; + const { data: promo, error } = await admin.from("promo_codes") + .insert({ code, type: "free_month", config: { days: 30 }, max_redemptions: 1 }) + .select("id") + .single(); + if (error) throw error; + + try { + await brainstorm(page); + await expect(page.getByRole("dialog").getByText("Free Trial Ended")).toBeVisible({ timeout: 30_000 }); + expect((await fake.requests()).filter((request) => request.feature === "brainstorm")).toEqual([]); + + await page.goto("/app/settings"); + await expect(page.getByText("Expired", { exact: true })).toBeVisible(); + await page.getByRole("button", { name: "Upgrade to Pro" }).click(); + await page.getByRole("button", { name: "Have a promo code?" }).click(); + await page.getByPlaceholder("Enter promo code").fill(code.toLowerCase()); + await page.getByRole("button", { name: "Apply", exact: true }).click(); + await expect(page.getByText("Promo code applied!").first()).toBeVisible(); + await expect(page.getByRole("dialog")).toBeHidden(); + await expect(page.getByText("Pro", { exact: true })).toBeVisible(); + expect(await tier(userId)).toBe("pro"); + + await brainstorm(page); + await expect(page.getByText(`${MARK} idea 1`)).toBeVisible({ timeout: 30_000 }); + await providerCall("brainstorm"); + } finally { + await admin.from("promo_codes").delete().eq("id", promo.id); + } +}); + +test("signed Stripe webhooks grant and revoke Pro, and forged ones change nothing", async ({ signedIn: page, user }) => { + const userId = await user.userId(); + await expireAccess(userId); + const subscriptionId = `sub_e2e_${randomUUID()}`; + const now = Date.now(); + const { error } = await admin.from("subscriptions").insert({ + user_id: userId, + stripe_customer_id: `cus_e2e_${randomUUID()}`, + stripe_subscription_id: subscriptionId, + status: "incomplete", + current_period_start: new Date(now).toISOString(), + current_period_end: new Date(now + 30 * 86_400_000).toISOString(), + }); + if (error) throw error; + + const activated = subscriptionEvent("customer.subscription.updated", subscriptionId, "active"); + const forged = await sendWebhook(activated, false); + expect(forged.status).toBe(400); + expect(await tier(userId)).toBe("expired"); + + const accepted = await sendWebhook(activated); + expect(accepted.status).toBe(200); + expect(await tier(userId)).toBe("pro"); + // Stripe retries deliveries, so a replayed event must be acknowledged, not reapplied. + expect(await (await sendWebhook(activated)).json()).toMatchObject({ duplicate: true }); + + await page.goto("/app/settings"); + await expect(page.getByText("Pro", { exact: true })).toBeVisible(); + await expect(page.getByRole("button", { name: "Manage Subscription" })).toBeVisible(); + + const canceled = await sendWebhook(subscriptionEvent("customer.subscription.deleted", subscriptionId, "canceled")); + expect(canceled.status).toBe(200); + expect(await tier(userId)).toBe("expired"); + const { data: row } = await admin.from("subscriptions").select("status").eq("stripe_subscription_id", subscriptionId).single(); + expect(row?.status).toBe("canceled"); + + await page.reload(); + await expect(page.getByText("Expired", { exact: true })).toBeVisible(); +}); diff --git a/e2e/full-stack/case-generation.spec.ts b/e2e/full-stack/case-generation.spec.ts index 406da9df1..a50b8ccb5 100644 --- a/e2e/full-stack/case-generation.spec.ts +++ b/e2e/full-stack/case-generation.spec.ts @@ -1,3 +1,4 @@ +import { readFile } from "node:fs/promises"; import { admin, expect, fake, MARK, providerCall, signIn, stackReady, test } from "./stack"; test.skip(!stackReady, "Local full stack is required (run ./.superset/db-gate.sh)"); @@ -84,3 +85,20 @@ test("a saved case persists to the account and reloads from History", async ({ s await expect(other.getByText(title)).toBeVisible(); await fresh.close(); }); + +test("a generated case exports to PDF and DOCX files", async ({ signedIn: page }) => { + await page.goto("/app/generate"); + await page.getByRole("textbox", { name: "Motion / Topic *" }).fill(motion); + await page.getByRole("button", { name: "Generate Debate Case" }).click(); + await expect(page.getByText(`${MARK} case thesis`)).toBeVisible({ timeout: 30_000 }); + + // PDF files start with "%PDF"; DOCX files are ZIP archives starting with "PK". + for (const [button, extension, magic] of [["PDF", ".pdf", "%PDF"], ["DOCX", ".docx", "PK"]] as const) { + const download = page.waitForEvent("download"); + await page.getByRole("button", { name: button, exact: true }).click(); + const file = await download; + expect(file.suggestedFilename()).toMatch(new RegExp(`\\${extension}$`)); + const bytes = await readFile((await file.path())!); + expect(bytes.subarray(0, magic.length).toString("latin1"), `${button} file signature`).toBe(magic); + } +}); diff --git a/e2e/full-stack/fake-openrouter.mjs b/e2e/full-stack/fake-openrouter.mjs index aa9ff1f42..303f58a8d 100644 --- a/e2e/full-stack/fake-openrouter.mjs +++ b/e2e/full-stack/fake-openrouter.mjs @@ -3,11 +3,22 @@ // OPENROUTER_API_URL, so their auth, quota, source loading, prompt building, // and output validation all run for real. Unknown prompts fail loudly: a prompt // change must update this file rather than silently receive a generic answer. +// +// With UPSTREAM set, it becomes a recording proxy for the live gate instead: +// every chat call goes to the real provider with the model pinned to +// LIVE_MODEL, and calls past MAX_UPSTREAM_CALLS are refused to bound cost. +// LIVE_MODEL=as-requested forwards the model the Edge Function chose. import { createServer } from "node:http"; const port = Number(process.env.PORT ?? 8787); +const upstream = process.env.UPSTREAM; +const liveModel = process.env.LIVE_MODEL; +const chatModel = (requested) => (liveModel === "as-requested" ? requested : liveModel); +const maxUpstreamCalls = Number(process.env.MAX_UPSTREAM_CALLS ?? 200); +if (upstream && !liveModel) throw new Error("LIVE_MODEL is required with UPSTREAM"); const requests = []; let failures = []; +let upstreamCalls = 0; const MARK = "FAKE-PROVIDER"; @@ -166,6 +177,17 @@ function classify(body) { return { feature: route[0], system, user, output: route[2](user) }; } +function jevAnswers(questions) { + return Object.fromEntries(Object.entries(questions).map(([id, question]) => { + if (question.type === "noul") return [id, { type: "noul", noul: 0.8 }]; + const labels = question.type === "choice" ? Object.keys(question.criteria) : question.criteria.map((_, index) => String(index)); + const probabilities = Object.fromEntries(labels.map((label, index) => [label, index === 0 ? 0.9 : 0.1 / (labels.length - 1)])); + if (question.type === "choice") return [id, { type: "choice", choice: labels[0], confidence: 0.9, probabilities }]; + const legend = Object.fromEntries(question.criteria.map((criterion, index) => [String(index), criterion])); + return [id, { type: "score", score: 0, confidence: 0.9, probabilities, legend }]; + })); +} + function json(res, status, value) { res.writeHead(status, { "content-type": "application/json" }); res.end(JSON.stringify(value)); @@ -188,6 +210,40 @@ async function readBody(req) { return Buffer.concat(chunks).toString("utf8"); } +async function proxy(req, res, path) { + const body = JSON.parse(await readBody(req)); + const chat = path === "/api/v1/chat/completions"; + const record = chat + ? { ...classify(body), requestedModel: body.model, model: chatModel(body.model), stream: body.stream === true } + : { feature: "jev", model: body.model }; + delete record.output; + requests.push(record); + if (++upstreamCalls > maxUpstreamCalls) { + record.status = 429; + return json(res, 429, { error: { message: `Live gate stopped after ${maxUpstreamCalls} provider calls` } }); + } + const started = Date.now(); + try { + const response = await fetch(`${upstream}${path.replace(/^\/api/, "")}`, { + method: "POST", + headers: { authorization: req.headers.authorization ?? "", "content-type": "application/json" }, + body: JSON.stringify(chat ? { ...body, model: chatModel(body.model) } : body), + }); + record.status = response.status; + res.writeHead(response.status, { "content-type": response.headers.get("content-type") ?? "application/json" }); + for await (const chunk of response.body ?? []) res.write(chunk); + res.end(); + } catch (error) { + record.status = 502; + const message = error instanceof Error ? error.message : "upstream connection failed"; + console.error(`live provider request failed: ${message}`); + if (!res.headersSent) json(res, 502, { error: { message: "Live provider request failed" } }); + else res.destroy(); + } finally { + record.ms = Date.now() - started; + } +} + createServer(async (req, res) => { const url = new URL(req.url ?? "/", "http://fake"); if (url.pathname === "/__requests" && req.method === "GET") return json(res, 200, requests); @@ -201,10 +257,15 @@ createServer(async (req, res) => { failures.push({ feature, count, status }); return json(res, 200, { ok: true }); } - if (url.pathname === "/api/alpha/decisions") { - // Jev is optional: callers fall back to deterministic ranking and signals. - requests.push({ feature: "jev", path: url.pathname }); - return json(res, 503, { error: { message: "Jev is not faked" } }); + if (upstream && req.method === "POST" && (url.pathname === "/api/v1/chat/completions" || url.pathname === "/api/alpha/decisions")) { + return proxy(req, res, url.pathname); + } + if (url.pathname === "/api/alpha/decisions" && req.method === "POST") { + // Confident, schema-valid answers so Jev-dependent paths (Crossfire + // signals, source ranking) run end to end instead of falling back. + const body = JSON.parse(await readBody(req)); + requests.push({ feature: "jev", model: body.model, status: 200 }); + return json(res, 200, { model: body.model, answers: jevAnswers(body.questions ?? {}), usage: { input_tokens: 40, output_tokens: 8, cost: 0.00001 } }); } if (url.pathname !== "/api/v1/chat/completions" || req.method !== "POST") { return json(res, 404, { error: { message: `Unexpected ${req.method} ${url.pathname}` } }); @@ -228,4 +289,4 @@ createServer(async (req, res) => { choices: [{ index: 0, finish_reason: "stop", message: { role: "assistant", content: JSON.stringify(output) } }], usage, }); -}).listen(port, "0.0.0.0", () => console.log(`fake OpenRouter listening on ${port}`)); +}).listen(port, "0.0.0.0", () => console.log(`${upstream ? "live proxy" : "fake OpenRouter"} listening on ${port}`)); diff --git a/e2e/full-stack/practice.spec.ts b/e2e/full-stack/practice.spec.ts index 35823cb72..4367c0479 100644 --- a/e2e/full-stack/practice.spec.ts +++ b/e2e/full-stack/practice.spec.ts @@ -44,6 +44,9 @@ test("Crossfire answers a question and produces a debrief", async ({ signedIn: p await page.getByTestId("submit-turn").click(); await expect(page.getByText(`${MARK} crossfire answer`)).toBeVisible({ timeout: 30_000 }); expect((await providerCall("crossfire")).user).toContain("What evidence supports the safety mechanism?"); + // The Jev signal attaches to the answer turn, which every later request resends. + await expect(page.getByText("Direct answer")).toBeVisible(); + await providerCall("jev"); await page.getByTestId("end-crossfire").click(); await expect(page.getByTestId("crossfire-debrief")).toBeVisible({ timeout: 30_000 }); diff --git a/e2e/live/live.ts b/e2e/live/live.ts new file mode 100644 index 000000000..7033b9d90 --- /dev/null +++ b/e2e/live/live.ts @@ -0,0 +1,32 @@ +import type { Page } from "@playwright/test"; +import { expect, fake } from "../full-stack/stack"; + +// Live runs call real providers, so outputs vary. Assert structure, never text. +export const LIVE_TIMEOUT = 180_000; + +/** Resolves with the parsed body of the next `ai` Edge Function response for a feature. */ +export function aiResponse(page: Page, feature: string) { + return page.waitForResponse((response) => { + if (!response.url().includes("/functions/v1/ai") || response.request().method() !== "POST") return false; + try { + return (response.request().postDataJSON() as { feature?: string }).feature === feature; + } catch { + return false; + } + }, { timeout: LIVE_TIMEOUT }).then(async (response) => { + expect(response.status(), `ai ${feature} status`).toBe(200); + return response.json() as Promise<{ output: Record; discoveredSources?: unknown[] }>; + }); +} + +/** Every provider call went through the pinned live model and succeeded. */ +export async function expectProviderCallsHealthy(features: string[]) { + const calls = await fake.requests() as Array<{ feature: string | null; model?: string; status?: number }>; + for (const feature of features) { + expect(calls.map((call) => call.feature), `provider calls include ${feature}`).toContain(feature); + } + const failed = calls.filter((call) => call.status !== 200); + expect(failed, "provider calls that did not return 200").toEqual([]); + const chatModels = new Set(calls.filter((call) => call.feature !== "jev").map((call) => call.model)); + expect([...chatModels], "chat calls use only the live model").toEqual([process.env.LIVE_MODEL]); +} diff --git a/e2e/live/models.spec.ts b/e2e/live/models.spec.ts new file mode 100644 index 000000000..7deef9a40 --- /dev/null +++ b/e2e/live/models.spec.ts @@ -0,0 +1,95 @@ +import { admin, expect, fake, stackReady, test } from "../full-stack/stack"; +import { aiResponse, expectProviderCallsHealthy, LIVE_TIMEOUT } from "./live"; + +test.skip(!stackReady, "Live gate is required (LIVE_PROVIDERS=1 ./.superset/db-gate.sh)"); +test.describe.configure({ timeout: LIVE_TIMEOUT + 60_000 }); + +test("Ask streams a live answer", async ({ signedIn: page }) => { + await page.goto("/app/ask"); + await page.getByPlaceholder("Ask your debate coach...").fill("In two sentences, how should I weigh access against cost?"); + const stream = page.waitForResponse((response) => response.url().endsWith("/functions/v1/ai-stream"), { timeout: LIVE_TIMEOUT }); + await page.getByRole("button", { name: "Send message" }).click(); + + // A 200 stream can still end early; only the final event proves a complete answer. + const response = await stream; + expect(response.status()).toBe(200); + const body = await response.text(); + expect(body).toContain("event: final"); + expect(body).not.toContain("event: error"); + await expect(page.getByText("Response was interrupted")).toHaveCount(0); + // The client falls back to the structured endpoint if streaming fails. + expect((await fake.requests()).map((call) => call.feature)).not.toContain("ask"); + await expectProviderCallsHealthy(["ask-stream"]); +}); + +test("Attack and Brainstorm return valid structured analyses", async ({ signedIn: page }) => { + await page.goto("/app/attack"); + await page.getByLabel("Your Argument *").fill("Free transit cuts congestion because riders leave their cars at home, so cities should abolish fares."); + const attack = aiResponse(page, "attack"); + await page.getByRole("button", { name: "Find Weaknesses" }).click(); + expect(((await attack).output.weaknesses as unknown[]).length).toBeGreaterThan(0); + + await page.goto("/app/brainstorm"); + await page.getByLabel("Motion / Topic *").fill("Universal basic income should replace targeted welfare"); + const brainstorm = aiResponse(page, "brainstorm"); + await page.getByRole("button", { name: "Start Brainstorming" }).click(); + expect(((await brainstorm).output.topFive as unknown[]).length).toBeGreaterThan(0); + await expectProviderCallsHealthy(["attack", "brainstorm"]); +}); + +test("a live sparring round is judged and saved", async ({ signedIn: page, user }) => { + await page.goto("/app/sparring"); + await page.getByLabel("Motion").fill("This House would make public transport free at the point of use."); + await page.getByRole("combobox", { name: "Round format" }).click(); + await page.getByRole("option", { name: /Lincoln-Douglas ยท 5 speeches/ }).click(); + await page.getByRole("button", { name: "Start round" }).click(); + + const speeches = [ + ["Affirmative constructive", "Universal access removes the direct price barrier to participation in work, school, and civic life."], + ["First affirmative rebuttal", "Targeted discounts leave eligibility gaps and add administrative friction that keeps poor riders out."], + ["Second affirmative rebuttal", "Weigh reliability: the universal policy covers every marginal trip, while targeting misses many of them."], + ] as const; + for (const [turn, content] of speeches) { + const draft = page.getByRole("textbox", { name: `Draft for ${turn}` }); + await expect(draft).toBeVisible({ timeout: LIVE_TIMEOUT }); + await draft.fill(content); + await page.getByRole("button", { name: "Submit speech" }).click(); + } + + await expect(page.getByRole("heading", { name: "Here is what happened in the room." })).toBeVisible({ timeout: LIVE_TIMEOUT }); + await expect.poll(async () => { + const { data } = await admin.from("sparring_sessions").select("status").eq("user_id", await user.userId()); + return data?.map((row) => row.status); + }).toEqual(["complete"]); + await expectProviderCallsHealthy(["sparring"]); +}); + +test("Crossfire answers live and produces a debrief", async ({ signedIn: page }) => { + await page.goto("/app/crossfire"); + await page.getByTestId("crossfire-resolution").fill("Resolved: cities should ban private cars downtown."); + await page.getByTestId("crossfire-argument").fill("Private cars create congestion and unsafe streets."); + await page.getByTestId("mode-question-ai").click(); + await page.getByTestId("start-crossfire").click(); + + await page.getByTestId("crossfire-input").fill("What evidence links fewer private cars to safer streets?"); + const turn = aiResponse(page, "crossfire"); + const signal = page.waitForResponse((response) => { + if (!response.url().includes("/functions/v1/ai")) return false; + try { + return (response.request().postDataJSON() as { input?: { operation?: string } }).input?.operation === "signal"; + } catch { + return false; + } + }, { timeout: LIVE_TIMEOUT }); + await page.getByTestId("submit-turn").click(); + const answer = (await turn).output as { type: string; text: string }; + expect(answer.type).toBe("answer"); + await expect(page.getByText(answer.text.slice(0, 40), { exact: false })).toBeVisible(); + // The answer's signal is judged by a live Jev decision, recorded by the proxy. + expect((await signal).status()).toBe(200); + await expect.poll(async () => (await fake.requests()).some((call) => call.feature === "jev")).toBe(true); + + await page.getByTestId("end-crossfire").click(); + await expect(page.getByTestId("crossfire-debrief")).toBeVisible({ timeout: LIVE_TIMEOUT }); + await expectProviderCallsHealthy(["crossfire", "jev"]); +}); diff --git a/e2e/live/research.spec.ts b/e2e/live/research.spec.ts new file mode 100644 index 000000000..980d05a0d --- /dev/null +++ b/e2e/live/research.spec.ts @@ -0,0 +1,77 @@ +import { admin, expect, stackReady, test } from "../full-stack/stack"; +import { aiResponse, expectProviderCallsHealthy, LIVE_TIMEOUT } from "./live"; + +test.skip(!stackReady, "Live gate is required (LIVE_PROVIDERS=1 ./.superset/db-gate.sh)"); +test.describe.configure({ timeout: LIVE_TIMEOUT + 60_000 }); + +const httpLink = /^https?:\/\//; + +test("case generation researches the web and cites what it found", async ({ signedIn: page, user }) => { + await page.goto("/app/generate"); + await page.getByRole("textbox", { name: "Motion / Topic *" }).fill("This House would make public transport free at the point of use"); + const response = aiResponse(page, "generate"); + await page.getByRole("button", { name: "Generate Debate Case" }).click(); + const body = await response; + + // Standard length asks for exactly three contentions. + expect(body.output.contentions).toHaveLength(3); + const sources = body.discoveredSources as Array<{ url: string }>; + expect(sources.length, "web sources found by research").toBeGreaterThan(0); + expect(sources.every((source) => httpLink.test(source.url))).toBe(true); + + const panel = page.getByText("Researched sources").locator(".."); + await expect(panel).toBeVisible(); + await expect(panel.getByRole("link").first()).toHaveAttribute("href", httpLink); + + const { data } = await admin.from("ai_requests").select("feature,status").eq("user_id", await user.userId()).eq("feature", "generate"); + expect(data).toEqual([expect.objectContaining({ status: "success" })]); + await expectProviderCallsHealthy(["generate"]); +}); + +test("Factoids returns verified evidence with working citations", async ({ signedIn: page }) => { + await page.goto("/app/factoids"); + await page.getByLabel("Statement *").fill("Fare-free public transport increases ridership"); + const response = aiResponse(page, "factoids"); + await page.getByRole("button", { name: "Find Factoids" }).click(); + const body = await response; + + // The server keeps only factoids whose citation matches fetched research. + const factoids = body.output.factoids as Array<{ shortClaim: string; citation: { url?: string; verified: string } }>; + expect(factoids.length, "verified factoids").toBeGreaterThan(0); + for (const factoid of factoids) { + expect(factoid.citation.verified).toBe("verified"); + expect(factoid.citation.url).toMatch(httpLink); + } + await expect(page.getByText(factoids[0].shortClaim).first()).toBeVisible(); + await expectProviderCallsHealthy(["factoids"]); +}); + +test("Ask with web search shows the sources it used", async ({ signedIn: page }) => { + await page.goto("/app/ask"); + await page.locator("#web-search-mode").click(); + await page.getByPlaceholder("Ask your debate coach...").fill("What did recent studies find about fare-free public transport ridership?"); + const response = aiResponse(page, "ask"); + await page.getByRole("button", { name: "Send message" }).click(); + const body = await response; + + const searchInfo = body.output.searchInfo as { webSearchUsed: boolean; sources: Array<{ url: string }> }; + expect(searchInfo.webSearchUsed).toBe(true); + expect(searchInfo.sources.length).toBeGreaterThan(0); + await expect(page.getByText("Web research used:")).toBeVisible(); + await expectProviderCallsHealthy(["ask"]); +}); + +test("a web page imports into the Library", async ({ signedIn: page, user }) => { + const url = "https://en.wikipedia.org/wiki/Free_public_transport"; + await page.goto("/app/library"); + await page.getByRole("button", { name: "Add Source" }).click(); + await page.getByRole("textbox", { name: "Import from URL" }).fill(url); + await page.getByRole("button", { name: "Import URL" }).click(); + await expect(page.getByText("Source Ready", { exact: true })).toBeVisible({ timeout: 60_000 }); + await page.getByRole("button", { name: "Save imported source" }).click(); + + await expect.poll(async () => { + const { data } = await admin.from("sources").select("origin,content").eq("user_id", await user.userId()); + return data?.map((row) => ({ origin: row.origin, long: row.content.length > 2_000 })); + }).toEqual([{ origin: url, long: true }]); +}); diff --git a/e2e/model-matrix/generate.spec.ts b/e2e/model-matrix/generate.spec.ts new file mode 100644 index 000000000..02ead3188 --- /dev/null +++ b/e2e/model-matrix/generate.spec.ts @@ -0,0 +1,29 @@ +import { MODEL_CATALOG } from "../../supabase/functions/_shared/model-catalog"; +import { expect, fake, stackReady, test } from "../full-stack/stack"; +import { aiResponse, LIVE_TIMEOUT } from "../live/live"; + +// Every model a user can pick must generate a case. The regular live suite pins +// one cheap model, so it cannot see a production model that times out or +// returns invalid JSON. Run with LIVE_MODEL=as-requested (see docs/testing.md). +test.skip(!stackReady, "Live gate is required (LIVE_PROVIDERS=1 LIVE_MODEL=as-requested ./.superset/db-gate.sh)"); +test.describe.configure({ timeout: LIVE_TIMEOUT + 60_000 }); + +for (const model of MODEL_CATALOG) { + test(`${model.name} generates a case`, async ({ signedIn: page }) => { + await page.goto("/app/generate"); + await page.getByRole("combobox", { name: "Select AI model for generate" }).click(); + await page.getByRole("option", { name: new RegExp(`^${model.name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`) }).click(); + await page.getByRole("textbox", { name: "Motion / Topic *" }).fill("This House would make public transport free at the point of use"); + const response = aiResponse(page, "generate"); + await page.getByRole("button", { name: "Generate Debate Case" }).click(); + const body = await response; + expect(body.output.contentions).toHaveLength(3); + + const calls = (await fake.requests()) as Array<{ feature: string | null; model?: string; status?: number; ms?: number }>; + const generate = calls.filter((call) => call.feature === "generate"); + expect(generate.map((call) => call.model), "the case came from the selected model").toContain(model.id); + expect(generate.filter((call) => call.status !== 200), "failed provider calls").toEqual([]); + // The ai function shares a 130 s request budget; print the provider portion. + console.log(`latency ${model.id}: ${generate.map((call) => call.ms).join(", ")} ms`); + }); +} diff --git a/e2e/production/canary.spec.ts b/e2e/production/canary.spec.ts new file mode 100644 index 000000000..0a0e19241 --- /dev/null +++ b/e2e/production/canary.spec.ts @@ -0,0 +1,122 @@ +import { createClient } from "@supabase/supabase-js"; +import { expect, test, type Page } from "@playwright/test"; + +// Production canary. Everything here is read-only except the last two tests, +// which make real AI requests as the dedicated canary account and run only on +// the daily and manual canary. +const supabaseUrl = process.env.PROD_SUPABASE_URL ?? "https://uziwdtkzofptihahfooo.supabase.co"; +const anonKey = process.env.PROD_SUPABASE_ANON_KEY; +const email = process.env.PROD_CANARY_EMAIL; +const password = process.env.PROD_CANARY_PASSWORD; +// workspace-invites joins this list when production enables Workspaces +// (VITE_COLLABORATION_URL); until then the routes are hidden and it is not deployed. +const functions = [ + "admin", "ai", "ai-stream", "announcements", "create-checkout", "create-portal", "demo-code-login", + "export-gdoc", "import-source", "redeem-promo", "share", +]; + +if (process.env.CI && !(anonKey && email && password)) throw new Error("Production canary secrets are incomplete"); +// The canary account allows 10 AI requests a day and main can deploy more often +// than that, so the workflow sets this to false for deployment runs. +const aiRequests = process.env.CANARY_AI_REQUESTS !== "false"; + +async function signInAsCanary(page: Page, { aiRequest }: { aiRequest: boolean }) { + test.skip(aiRequest && !aiRequests, "Real AI requests run only on the daily and manual canary"); + test.skip(!(anonKey && email && password), "Canary credentials are not configured"); + const client = createClient(supabaseUrl, anonKey!, { auth: { persistSession: false } }); + const { data, error } = await client.auth.signInWithPassword({ email: email!, password: password! }); + if (error || !data.session) throw error ?? new Error("Canary sign-in failed"); + await page.addInitScript((session) => { + localStorage.setItem("debate-auth-production", JSON.stringify(session)); + localStorage.setItem("debate-trainer-onboarding", JSON.stringify({ state: { hasCompletedOnboarding: true, isTourRunning: false }, version: 0 })); + }, data.session); +} + +// GPT-5.6 Luna is the Basic default, so this is the path most users take. +async function pickLuna(page: Page, feature: string) { + await page.getByRole("combobox", { name: `Select AI model for ${feature}` }).click(); + await page.getByRole("option", { name: /GPT-5\.6 Luna/ }).first().click(); +} + +test("the production site loads and guards the app", async ({ page, baseURL }) => { + const errors: string[] = []; + page.on("pageerror", (error) => errors.push(error.message)); + await page.goto("/"); + await expect(page).toHaveTitle(/DebateLab/); + await page.goto("/login"); + await expect(page.getByRole("button", { name: "Continue with Google" })).toBeVisible(); + await page.goto("/app/generate"); + await expect(page).toHaveURL(`${baseURL}/login`); + expect(errors).toEqual([]); +}); + +test("every browser-facing Edge Function is deployed and passes the browser's preflight", async ({ request, baseURL }) => { + // The same preflight Chromium sends before supabase-js POSTs with a session. + const requestedHeaders = ["authorization", "apikey", "content-type", "x-client-info"]; + for (const name of functions) { + const response = await request.fetch(`${supabaseUrl}/functions/v1/${name}`, { + method: "OPTIONS", + headers: { + Origin: baseURL!, + "Access-Control-Request-Method": "POST", + "Access-Control-Request-Headers": requestedHeaders.join(", "), + }, + }); + expect(response.status(), `${name} preflight`).toBeLessThan(300); + const headers = response.headers(); + expect(headers["access-control-allow-origin"], `${name} allowed origin`).toBe(baseURL); + expect(headers["access-control-allow-methods"], `${name} allowed methods`).toMatch(/\bPOST\b/); + const allowed = (headers["access-control-allow-headers"] ?? "").toLowerCase().split(/\s*,\s*/); + expect(allowed, `${name} allowed headers`).toEqual(expect.arrayContaining(requestedHeaders)); + } +}); + +test("the AI function rejects unauthenticated calls instead of crashing", async ({ request }) => { + const response = await request.post(`${supabaseUrl}/functions/v1/ai`, { data: { feature: "ask", input: { message: "hi" } } }); + expect(response.status()).toBe(401); +}); + +// Runs on every deployment without spending quota. The Pro badge comes from a +// row-level-security read, so it appears only for a working session. +test("the canary account signs in and loads its account", async ({ page }) => { + await signInAsCanary(page, { aiRequest: false }); + await page.goto("/app/generate"); + await expect(page.getByRole("button", { name: "Generate Debate Case" })).toBeVisible(); + await page.goto("/app/settings"); + await expect(page.getByText("Pro", { exact: true })).toBeVisible(); +}); + +test("the canary account gets a streamed answer from a real model", async ({ page }) => { + await signInAsCanary(page, { aiRequest: true }); + await page.goto("/app/ask"); + await pickLuna(page, "ask"); + await page.getByPlaceholder("Ask your debate coach...").fill("Reply with one short sentence: what is a rebuttal?"); + const stream = page.waitForResponse((response) => response.url().endsWith("/functions/v1/ai-stream"), { timeout: 90_000 }); + await page.getByRole("button", { name: "Send message" }).click(); + const response = await stream; + expect(response.status()).toBe(200); + // ai-stream emits SSE blocks of `event: delta` with `data: {"text": ...}`. + const body = await response.text(); + expect(body).toContain("event: final"); + expect(body).not.toContain("event: error"); + const text = body.split("\n\n") + .filter((block) => block.includes("event: delta")) + .map((block) => JSON.parse(block.split("\n").find((line) => line.startsWith("data:"))!.slice(5)).text as string) + .join(""); + expect(text.trim().length, "streamed answer text").toBeGreaterThan(0); + await expect(page.getByText("Response was interrupted")).toHaveCount(0); +}); + +test("the canary account generates a case", async ({ page }) => { + test.setTimeout(180_000); + await signInAsCanary(page, { aiRequest: true }); + await page.goto("/app/generate"); + await pickLuna(page, "generate"); + await page.getByRole("textbox", { name: "Motion / Topic *" }).fill("This House would make public transport free"); + const generated = page.waitForResponse((response) => response.url().endsWith("/functions/v1/ai") && response.request().method() === "POST", { timeout: 150_000 }); + await page.getByRole("button", { name: "Generate Debate Case" }).click(); + const response = await generated; + expect(response.status(), "ai generate status").toBe(200); + const body = await response.json() as { output: { contentions?: unknown[] } }; + expect(body.output.contentions, "generated contentions").toHaveLength(3); +}); diff --git a/playwright.config.ts b/playwright.config.ts index 8f9e403d2..cfb684497 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -43,17 +43,17 @@ export default defineConfig({ projects: [ { name: 'desktop-chromium', - testIgnore: /(staging|full-stack)\//, + testIgnore: /(staging|full-stack|live|model-matrix|production)\//, use: { ...devices['Desktop Chrome'], viewport: { width: 1280, height: 800 } }, }, { name: 'mobile-chromium', - testIgnore: /(staging|full-stack)\//, + testIgnore: /(staging|full-stack|live|model-matrix|production)\//, use: { ...devices['Pixel 5'] }, }, { name: 'webkit', - testIgnore: /(staging|full-stack)\//, + testIgnore: /(staging|full-stack|live|model-matrix|production)\//, use: { ...devices['Desktop Safari'], viewport: { width: 1280, height: 800 } }, }, ...(stagingEnabled ? [{ diff --git a/playwright.production.config.ts b/playwright.production.config.ts new file mode 100644 index 000000000..85171dc41 --- /dev/null +++ b/playwright.production.config.ts @@ -0,0 +1,20 @@ +import { defineConfig, devices } from "@playwright/test"; + +// Canary against the live production site. No web server, no captures: +// traces and screenshots could retain the canary account's session. +export default defineConfig({ + testDir: "./e2e/production", + fullyParallel: false, + workers: 1, + retries: 1, + reporter: process.env.CI ? [["github"], ["list"]] : "list", + expect: { timeout: 20_000 }, + use: { + baseURL: process.env.PROD_URL ?? "https://debate.pi-stack.org", + locale: "en-US", + trace: "off", + screenshot: "off", + video: "off", + }, + projects: [{ name: "production-chromium", use: { ...devices["Desktop Chrome"], viewport: { width: 1280, height: 800 } } }], +}); diff --git a/playwright.workspace.config.ts b/playwright.workspace.config.ts index c7426d4a0..e633620d1 100644 --- a/playwright.workspace.config.ts +++ b/playwright.workspace.config.ts @@ -4,7 +4,7 @@ const baseURL = process.env.PLAYWRIGHT_BASE_URL ?? "http://127.0.0.1:18081"; export default defineConfig({ testDir: "./e2e", - testMatch: /(workspace-collaboration|full-stack\/.+)\.spec\.ts$/, + testMatch: /(workspace-collaboration|(full-stack|live|model-matrix)\/.+)\.spec\.ts$/, fullyParallel: false, workers: 1, reporter: "list", diff --git a/src/lib/ai/client.ts b/src/lib/ai/client.ts index 56148ec8e..73b82ca4b 100644 --- a/src/lib/ai/client.ts +++ b/src/lib/ai/client.ts @@ -765,7 +765,7 @@ export async function crossfire( return await callSupabaseAI( 'crossfire', - withOutputLanguage(input.data as unknown as Record), + withOutputLanguage({ ...input.data, transcript: serverTranscript(input.data.transcript) }), config, { includeActiveSources: input.data.config.useSourceLibrary, @@ -804,6 +804,12 @@ export async function crossfire( ); } +// Turn signals are client annotations. The Edge Function accepts only the +// turn fields it signs and prompts with, so strip them from every request. +function serverTranscript(turns: CrossfireAIInput['transcript']) { + return turns.map(({ signal: _signal, ...turn }) => turn); +} + export async function crossfireSignal( rawInput: CrossfireSignalInput, config: ProviderConfig, @@ -815,7 +821,7 @@ export async function crossfireSignal( } return await callSupabaseAI( 'crossfire', - withOutputLanguage({ operation: 'signal', ...input.data }), + withOutputLanguage({ operation: 'signal', ...input.data, transcript: serverTranscript(input.data.transcript) }), config, { includeActiveSources: false, diff --git a/src/test/ai/crossfire-client.test.ts b/src/test/ai/crossfire-client.test.ts index 27625e8ea..0208c142f 100644 --- a/src/test/ai/crossfire-client.test.ts +++ b/src/test/ai/crossfire-client.test.ts @@ -119,6 +119,28 @@ describe('Crossfire AI client', () => { expect(secondCall.body.sourceIds).toEqual(['aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa']); }); + it('strips client-only turn signals from every transcript it sends', async () => { + const signal = { + answerTurnId: 'T2', + answerQuality: 'direct', + answerQualityConfidence: 0.9, + concession: 'none', + concessionConfidence: 0.8, + followUpOpportunity: 0.7, + } as const; + const input = debriefInput(); + input.transcript[1] = { ...input.transcript[1], signal }; + mocks.invoke.mockResolvedValue({ data: { output: validDebrief }, error: null }); + + await crossfire(input, config); + await crossfireSignal({ config: input.config, transcript: input.transcript, reply: { type: 'question', text: 'Which count?' }, signalToken: 'token' }, config); + + // The Edge Function rejects any turn key beyond the ones it signs. + for (const [, request] of mocks.invoke.mock.calls) { + for (const turn of request.body.input.transcript) expect(turn).not.toHaveProperty('signal'); + } + }); + it('sends only bounded Crossfire source IDs when stored content is large', async () => { mocks.session.sourceLibrary.splice(0, mocks.session.sourceLibrary.length, ...Array.from({ length: 12 }, (_, index) => ({ diff --git a/supabase/functions/ai-stream/handler_test.ts b/supabase/functions/ai-stream/handler_test.ts index a1168ffb3..bca690c68 100644 --- a/supabase/functions/ai-stream/handler_test.ts +++ b/supabase/functions/ai-stream/handler_test.ts @@ -163,6 +163,13 @@ Deno.test("streaming Ask forwards the shared prompt and returns provider SSE", a if (streamOptions?.include_usage !== true) { throw new Error("streaming handler did not request terminal usage"); } + const routing = providerRequest?.provider; + if (!routing || typeof routing !== "object" || Array.isArray(routing)) throw new Error("provider preferences were missing"); + const preferences = routing as Record; + if (preferences.require_parameters !== true) throw new Error("parameter support was not required"); + if ("sort" in preferences || "order" in preferences) { + throw new Error(`sort/order disables OpenRouter balanced routing: ${JSON.stringify(preferences)}`); + } const system = messages?.find((message) => message.role === "system")?.content ?? ""; assertIncludes(system, "Answer in clear Markdown that can stream smoothly token by token."); assertIncludes(system, "Follow any explicit output-language instruction."); diff --git a/supabase/functions/ai-stream/index.ts b/supabase/functions/ai-stream/index.ts index f885defc5..0c5deee6a 100644 --- a/supabase/functions/ai-stream/index.ts +++ b/supabase/functions/ai-stream/index.ts @@ -237,6 +237,7 @@ export async function handleRequest(req: Request): Promise { messages, max_tokens: Math.min(modelEntry.maxTokens, MAX_REQUEST_OUTPUT_TOKENS), reasoning: { effort: reasoningEffort, exclude: true }, + // Omitting sort and order keeps OpenRouter's balanced provider load balancing. provider: { require_parameters: true }, stream: true, stream_options: { include_usage: true }, diff --git a/supabase/functions/ai/handler-research.test.ts b/supabase/functions/ai/handler-research.test.ts index 0fa81831a..44e9821a7 100644 --- a/supabase/functions/ai/handler-research.test.ts +++ b/supabase/functions/ai/handler-research.test.ts @@ -15,10 +15,36 @@ const environment = { for (const [key, value] of Object.entries(environment)) Deno.env.set(key, value); -const { handleRequest } = await import("./index.ts"); +const { createRequestBudgetSignal, handleRequest } = await import("./index.ts"); const { handleRequest: handleStreamRequest } = await import("../ai-stream/index.ts"); const { MAX_REQUEST_OUTPUT_TOKENS, MODEL_CATALOG } = await import("../_shared/model-catalog.ts"); +Deno.test("AI request work shares one deadline signal", () => { + const caller = new AbortController(); + const deadline = new AbortController(); + const signal = createRequestBudgetSignal(caller.signal, deadline.signal); + const reason = new DOMException("Budget exhausted", "TimeoutError"); + + deadline.abort(reason); + + if (!signal.aborted || signal.reason !== reason) { + throw new Error("shared work did not stop at the request deadline"); + } +}); + +Deno.test("AI request work keeps caller cancellation", () => { + const caller = new AbortController(); + const deadline = new AbortController(); + const signal = createRequestBudgetSignal(caller.signal, deadline.signal); + const reason = new DOMException("Caller disconnected", "AbortError"); + + caller.abort(reason); + + if (!signal.aborted || signal.reason !== reason) { + throw new Error("shared work ignored caller cancellation"); + } +}); + Deno.test("AI handlers reject malformed controls before quota or provider work", async () => { const originalFetch = globalThis.fetch; const calls: string[] = []; @@ -653,6 +679,7 @@ Deno.test("AI handlers use strict OpenRouter routing and server-side reasoning t const provider = body.provider as Record; if (reasoning.effort !== (entry.isProOnly ? "high" : "low")) throw new Error(`tier effort was not enforced for ${entry.id}`); if (reasoning.exclude !== true || provider.require_parameters !== true) throw new Error(`strict reasoning route missing for ${entry.id}`); + if ("sort" in provider || "order" in provider) throw new Error(`sort/order disabled balanced routing for ${entry.id}`); if ((body.response_format as Record)?.type !== "json_object") throw new Error(`JSON response format missing for ${entry.id}`); if (entry.supportsTemperature !== ("temperature" in body)) throw new Error(`temperature support mismatch for ${entry.id}`); } diff --git a/supabase/functions/ai/index.ts b/supabase/functions/ai/index.ts index bb888fb9c..020ab629c 100644 --- a/supabase/functions/ai/index.ts +++ b/supabase/functions/ai/index.ts @@ -737,6 +737,19 @@ async function callAIOnce( ); } +// Supabase answers 504 when a function has not responded within 150 s. Every +// provider call shares one budget below that, so a slow model can use the time +// research left instead of a fixed 60 s. The margin covers the post-generation +// source-usage check (its own 8 s Jev timeout), logging, and saving. +const REQUEST_BUDGET_MS = 130_000; + +export function createRequestBudgetSignal( + requestSignal: AbortSignal, + deadlineSignal = AbortSignal.timeout(REQUEST_BUDGET_MS), +): AbortSignal { + return AbortSignal.any([requestSignal, deadlineSignal]); +} + async function callOpenRouter( messages: Array<{ role: string; content: string }>, model: string, @@ -754,6 +767,7 @@ async function callOpenRouter( max_tokens: Math.min(modelEntry.maxTokens, MAX_REQUEST_OUTPUT_TOKENS), response_format: { type: "json_object" }, reasoning: { effort: reasoningEffort, exclude: true }, + // Omitting sort and order keeps OpenRouter's balanced provider load balancing. provider: { require_parameters: true }, }; if (modelEntry.supportsTemperature) payload.temperature = temperature; @@ -762,7 +776,7 @@ async function callOpenRouter( method: "POST", headers: { "Content-Type": "application/json", "Authorization": `Bearer ${apiKey}` }, body: JSON.stringify(payload), - }, 60000, signal); + }, REQUEST_BUDGET_MS, signal); if (!response.ok) throw new Error(`OpenRouter error: ${errorBody ?? ""}`); if (!data?.choices || !Array.isArray(data.choices) || data.choices.length === 0) throw new Error("OpenRouter returned no choices"); const usage = normalizeProviderUsage(data.usage) ?? missingOpenRouterUsage("response"); @@ -839,6 +853,7 @@ export async function handleRequest(req: Request): Promise { if (req.method === "OPTIONS") { return new Response("ok", { headers: corsHeaders }); } + const workSignal = createRequestBudgetSignal(req.signal); if (req.method !== "POST") { return new Response("Method Not Allowed", { status: 405, headers: corsHeaders }); } @@ -1054,7 +1069,7 @@ export async function handleRequest(req: Request): Promise { const evaluated = await evaluateCrossfireExchange({ transcript: crossfireInput.transcript, reply: crossfireInput.reply, - signal: req.signal, + signal: workSignal, }); const signalUsage = evaluated.jev?.usage ?? evaluated.jevUsage; if (signalUsage) await logUsage(userId, "jev_crossfire_signal", provider, JEV_MODEL, signalUsage, undefined, evaluated.jevUsage ? "error" : "success", evaluated.jevUsage ? "jev_incomplete" : undefined); @@ -1235,7 +1250,7 @@ export async function handleRequest(req: Request): Promise { canAccessProModels: Boolean(rateLimit.canAccessProModels), reasoningEffort: requestedReasoningEffort, sourceCount: Array.isArray(rawSources) ? rawSources.length : 0, - signal: req.signal, + signal: workSignal, }); model = resolution.resolvedModel; modelEntry = resolution.modelEntry; @@ -1285,7 +1300,7 @@ export async function handleRequest(req: Request): Promise { feature, requestText: JSON.stringify(sanitizedInput), sources: sourceDecisionCandidates, - signal: req.signal, + signal: workSignal, }); sourceAssessments = ranked.assessments; sources = ranked.sources.map(({ title, content, origin, type }) => ({ title, content, ...(origin ? { origin } : {}), type })); @@ -1314,7 +1329,7 @@ export async function handleRequest(req: Request): Promise { provider, model, reasoningEffort, - signal: req.signal, + signal: workSignal, onTimeout: () => { researchAccountingIncomplete = true; }, @@ -1376,7 +1391,7 @@ export async function handleRequest(req: Request): Promise { const systemPrompt = await buildSystemPromptFor(feature); const userPrompt = buildUserPrompt(feature, sparringRequest as unknown as Record, ""); const messages = [{ role: "user", content: userPrompt }]; - const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, temperature, reasoningEffort, req.signal)); + const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, temperature, reasoningEffort, workSignal)); const output = cleanGeneratedText(aiResult.output); const normalizedOutput = normalizeSparringOutput(output, sparringRequest.action); const parsedOutput = parseSparringProviderOutput(normalizedOutput, sparringRequest.action); @@ -1445,7 +1460,7 @@ export async function handleRequest(req: Request): Promise { [{ role: "user", content: userPrompt }], temperature, reasoningEffort, - req.signal, + workSignal, )); const cleanedOutput = cleanGeneratedText(aiResult.output); const output = crossfireInput.operation === "next-turn" @@ -1487,7 +1502,7 @@ export async function handleRequest(req: Request): Promise { const userPrompt = buildUserPrompt(feature, sanitizedInput, searchContext); const messages = [{ role: "user", content: userPrompt }]; - const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, temperature, reasoningEffort, req.signal)); + const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, temperature, reasoningEffort, workSignal)); const output = cleanGeneratedText(aiResult.output); // Post-process: quality floor. Only citations whose URL matches fetched, @@ -1545,7 +1560,7 @@ export async function handleRequest(req: Request): Promise { let urlContext = ""; if (urls.size > 0) { const contents = await Promise.all(Array.from(urls).slice(0, 3).map(async (u) => { - const content = await exaGetContents(u, req.signal); + const content = await exaGetContents(u, workSignal); return content.trim() ? `--- Content from ${u} ---\n${content.slice(0, 8000)}\n--- End of ${u} ---` : ""; @@ -1600,7 +1615,7 @@ export async function handleRequest(req: Request): Promise { const systemPrompt = await buildSystemPromptFor(feature, effectiveResearch); const prompt = `${basePrompt}${urlContext}${webSearchContext}`; const messages = [...history, { role: "user", content: prompt }]; - const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, temperature, reasoningEffort, req.signal)); + const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, temperature, reasoningEffort, workSignal)); const output = cleanGeneratedText(aiResult.output); if (output && typeof output === "object") { @@ -1688,7 +1703,7 @@ export async function handleRequest(req: Request): Promise { feature, requestText: JSON.stringify(sanitizedInput), sources: discoveredCandidates, - signal: req.signal, + signal: workSignal, }); const discoveredRankingUsage = discoveredRanking.jev?.usage ?? discoveredRanking.jevUsage; if (discoveredRankingUsage) await logUsage(userId, "jev_source_ranking", provider, JEV_MODEL, discoveredRankingUsage, undefined, discoveredRanking.jevUsage ? "error" : "success", discoveredRanking.jevUsage ? "jev_incomplete" : undefined); @@ -1717,7 +1732,7 @@ export async function handleRequest(req: Request): Promise { const systemPrompt = await buildSystemPromptFor(feature, research); const userPrompt = buildUserPrompt(feature, sanitizedInput, searchContext); const messages = [{ role: "user", content: userPrompt }]; - const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, temperature, reasoningEffort, req.signal)); + const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, temperature, reasoningEffort, workSignal)); const output = cleanGeneratedText(aiResult.output); // Preserve model evidence warnings when research exists. If research @@ -1762,6 +1777,8 @@ export async function handleRequest(req: Request): Promise { strength: discoveredAssessmentByKey.get(source.key)?.strength ?? 0, })), ].slice(0, 10); + // The case is already complete, so this optional check must not inherit + // an almost spent request budget and turn a finished case into a 500. const caseUsage = await identifyCaseSourceUsage({ caseText: JSON.stringify(output), sources: usageCandidates, @@ -1812,7 +1829,7 @@ export async function handleRequest(req: Request): Promise { const systemPrompt = await buildSystemPromptFor(feature, research); const userPrompt = buildUserPrompt(feature, sanitizedInput, searchContext); const messages = [{ role: "user", content: userPrompt }]; - const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, temperature, reasoningEffort, req.signal)); + const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, temperature, reasoningEffort, workSignal)); const output = cleanGeneratedText(aiResult.output); enforceBrainstormControls(output, side, outputMode); @@ -1840,7 +1857,7 @@ export async function handleRequest(req: Request): Promise { const systemPrompt = await buildSystemPromptFor(feature, research); const userPrompt = buildUserPrompt(feature, sanitizedInput, searchContext); const messages = [{ role: "user", content: userPrompt }]; - const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, temperature, reasoningEffort, req.signal)); + const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, temperature, reasoningEffort, workSignal)); const output = cleanGeneratedText(aiResult.output); enforceAttackControls(output, detectFallacies, analyzeWeighing); @@ -1856,7 +1873,7 @@ export async function handleRequest(req: Request): Promise { const systemPrompt = await buildSystemPromptFor(feature); const userPrompt = buildUserPrompt(feature, sanitizedInput, ""); const messages = [{ role: "user", content: userPrompt }]; - const fallbackResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, temperature, reasoningEffort, req.signal)); + const fallbackResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, temperature, reasoningEffort, workSignal)); const output = cleanGeneratedText(fallbackResult.output); const latencyMs = Date.now() - startTime; @@ -1890,7 +1907,7 @@ export async function handleRequest(req: Request): Promise { : undefined; // A successful main call takes precedence over usage attached to a later // post-processing error. This avoids counting its repair attempts twice. - const mainErrorUsage = completedMainUsage ?? selectObservedUsage(usageError, req.signal?.aborted === true); + const mainErrorUsage = completedMainUsage ?? selectObservedUsage(usageError, workSignal?.aborted === true); await logUsage( userId, feature,