test(e2e): live provider journeys, production canary, and required checks

.github/workflows/deploy.yml

@@ -93,6 +93,8 @@ jobs:
 
       - name: Database and real workspace journey
         env:
+          # ghcr.io rate-limits shared runners; the same images are on ECR.
+          SUPABASE_INTERNAL_IMAGE_REGISTRY: public.ecr.aws
           WORKSPACE_GATE_ARTIFACT_ROOT: ${{ runner.temp }}/workspace-gate
         run: ./.superset/db-gate.sh
 

.github/workflows/live-e2e.yml

@@ -0,0 +1,68 @@
+name: Live provider E2E
+
+# Real OpenRouter, Jev, and Exa behind the local stack. Runs on pull requests,
+# nightly to catch provider drift without code changes, and on demand.
+on:
+  pull_request:
+  schedule:
+    - cron: "17 3 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: live-e2e-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  live-e2e:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Set up Node
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22
+
+      - name: Set up Supabase CLI
+        uses: supabase/setup-cli@46f7f98c7f948ad727d22c1e67fab04c223a0520 # v3.0.0
+        with:
+          version: 2.116.0
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Install Chromium
+        run: npx playwright install --with-deps chromium
+
+      # db-gate.sh fails before starting anything when either key is missing.
+      # Accepted risk (2026-09-24): pull-request code runs with these keys, so
+      # anyone who can push a branch here could read them. Keep them low-limit.
+      - name: Live provider journeys
+        env:
+          LIVE_PROVIDERS: 1
+          # Repository variable override; empty falls back to db-gate.sh's default.
+          LIVE_MODEL: ${{ vars.LIVE_MODEL }}
+          LIVE_OPENROUTER_API_KEY: ${{ secrets.LIVE_OPENROUTER_API_KEY }}
+          LIVE_EXA_API_KEY: ${{ secrets.LIVE_EXA_API_KEY }}
+          SUPABASE_INTERNAL_IMAGE_REGISTRY: public.ecr.aws
+          WORKSPACE_GATE_ARTIFACT_ROOT: ${{ runner.temp }}/workspace-gate
+        run: ./.superset/db-gate.sh
+
+      - name: Upload live failure artifacts
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: live-e2e-failure-${{ github.run_id }}
+          path: |
+            ${{ runner.temp }}/workspace-gate/
+            test-results/
+            playwright-report/
+          if-no-files-found: ignore
+          retention-days: 7

.github/workflows/model-matrix.yml

@@ -0,0 +1,75 @@
+name: Model matrix
+
+# Generates one case with every model in the catalog, unpinned, so a model that
+# times out or returns invalid JSON fails here before users hit it. One run
+# costs about $1, so it runs weekly and when model routing, prompts, or the ai
+# function change.
+on:
+  pull_request:
+    paths:
+      - supabase/functions/_shared/**
+      - supabase/functions/ai/**
+      - src/lib/ai/**
+      - e2e/model-matrix/**
+      - .github/workflows/model-matrix.yml
+  schedule:
+    - cron: "29 4 * * 1"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: model-matrix-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  model-matrix:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Set up Node
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22
+
+      - name: Set up Supabase CLI
+        uses: supabase/setup-cli@46f7f98c7f948ad727d22c1e67fab04c223a0520 # v3.0.0
+        with:
+          version: 2.116.0
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Install Chromium
+        run: npx playwright install --with-deps chromium
+
+      # db-gate.sh fails before starting anything when either key is missing.
+      # Accepted risk (2026-09-24): pull-request code runs with these keys, so
+      # anyone who can push a branch here could read them. Keep them low-limit.
+      - name: Case generation with every catalog model
+        env:
+          LIVE_PROVIDERS: 1
+          LIVE_MODEL: as-requested
+          LIVE_OPENROUTER_API_KEY: ${{ secrets.LIVE_OPENROUTER_API_KEY }}
+          LIVE_EXA_API_KEY: ${{ secrets.LIVE_EXA_API_KEY }}
+          SUPABASE_INTERNAL_IMAGE_REGISTRY: public.ecr.aws
+          WORKSPACE_GATE_ARTIFACT_ROOT: ${{ runner.temp }}/workspace-gate
+        run: ./.superset/db-gate.sh
+
+      - name: Upload live failure artifacts
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: model-matrix-failure-${{ github.run_id }}
+          path: |
+            ${{ runner.temp }}/workspace-gate/
+            test-results/
+            playwright-report/
+          if-no-files-found: ignore
+          retention-days: 7

.github/workflows/prod-canary.yml

@@ -0,0 +1,49 @@
+name: Production canary
+
+# Checks the live site after every Vercel production deployment and daily.
+# Read-only on deployments. The daily and manual runs also make two real AI
+# requests (a streamed Ask and a case) as the dedicated canary account.
+on:
+  deployment_status:
+  schedule:
+    - cron: "41 6 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: production-canary
+  cancel-in-progress: false
+
+jobs:
+  canary:
+    if: >-
+      github.event_name != 'deployment_status' ||
+      (github.event.deployment_status.state == 'success' && github.event.deployment.environment == 'Production')
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Set up Node
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Install Chromium
+        run: npx playwright install --with-deps chromium
+
+      - name: Production canary
+        env:
+          PROD_SUPABASE_ANON_KEY: ${{ vars.PROD_SUPABASE_ANON_KEY }}
+          PROD_CANARY_EMAIL: ${{ secrets.PROD_CANARY_EMAIL }}
+          PROD_CANARY_PASSWORD: ${{ secrets.PROD_CANARY_PASSWORD }}
+          CANARY_AI_REQUESTS: ${{ github.event_name != 'deployment_status' }}
+        run: npx playwright test --config=playwright.production.config.ts

.superset/db-gate.sh

@@ -14,6 +14,17 @@ source "$DIR/lib/lock.sh"
 
 DB_LOCK="$HOME/.superset/debatelab-db.lock"
 FAKE_PROVIDER=debatelab-fake-openrouter
+# LIVE_PROVIDERS=1 runs e2e/live against real OpenRouter, Jev, and Exa instead of
+# the fake provider. It needs LIVE_OPENROUTER_API_KEY and LIVE_EXA_API_KEY.
+# LIVE_MODEL=as-requested runs e2e/model-matrix instead: one case per catalog model.
+LIVE_PROVIDERS="${LIVE_PROVIDERS:-0}"
+# GLM Flash is inexpensive and has broad endpoint coverage, so OpenRouter's
+# default balanced routing can fail over between providers during this gate.
+LIVE_MODEL="${LIVE_MODEL:-z-ai/glm-5.3-flash}"  # empty also means default
+if [ "$LIVE_PROVIDERS" = 1 ] && { [ -z "${LIVE_OPENROUTER_API_KEY:-}" ] || [ -z "${LIVE_EXA_API_KEY:-}" ]; }; then
+  echo "LIVE_PROVIDERS=1 requires LIVE_OPENROUTER_API_KEY and LIVE_EXA_API_KEY" >&2
+  exit 1
+fi
 stack_started=false
 runtime_dir=""
 owned_pids=()
@@ -67,10 +78,11 @@ for attempt in 1 2 3; do
   if supabase start >>"$runtime_dir/supabase.log" 2>&1; then break; fi
   if [ "$attempt" -eq 3 ]; then exit 1; fi
   echo "supabase start failed (attempt $attempt); retrying after $((attempt * 30))s"
+  tail -n 3 "$runtime_dir/supabase.log"
   supabase stop --no-backup >/dev/null 2>&1 || true
   sleep $((attempt * 30))
 done
-npm run test:db
+if [ "$LIVE_PROVIDERS" != 1 ]; then npm run test:db; fi
 
 status_json="$(supabase status -o json 2>/dev/null)"
 status_value() {
@@ -90,11 +102,20 @@ fi
 LOCAL_SERVICE_ROLE_KEY="$service_role_key" \
 LOCAL_MAILER_HOST="$mailer_host" \
   node scripts/workspace-test-env.mjs .superset/workspace-test.env "$runtime_dir/function.env"
+provider_mode=()
+if [ "$LIVE_PROVIDERS" = 1 ]; then
+  # printf is a builtin, so the keys never appear in a process list or log.
+  grep -v '^OPENROUTER_API_KEY=' "$runtime_dir/function.env" >"$runtime_dir/function.env.next"
+  printf 'OPENROUTER_API_KEY=%s\nEXA_API_KEY=%s\n' "$LIVE_OPENROUTER_API_KEY" "$LIVE_EXA_API_KEY" >>"$runtime_dir/function.env.next"
+  mv -- "$runtime_dir/function.env.next" "$runtime_dir/function.env"
+  chmod 600 "$runtime_dir/function.env"
+  provider_mode=(-e UPSTREAM=https://openrouter.ai/api -e "LIVE_MODEL=$LIVE_MODEL")
+fi
 
 # Edge Functions reach the fake provider by container name on the stack network.
 docker rm -f "$FAKE_PROVIDER" >/dev/null 2>&1 || true
 docker run -d --rm --name "$FAKE_PROVIDER" --network "supabase_network_$project_id" \
-  -p 127.0.0.1:18090:8787 -v "$(pwd)/e2e/full-stack/fake-openrouter.mjs:/fake.mjs:ro" \
+  "${provider_mode[@]}" -p 127.0.0.1:18090:8787 -v "$(pwd)/e2e/full-stack/fake-openrouter.mjs:/fake.mjs:ro" \
   node:22-alpine node /fake.mjs >/dev/null
 
 setsid supabase functions serve --no-verify-jwt \
@@ -125,6 +146,13 @@ for function_name in workspace-invites ai ai-stream demo-code-login admin announ
   curl --fail --silent --output /dev/null --request OPTIONS --header "Origin: $app_origin" "$function_url"
 done
 
+if [ "$LIVE_PROVIDERS" = 1 ] && [ "$LIVE_MODEL" = as-requested ]; then
+  test_selection=(e2e/model-matrix --project=desktop-chromium)
+elif [ "$LIVE_PROVIDERS" = 1 ]; then
+  test_selection=(e2e/live --project=desktop-chromium)
+else
+  test_selection=(e2e/workspace-collaboration.spec.ts e2e/full-stack)
+fi
 LOCAL_SUPABASE_URL="$api_url" \
 LOCAL_SUPABASE_ANON_KEY="$anon_key" \
 LOCAL_SUPABASE_SERVICE_ROLE_KEY="$service_role_key" \
@@ -135,6 +163,7 @@ COLLABORATION_ALLOWED_ORIGINS="$app_origin" \
 SUPABASE_URL="$api_url" \
 SUPABASE_ANON_KEY="$anon_key" \
 FAKE_OPENROUTER_URL=http://127.0.0.1:18090 \
+LIVE_MODEL="$LIVE_MODEL" \
 DEMO_CODE_PEPPER="$(sed -n 's/^DEMO_CODE_PEPPER=//p' .superset/workspace-test.env)" \
 PLAYWRIGHT_BASE_URL="$app_origin" \
-  npx playwright test --config=playwright.workspace.config.ts
+  npx playwright test --config=playwright.workspace.config.ts "${test_selection[@]}"

.superset/workspace-test.env

@@ -9,3 +9,5 @@ OPENROUTER_API_KEY=local-fake-openrouter-key
 OPENROUTER_API_URL=http://debatelab-fake-openrouter:8787/api
 DEMO_CODE_PEPPER=local-demo-code-pepper-for-browser-gate
 SPARRING_INTEGRITY_SECRET=local-sparring-integrity-secret-for-browser-gate
+STRIPE_SECRET_KEY=sk_test_local-full-stack-unused
+STRIPE_WEBHOOK_SECRET=whsec_local-full-stack-test

AGENTS.md

@@ -71,7 +71,11 @@ CI=1 PORT=18080 PLAYWRIGHT_BASE_URL=http://127.0.0.1:18080 npm run test:e2e:webk
 
 - Start bug fixes with a failing test or repeatable browser reproduction when a suitable test layer exists.
 - Stub sessions prove rendering only. Do not call them authenticated backend proof.
-- Never put staging or production credentials in jobs, logs, traces, screenshots, commits, comments, or docs.
+- Never put staging or production credentials in jobs, logs, traces, screenshots, commits, comments, or docs. Two scoped exceptions reach jobs only through GitHub Actions secrets:
+  - the protected staging values;
+  - the dedicated production canary account (`PROD_CANARY_EMAIL` and `PROD_CANARY_PASSWORD`, a regular user capped at 10 AI requests a day), which Adam approved on 2026-09-24.
+
+  Neither exception may appear in logs, traces, or screenshots. No other production credential may enter a job.
 - Never substitute the production project when local or staging access is unavailable.
 - Production deployments, data changes, migrations, Edge Function deployments, and secret changes require explicit authorization.
 - A local pass proves only the local commit. Check the current-head GitHub run before calling CI green.

docs/operations.md

@@ -25,6 +25,8 @@ Pull requests also start isolated local Supabase for pgTAP and run desktop plus
 
 Non-PR runs upload `dist/` briefly and may build a `linux/amd64` and `linux/arm64` Docker image when registry credentials are present. A successful quality run does not by itself prove that an image was pushed.
 
+[`.github/workflows/live-e2e.yml`](../.github/workflows/live-e2e.yml) runs the local stack against real providers on pull requests and nightly. [`.github/workflows/model-matrix.yml`](../.github/workflows/model-matrix.yml) generates a case with every catalog model weekly and when AI code changes. [`.github/workflows/prod-canary.yml`](../.github/workflows/prod-canary.yml) checks production after each Vercel production deployment and daily. See [testing.md](testing.md) for all three. Both Supabase-starting jobs pull images from `public.ecr.aws`, because ghcr.io rate-limits shared runners.
+
 [`.github/workflows/staging-e2e.yml`](../.github/workflows/staging-e2e.yml) owns protected staging. It accepts eligible successful main-branch quality runs or a manual dispatch from `main`. It runs only when `STAGING_E2E_ENABLED=true` and all protected values exist.
 
 ## Production image
@@ -111,6 +113,8 @@ Secret names may be documented; values may not. Server-side categories include:
 
 Never place server secrets in `VITE_*`, screenshots, traces, logs, comments, commits, or documentation.
 
+GitHub Actions secrets: `LIVE_OPENROUTER_API_KEY` and `LIVE_EXA_API_KEY` for live provider tests, and `PROD_CANARY_EMAIL` and `PROD_CANARY_PASSWORD` for the production canary account. The public anon key is the `PROD_SUPABASE_ANON_KEY` repository variable. To rotate the canary password, set it with the Supabase Auth admin API and pipe it into `gh secret set` over stdin.
+
 ## Rollback
 
 Choose rollback before changing production: