From db19c0af9147654d684087daebc3b771f2dca4dd Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 29 Oct 2026 16:20:07 +0000 Subject: [PATCH 1/2] Cut the pull-request check to about 3 minutes and move slow suites to schedules Quality now runs five parallel jobs with cached node_modules. Unit tests use Vitest's vmThreads pool (95s to 43s on 2 CPUs), and the @smoke set keeps boot, sign-in, every page render and the first-run tour; tool journeys stay nightly. The full-stack database gate, live providers, the agentic suite and the model matrix no longer run on pull requests. Schema PRs run pgTAP on Postgres alone. Staging E2E, the production canary and the preview note stop starting runs that only skip, and docs.yml folds into Quality. npm run verify runs in two lanes (231s to 98s locally). Prerender gets its own Vite cache so a build no longer breaks a running dev server's lazy routes. Co-Authored-By: Claude Opus 5.5 --- .github/actions/node-deps/action.yml | 24 +++++ .github/workflows/agentic-e2e.yml | 25 ++--- .github/workflows/browser-nightly.yml | 21 ++--- .github/workflows/database.yml | 80 ++++++---------- .github/workflows/deploy.yml | 86 +++++++++++------- .github/workflows/docs.yml | 29 ------ .github/workflows/live-e2e.yml | 20 ++-- .github/workflows/model-matrix.yml | 18 ++-- .github/workflows/prod-canary.yml | 73 ++++++++++----- .github/workflows/staging-e2e.yml | 3 + .github/workflows/staging-preview-note.yml | 9 +- .superset/db-gate.sh | 10 +- AGENTS.md | 10 +- README.md | 3 +- docs/operations.md | 36 +++++--- docs/standards/security.md | 2 +- docs/testing.md | 40 ++++---- e2e/admin-announcements.spec.ts | 2 +- e2e/crossfire.spec.ts | 2 +- e2e/first-run.spec.ts | 2 +- e2e/free-tools.spec.ts | 2 +- e2e/quality-backlog.spec.ts | 4 +- e2e/sparring.spec.ts | 2 +- package.json | 4 +- scripts/prerender.mjs | 5 + scripts/verify.mjs | 101 +++++++++++++++++++++ src/test/setup.ts | 11 +++ vitest.config.ts | 4 + 28 files changed, 381 insertions(+), 247 deletions(-) create mode 100644 .github/actions/node-deps/action.yml delete mode 100644 .github/workflows/docs.yml create mode 100644 scripts/verify.mjs diff --git a/.github/actions/node-deps/action.yml b/.github/actions/node-deps/action.yml new file mode 100644 index 000000000..d2d5d45a8 --- /dev/null +++ b/.github/actions/node-deps/action.yml @@ -0,0 +1,24 @@ +name: Node and dependencies +description: >- + Node 22 with node_modules restored from the cache for this lockfile. npm ci + runs only on a cache miss, so most jobs skip its 40 seconds. + +runs: + using: composite + steps: + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + + - name: Restore node_modules + id: cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: node_modules + key: node-modules-${{ runner.os }}-node22-${{ hashFiles('package-lock.json') }} + + - name: Install dependencies + if: steps.cache.outputs.cache-hit != 'true' + shell: bash + run: npm ci diff --git a/.github/workflows/agentic-e2e.yml b/.github/workflows/agentic-e2e.yml index b8d68505e..fa8638031 100644 --- a/.github/workflows/agentic-e2e.yml +++ b/.github/workflows/agentic-e2e.yml @@ -3,26 +3,13 @@ name: Agentic E2E # The e2e suite in agentic/ against the local stack in fake mode: an AI agent # drives goal-level flows and judges screens. Recorded flows replay from # agentic/replay without model calls; judgments always call the model. -# It runs nightly and on pull requests that touch the app or the suite. A -# weekly bug bash sends each persona exploring, and on demand one exploration -# runs instead of the suite. +# It runs nightly, not on pull requests: it takes far longer than the +# pull-request check and each judgment is a paid model call. Run it on a +# branch with `gh workflow run agentic-e2e.yml --ref `. A weekly bug +# bash sends each persona exploring, and on demand one exploration runs +# instead of the suite. # Not a required check. See docs/testing.md#agentic-browser-tests. on: - pull_request: - paths: - - "src/**" - - "packages/**" - - "public/**" - - "index.html" - - "supabase/functions/**" - - "supabase/migrations/**" - - "package-lock.json" - - "agentic/**" - - "e2e.config.ts" - - "e2e/full-stack/stack-core.ts" - - "e2e/full-stack/fake-openrouter.mjs" - - ".claude/skills/verify-debatelab/bin/verify.sh" - - ".github/workflows/agentic-e2e.yml" schedule: - cron: "47 3 * * *" # the suite - cron: "13 5 * * 0" # the weekly bug bash @@ -89,7 +76,7 @@ jobs: run: npx playwright install --with-deps chromium # Without the key every agent test skips, which would read as a pass. - # Same accepted risk as live-e2e.yml: pull-request code runs with it. + # Same accepted risk as live-e2e.yml: a dispatch can run branch code with it. - name: Require the testing agent's model key env: LIVE_OPENROUTER_API_KEY: ${{ secrets.LIVE_OPENROUTER_API_KEY }} diff --git a/.github/workflows/browser-nightly.yml b/.github/workflows/browser-nightly.yml index a189ae59e..6978bad10 100644 --- a/.github/workflows/browser-nightly.yml +++ b/.github/workflows/browser-nightly.yml @@ -1,8 +1,10 @@ name: Browser nightly -# The full credential-free browser suite on desktop and mobile Chromium. Pull -# requests run only the @smoke subset (deploy.yml), so this catches what the -# subset skips, at most a day late. +# The full credential-free browser suite on desktop and mobile Chromium, and +# the service worker's offline test against a production build. Pull requests +# run only the @smoke subset (deploy.yml), so this catches what the subset +# skips, at most a day late. Run it on a branch with +# `gh workflow run browser-nightly.yml --ref `. on: schedule: - cron: "47 2 * * *" @@ -25,14 +27,7 @@ jobs: with: persist-credentials: false - - name: Set up Node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: 22 - cache: npm - - - name: Install dependencies - run: npm ci + - uses: ./.github/actions/node-deps - name: Cache Playwright browsers uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -50,6 +45,10 @@ jobs: PLAYWRIGHT_BASE_URL: http://127.0.0.1:18080 run: npm run test:e2e + - name: Service worker offline test + if: ${{ !cancelled() }} + run: npm run build && npm run test:sw + - name: Upload browser failure artifacts if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml index efb81124a..1c4e164b8 100644 --- a/.github/workflows/database.yml +++ b/.github/workflows/database.yml @@ -1,51 +1,22 @@ name: Database -# Local Supabase, pgTAP, and the full-stack browser journeys. About 20 minutes, -# so pull requests run it only when they touch the backend, the stores and -# clients that talk to it, or the gate itself. Pushes to main use the same -# filter, and the weekly run catches frontend-only regressions. +# Pull requests that change the schema, its tests, or the gate run pgTAP and +# the database scripts against Postgres alone, about 2 minutes. The full gate +# (local Supabase, pgTAP, and the full-stack browser journeys on desktop and +# mobile, about 20 minutes) runs nightly and on demand. on: pull_request: paths: - - "supabase/**" - - "services/collaboration/**" - - "src/lib/supabase/**" - - "src/lib/stores/**" - - "src/lib/ai/**" - - "packages/**" - - "e2e/full-stack/**" - - "e2e/workspace-collaboration.spec.ts" - - "playwright.workspace.config.ts" - - ".superset/db-gate.sh" - - ".superset/lib/**" - - ".superset/workspace-test.env" + - "supabase/migrations/**" + - "supabase/tests/**" + - "supabase/seed.sql" + - "supabase/config.toml" - "scripts/test-*.sh" - - "scripts/workspace-test-env.mjs" - - "package.json" - - "package-lock.json" - - ".github/workflows/database.yml" - push: - branches: [ "main" ] - paths: - - "supabase/**" - - "services/collaboration/**" - - "src/lib/supabase/**" - - "src/lib/stores/**" - - "src/lib/ai/**" - - "packages/**" - - "e2e/full-stack/**" - - "e2e/workspace-collaboration.spec.ts" - - "playwright.workspace.config.ts" - ".superset/db-gate.sh" - ".superset/lib/**" - - ".superset/workspace-test.env" - - "scripts/test-*.sh" - - "scripts/workspace-test-env.mjs" - - "package.json" - - "package-lock.json" - ".github/workflows/database.yml" schedule: - - cron: "23 5 * * 0" + - cron: "23 5 * * *" workflow_dispatch: permissions: @@ -55,27 +26,38 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + # ghcr.io rate-limits shared runners; the same images are on ECR. + SUPABASE_INTERNAL_IMAGE_REGISTRY: public.ecr.aws + jobs: - database-tests: + pgtap: + if: github.event_name == 'pull_request' runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: Set up Node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: 22 + # The gate runs the repo-pinned Supabase CLI from node_modules. + - uses: ./.github/actions/node-deps - - name: Set up Supabase CLI - uses: supabase/setup-cli@46f7f98c7f948ad727d22c1e67fab04c223a0520 # v3.0.0 + - name: pgTAP and database scripts + run: ./.superset/db-gate.sh --db-only + + full-gate: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - version: 2.121.0 + persist-credentials: false - - name: Install dependencies - run: npm ci + - uses: ./.github/actions/node-deps - name: Cache Playwright browsers uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -88,8 +70,6 @@ jobs: - name: Database and real workspace journey env: - # ghcr.io rate-limits shared runners; the same images are on ECR. - SUPABASE_INTERNAL_IMAGE_REGISTRY: public.ecr.aws WORKSPACE_GATE_ARTIFACT_ROOT: ${{ runner.temp }}/workspace-gate run: ./.superset/db-gate.sh diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 851863857..ee440d3b8 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,14 +1,12 @@ name: Quality +# The pull-request check. Five jobs run side by side so the check takes as long +# as the slowest one, about 2.5 minutes. Slow, paid, and full-stack suites run +# on a schedule instead; docs/operations.md lists them. on: push: branches: [ "main" ] pull_request: - # Docs-only pull requests run docs.yml instead. - paths-ignore: - - "docs/**" - - "plans/**" - - "**/*.md" workflow_dispatch: permissions: @@ -19,28 +17,16 @@ concurrency: cancel-in-progress: true jobs: - quality: + lint: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: Set up Node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: 22 - cache: npm - - - name: Install dependencies - run: npm ci - - - name: Set up Deno - uses: denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed # v2.0.5 - with: - deno-version: 2.9.6 - cache: true + - uses: ./.github/actions/node-deps - name: Documentation run: npm run docs:check @@ -48,40 +34,74 @@ jobs: - name: Lint run: npm run lint + typecheck-build: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/node-deps + - name: Typecheck run: npm run typecheck + - name: Build + run: npm run build + + unit: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/node-deps + - name: Test run: npm test - name: Collaboration service tests run: npm run test:collaboration + # Edge Function tests import only jsr:, npm: and https: modules, so this job + # needs Deno and no npm install. + edge: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Deno + uses: denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed # v2.0.5 + with: + deno-version: 2.9.6 + cache: true + - name: Edge function tests run: npm run test:edge - - name: Build - run: npm run build - - # Runs beside quality so the pull-request check takes as long as the slower - # of the two. The full desktop and mobile suite runs in browser-nightly.yml. + # The @smoke set: the app boots, sign-in works, every public and signed-in + # page renders without errors, and the first-run tour starts a debate. The + # full desktop and mobile suite runs in browser-nightly.yml. browser-smoke: if: github.event_name == 'pull_request' runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: Set up Node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: 22 - cache: npm - - - name: Install dependencies - run: npm ci + - uses: ./.github/actions/node-deps - name: Cache Playwright browsers uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 6808f91cd..000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Docs - -# Docs-only pull requests skip the Quality workflow, so they get this check. -on: - pull_request: - paths: - - "docs/**" - - "plans/**" - - "**/*.md" - -permissions: - contents: read - -jobs: - docs: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Set up Node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: 22 - - - name: Documentation - run: npm run docs:check diff --git a/.github/workflows/live-e2e.yml b/.github/workflows/live-e2e.yml index e559a44b3..1046ef56b 100644 --- a/.github/workflows/live-e2e.yml +++ b/.github/workflows/live-e2e.yml @@ -1,18 +1,11 @@ name: Live provider E2E # Real OpenRouter, Jev, and Exa behind the local stack. Every run spends -# provider credit, so pull requests run it only when AI or research code -# changes. Nightly runs catch provider drift without code changes. +# provider credit and takes about 17 minutes, so it runs nightly and on demand, +# never on pull requests. To check a branch that changes AI or research code: +# `gh workflow run live-e2e.yml --ref `. Nightly runs also catch +# provider drift without code changes. on: - pull_request: - paths: - - "supabase/functions/**" - - "src/lib/ai/**" - - "packages/**" - - "e2e/live/**" - - "e2e/full-stack/fake-openrouter.mjs" - - ".superset/db-gate.sh" - - ".github/workflows/live-e2e.yml" schedule: - cron: "17 3 * * *" workflow_dispatch: @@ -51,8 +44,9 @@ jobs: run: npx playwright install --with-deps chromium # db-gate.sh fails before starting anything when either key is missing. - # Accepted risk (2026-09-24): pull-request code runs with these keys, so - # anyone who can push a branch here could read them. Keep them low-limit. + # Accepted risk (2026-09-24): a dispatch can run branch code with these + # keys, so anyone who can push a branch here could read them. Keep them + # low-limit. - name: Live provider journeys env: LIVE_PROVIDERS: 1 diff --git a/.github/workflows/model-matrix.yml b/.github/workflows/model-matrix.yml index 8b9158c72..83fc40161 100644 --- a/.github/workflows/model-matrix.yml +++ b/.github/workflows/model-matrix.yml @@ -2,17 +2,10 @@ name: Model matrix # Generates one case with every model in the catalog, unpinned, so a model that # times out or returns invalid JSON fails here before users hit it. One run -# costs about $1, so it runs weekly and when model routing, prompts, or the ai -# function change. +# costs about $1, so it runs weekly and on demand. A branch that changes model +# routing, prompts, or the ai function runs it with +# `gh workflow run model-matrix.yml --ref `. on: - pull_request: - paths: - - supabase/functions/_shared/** - - supabase/functions/ai/** - - src/lib/ai/** - - packages/** - - e2e/model-matrix/** - - .github/workflows/model-matrix.yml schedule: - cron: "29 4 * * 1" workflow_dispatch: @@ -51,8 +44,9 @@ jobs: run: npx playwright install --with-deps chromium # db-gate.sh fails before starting anything when either key is missing. - # Accepted risk (2026-09-24): pull-request code runs with these keys, so - # anyone who can push a branch here could read them. Keep them low-limit. + # Accepted risk (2026-09-24): a dispatch can run branch code with these + # keys, so anyone who can push a branch here could read them. Keep them + # low-limit. - name: Case generation with every catalog model env: LIVE_PROVIDERS: 1 diff --git a/.github/workflows/prod-canary.yml b/.github/workflows/prod-canary.yml index 40f5f67bd..6d04d03b6 100644 --- a/.github/workflows/prod-canary.yml +++ b/.github/workflows/prod-canary.yml @@ -1,16 +1,21 @@ name: Production canary -# Checks the live site after every Vercel production deployment and daily. -# Read-only on deployments. The daily and manual runs also make two real AI -# requests (a streamed Ask and a case) as the dedicated canary account. +# Checks the live site after every push to main, once Vercel has deployed it, +# and daily. Read-only after a push. The daily and manual runs also make two +# real AI requests (a streamed Ask and a case) as the dedicated canary account. +# It used to start on deployment_status, which fires for every status of every +# Vercel deployment, both projects included; most of those runs only skipped. on: - deployment_status: + push: + branches: [ "main" ] schedule: - cron: "41 6 * * *" workflow_dispatch: permissions: contents: read + # Reads the Vercel production deployment of the pushed commit. + deployments: read concurrency: group: production-canary @@ -18,37 +23,63 @@ concurrency: jobs: canary: - # Vercel names the deployment environment after the project. The live site - # is debate-navigator; debatelab-staging's "production" is staging, so it - # must not start the canary. Plain 'Production' is the pre-2026-10 name. - if: >- - github.event_name != 'deployment_status' || - (github.event.deployment_status.state == 'success' && - (github.event.deployment.environment == 'Production – debate-navigator' || - github.event.deployment.environment == 'Production')) runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 20 steps: + # Vercel names the deployment environment after the project. The live + # site is debate-navigator; debatelab-staging's "production" is staging. + # Plain 'Production' is the pre-2026-10 name. + - name: Wait for the production deployment + id: deployment + if: github.event_name == 'push' + env: + GH_TOKEN: ${{ github.token }} + SHA: ${{ github.sha }} + shell: bash + run: | + for _ in $(seq 1 60); do + main="$(gh api "repos/${GITHUB_REPOSITORY}/commits/main" --jq .sha)" + if [ "$main" != "$SHA" ]; then + echo "::notice::main moved on to ${main}; its own run checks the newer deployment. Skipping ${SHA}." + echo "current=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + statuses="$(gh api "repos/${GITHUB_REPOSITORY}/deployments?sha=${SHA}&per_page=20" \ + --jq '[.[] | select(.environment == "Production – debate-navigator" or .environment == "Production")][0].statuses_url // ""')" + state=missing + if [ -n "$statuses" ]; then state="$(gh api "$statuses" --jq '.[0].state // "pending"')"; fi + case "$state" in + success) + echo "Vercel deployed ${SHA} to production." + echo "current=true" >> "$GITHUB_OUTPUT" + exit 0 ;; + failure|error) + echo "::error::Vercel's production deployment of ${SHA} ended in ${state}." + exit 1 ;; + esac + sleep 15 + done + echo "::error::No successful production deployment of ${SHA} after 15 minutes." + exit 1 + - name: Checkout code + if: steps.deployment.outputs.current != 'false' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: Set up Node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: 22 - - - name: Install dependencies - run: npm ci + - uses: ./.github/actions/node-deps + if: steps.deployment.outputs.current != 'false' - name: Install Chromium + if: steps.deployment.outputs.current != 'false' run: npx playwright install --with-deps chromium - name: Production canary + if: steps.deployment.outputs.current != 'false' env: PROD_SUPABASE_ANON_KEY: ${{ vars.PROD_SUPABASE_ANON_KEY }} PROD_CANARY_EMAIL: ${{ secrets.PROD_CANARY_EMAIL }} PROD_CANARY_PASSWORD: ${{ secrets.PROD_CANARY_PASSWORD }} - CANARY_AI_REQUESTS: ${{ github.event_name != 'deployment_status' }} + CANARY_AI_REQUESTS: ${{ github.event_name != 'push' }} run: npx playwright test --config=playwright.production.config.ts diff --git a/.github/workflows/staging-e2e.yml b/.github/workflows/staging-e2e.yml index 4531f3694..17a6ae8b1 100644 --- a/.github/workflows/staging-e2e.yml +++ b/.github/workflows/staging-e2e.yml @@ -4,6 +4,9 @@ on: workflow_run: workflows: ["Quality"] types: [completed] + # Without this, every pull-request Quality run starts a run here that + # only skips (about 600 a month). + branches: [main] workflow_dispatch: permissions: diff --git a/.github/workflows/staging-preview-note.yml b/.github/workflows/staging-preview-note.yml index 1adf68fba..8cfb8bc23 100644 --- a/.github/workflows/staging-preview-note.yml +++ b/.github/workflows/staging-preview-note.yml @@ -1,12 +1,13 @@ name: Staging preview note -# Staging runs main's backend (docs/staging.md). A PR that changes supabase/ -# gets a comment saying its staging preview still uses main's migrations and -# Edge Functions until it merges. +# Staging runs main's backend (docs/staging.md). A PR that opens with +# changes to supabase/ gets a comment saying its staging preview still uses +# main's migrations and Edge Functions until it merges. on: pull_request: - types: [opened, reopened, synchronize] + # The note posts once, so later pushes have nothing to add. + types: [opened, reopened] paths: - "supabase/migrations/**" - "supabase/functions/**" diff --git a/.superset/db-gate.sh b/.superset/db-gate.sh index 139b1964c..5f6915b2b 100755 --- a/.superset/db-gate.sh +++ b/.superset/db-gate.sh @@ -8,8 +8,10 @@ # `supabase db start` directly whenever more than one workspace might be # active. # -# --db-only stops after `npm run test:db` (pgTAP and the database scripts), -# still holding the lock and stopping the stack on exit. +# --db-only starts only Postgres (`supabase db start`) and stops after +# `npm run test:db` (pgTAP and the database scripts), still holding the lock +# and stopping the stack on exit. pgTAP needs nothing else, and Postgres is +# one image pull instead of the full stack's dozen. set -euo pipefail DB_ONLY=false @@ -95,9 +97,11 @@ else runtime_dir="$(mktemp -d)" fi # Shared CI runners intermittently hit registry rate limits while pulling images. +start_command=(start) +if [ "$DB_ONLY" = true ]; then start_command=(db start); fi stack_started=true for attempt in 1 2 3; do - if supabase start >>"$runtime_dir/supabase.log" 2>&1; then break; fi + if supabase "${start_command[@]}" >>"$runtime_dir/supabase.log" 2>&1; then break; fi if [ "$attempt" -eq 3 ]; then exit 1; fi echo "supabase start failed (attempt $attempt); retrying after $((attempt * 30))s" tail -n 3 "$runtime_dir/supabase.log" diff --git a/AGENTS.md b/AGENTS.md index ae27cec78..abb6a3325 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,7 +46,7 @@ That preview is pinned to staging. Do not pass `--prod`, and do not run `vercel ## Focused verification -Run the smallest relevant check first: +While working, run only the check that covers what you touched: the test file for the module (`npx vitest run `), `npm run lint` or `npm run typecheck` when types moved, one spec (`npx playwright test `). Each of these takes seconds. ```bash npm run docs:check @@ -59,9 +59,11 @@ npm run build P=$(bash .superset/e2e-port.sh) && CI=1 PORT=$P PLAYWRIGHT_BASE_URL=http://127.0.0.1:$P npm run test:e2e ``` -`npm run verify` runs the credential-free ladder with the desktop Chromium `@smoke` browser set (about a minute). `npm run verify:full` adds the full desktop and mobile suite. +Run `npm run verify` once, at the final head, before you push. It is the pull-request check run locally (about 100 seconds) and prints each failed step's log. `npm run verify:full` adds the full desktop and mobile browser suite; run it only when a change spans many pages or Adam asks. -If GitHub Actions jobs fail within 2 to 5 seconds with no logs, they did not run: the account is out of Actions minutes (billing), not failing on the code. In that case run the same checks locally at the final head (`npm run verify` and the database gate) and record the exact results in the PR. A local pass is not a CI pass; say which one you have. +Push once, when the work is done and `verify` passes. Every push reruns the whole pull-request check, so batch review fixes into one push. CI on a pull request is the `Quality` workflow (about 3 minutes), plus pgTAP when the schema changes. The full-stack journeys, the full browser suite, live providers, the agentic suite, and the model matrix run on a schedule, never on pull requests. When your change is squarely in one of those areas (AI or research code, a full-stack flow, many pages), dispatch that suite on your branch once, for example `gh workflow run live-e2e.yml --ref `, and report its result. [docs/operations.md](docs/operations.md#ci-workflows) lists them. + +If GitHub Actions jobs fail within 2 to 5 seconds with no logs, they did not run: the account is out of Actions minutes (billing), not failing on the code. In that case run the same checks locally at the final head (`npm run verify`, and `./.superset/db-gate.sh --db-only` when the schema changed) and record the exact results in the PR. A local pass is not a CI pass; say which one you have. When a CI job fails, start with `npm run ci:failures -- `. It prints the failing jobs and specs, downloads the failure artifact into `.ci-failures/`, and shows each failed test's error and the dialogs and headings on the page from its `error-context.md`. Read that before guessing at timeouts or infrastructure. @@ -69,7 +71,7 @@ Database migrations and pgTAP policies use a separate Docker-backed gate. In Sup ```bash ./.superset/db-gate.sh -./.superset/db-gate.sh --db-only # pgTAP and database scripts only +./.superset/db-gate.sh --db-only # pgTAP and database scripts on Postgres alone, about 35 s ``` Outside Superset, run the underlying commands directly: diff --git a/README.md b/README.md index 940c8985a..6f109ff18 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,10 @@ npm run lint npm run typecheck npm test npm run build -npm run verify # credential-free gate with the desktop Chromium smoke set +npm run verify # the pull-request check, locally, in about 100 seconds npm run verify:full # adds the full desktop and mobile browser suite ./.superset/db-gate.sh # migrations, pgTAP, and full-stack journeys on local Supabase +./.superset/db-gate.sh --db-only # migrations and pgTAP on Postgres alone ``` [`docs/testing.md`](docs/testing.md) explains what each gate proves and covers WebKit, protected staging, live-provider runs, and the production canary. diff --git a/docs/operations.md b/docs/operations.md index 27cf810a7..f3d5b6c33 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -10,25 +10,31 @@ Never infer authorization from access. Never use production to replace missing l ## CI workflows -[`.github/workflows/deploy.yml`](../.github/workflows/deploy.yml) (the `Quality` workflow) owns credential-free quality and the pull-request browser smoke set. It does not deploy anything; Vercel does. +A pull request gets one check, [`.github/workflows/deploy.yml`](../.github/workflows/deploy.yml) (the `Quality` workflow), plus a short database job when it changes the schema. Both finish in about 3 minutes. Everything slow, paid, or full-stack runs on a schedule and on demand, never on pull requests. `Quality` does not deploy anything; Vercel does. -The quality job uses Node 22, `npm ci`, and Deno 2.9.6, then runs: +`Quality` runs five jobs side by side on every pull request and every push to `main`. Each restores `node_modules` from a cache keyed on `package-lock.json` (`.github/actions/node-deps`) and runs `npm ci` only on a miss: -- `npm run docs:check` -- `npm run lint` -- `npm run typecheck` -- `npm test` -- `npm run test:collaboration` -- `npm run test:edge` -- `npm run build` +- `lint`: `npm run docs:check` and `npm run lint` +- `typecheck-build`: `npm run typecheck` and `npm run build` +- `unit`: `npm test` and `npm run test:collaboration` +- `edge`: `npm run test:edge` under Deno 2.9.6, with no npm install +- `browser-smoke` (pull requests only): the Playwright tests tagged `@smoke` on desktop Chromium, port `18080`. They check that the app boots, that sign-in works, that every public and signed-in page renders without errors, and that the first-run tour starts a debate. -On pull requests a parallel `browser-smoke` job runs the Playwright tests tagged `@smoke` (about 30) on desktop Chromium, port `18080`, with the browser download cached. [`.github/workflows/browser-nightly.yml`](../.github/workflows/browser-nightly.yml) runs the full desktop and mobile suite nightly and on demand. Pull requests that change only `docs/`, `plans/`, or Markdown skip `Quality` and run [`.github/workflows/docs.yml`](../.github/workflows/docs.yml), which runs `npm run docs:check` alone. +Per-step timings taken on 2 CPUs, the size of a private-repository runner, put the slowest job (`browser-smoke`) at about 2.5 minutes including setup. -[`.github/workflows/database.yml`](../.github/workflows/database.yml) starts isolated local Supabase for pgTAP and the full-stack browser journeys. It takes about 20 minutes, so it runs on pull requests and `main` pushes only when they change the backend, the stores or clients that talk to it, or the gate itself (the `paths` list in the file), plus weekly and on demand. +[`.github/workflows/database.yml`](../.github/workflows/database.yml) runs on pull requests that change migrations, `supabase/tests/`, the seed, `supabase/config.toml`, the database scripts, or the gate. The pull-request job runs `./.superset/db-gate.sh --db-only`, which starts Postgres alone and runs pgTAP and the concurrency scripts. The full gate (local Supabase, pgTAP, and the full-stack journeys on desktop and mobile, about 20 minutes) runs daily at 05:23 UTC and on demand. -[`.github/workflows/live-e2e.yml`](../.github/workflows/live-e2e.yml) runs the local stack against real providers nightly and on pull requests that change AI or research code. [`.github/workflows/model-matrix.yml`](../.github/workflows/model-matrix.yml) generates a case with every catalog model weekly and when AI code changes. [`.github/workflows/prod-canary.yml`](../.github/workflows/prod-canary.yml) checks production after each Vercel production deployment and daily. See [testing.md](testing.md) for all three. Both Supabase-starting jobs pull images from `public.ecr.aws`, because ghcr.io rate-limits shared runners. +Scheduled and on-demand suites: -[`.github/workflows/staging-e2e.yml`](../.github/workflows/staging-e2e.yml) owns protected staging. It accepts eligible successful main-branch `Quality` runs or a manual dispatch from `main`. It runs only when `STAGING_E2E_ENABLED=true` and all protected values exist. +- [`.github/workflows/browser-nightly.yml`](../.github/workflows/browser-nightly.yml): the full desktop and mobile browser suite and the service-worker offline test, daily at 02:47 UTC. +- [`.github/workflows/live-e2e.yml`](../.github/workflows/live-e2e.yml): the local stack against real providers, daily at 03:17 UTC. +- [`.github/workflows/agentic-e2e.yml`](../.github/workflows/agentic-e2e.yml): the agentic suite daily at 03:47 UTC, and the bug bash on Sundays. +- [`.github/workflows/model-matrix.yml`](../.github/workflows/model-matrix.yml): a case from every catalog model, Mondays at 04:29 UTC. +- [`.github/workflows/prod-canary.yml`](../.github/workflows/prod-canary.yml): after every push to `main`, once Vercel reports the commit's production deployment, and daily at 06:41 UTC. + +See [testing.md](testing.md) for what each proves. To run a scheduled suite on a branch before merging, for example when a change touches AI code, research, or the full-stack flows, dispatch it: `gh workflow run live-e2e.yml --ref ` (likewise `database.yml`, `browser-nightly.yml`, `agentic-e2e.yml`, `model-matrix.yml`). Supabase-starting jobs pull images from `public.ecr.aws`, because ghcr.io rate-limits shared runners. + +[`.github/workflows/staging-e2e.yml`](../.github/workflows/staging-e2e.yml) owns protected staging. It starts only from `main`-branch `Quality` runs (a `branches` filter, so pull-request runs no longer start it) or a manual dispatch from `main`. It runs only when `STAGING_E2E_ENABLED=true` and all protected values exist. [`.github/workflows/staging-preview-note.yml`](../.github/workflows/staging-preview-note.yml) comments once on a pull request that opens with `supabase/` changes. ## When GitHub Actions cannot run @@ -36,7 +42,7 @@ If Actions jobs fail within 2 to 5 seconds with no runner and no logs, the accou ## Frontend hosting on Vercel -Vercel builds and serves the frontend from the Git repository. A push to `main` deploys both projects: `debate-navigator` to `https://debateowl.com` (the previous domain, `https://debate.pi-stack.org`, 301s there with the path and query kept; see `vercel.json`; `https://www.debateowl.com` redirects to the apex through the project's Domains setting, not `vercel.json`, also keeping the path and query), and `debatelab-staging` to `https://debatelab-staging.vercel.app`. Any other branch waits for an explicit preview (see [Opt-in previews](#opt-in-previews)). For a staging-pinned preview on the tailnet, use `npm run dev:preview` (see [development.md](development.md#preview-on-the-tailnet)). The production canary runs after each successful Vercel `Production` deployment of `debate-navigator`. +Vercel builds and serves the frontend from the Git repository. A push to `main` deploys both projects: `debate-navigator` to `https://debateowl.com` (the previous domain, `https://debate.pi-stack.org`, 301s there with the path and query kept; see `vercel.json`; `https://www.debateowl.com` redirects to the apex through the project's Domains setting, not `vercel.json`, also keeping the path and query), and `debatelab-staging` to `https://debatelab-staging.vercel.app`. Any other branch waits for an explicit preview (see [Opt-in previews](#opt-in-previews)). For a staging-pinned preview on the tailnet, use `npm run dev:preview` (see [development.md](development.md#preview-on-the-tailnet)). The production canary runs after each push to `main`, once Vercel reports the commit's `debate-navigator` production deployment. [`vercel.json`](../vercel.json) owns: @@ -95,7 +101,7 @@ The staging browser suite disables trace, screenshot, and video capture. It crea Do not enable the staging repository variable before every prerequisite is present. A skipped staging job is not a pass. -`STAGING_E2E_ENABLED` has been `true` since 2026-09-18. The job runs after every successful `Quality` run from a push to `main`, and it first waits up to 20 minutes for a successful `staging-sync` status on that commit. Quality runs from pull requests also trigger it, and those show as skipped by design. Check the latest real result with `gh run list --workflow staging-e2e.yml` and `npm run ci:failures -- `, not this page. The run uploads no artifact, so the job log is the only evidence. On failure the log names the failed step and the URL path, for example `Staging step "add a source to the debate" failed at path /app/library`. +`STAGING_E2E_ENABLED` has been `true` since 2026-09-18. The job runs after every successful `Quality` run from a push to `main`, and it first waits up to 20 minutes for a successful `staging-sync` status on that commit. Check the latest real result with `gh run list --workflow staging-e2e.yml` and `npm run ci:failures -- `, not this page. The run uploads no artifact, so the job log is the only evidence. On failure the log names the failed step and the URL path, for example `Staging step "add a source to the debate" failed at path /app/library`. No staging run passed after 2026-09-23. Until 2026-10-01 the job did not run, because the `Quality` run it waits for never started. Once Actions came back, it failed on every `main` push through 2026-10-02. The cause was the spec, not staging: since the Current Debate change (#72), Library disables **Add source** until a debate is current, and the spec never started one. The spec now starts a uniquely named debate and deletes it afterwards. The suite drives the real UI, so a change to a flow it covers can break it with no backend drift; run it locally before blaming staging. diff --git a/docs/standards/security.md b/docs/standards/security.md index f771230cd..b975c7e77 100644 --- a/docs/standards/security.md +++ b/docs/standards/security.md @@ -22,7 +22,7 @@ Part of the [coding standards](../../CODING_STANDARDS.md). AGENTS.md "Hard bound These were raised in review, judged valid, and accepted on purpose. Changing them needs Adam's decision, not a reviewer's. -- Live provider keys reach same-repository PR jobs in `live-e2e.yml`. Adam accepted this on PR #68; the mitigation is dedicated low-limit keys ([#68](https://github.com/Programcode/DebateLab/pull/68#discussion_r4090563258)). +- Live provider keys reach branch code in `live-e2e.yml`, `model-matrix.yml` and `agentic-e2e.yml` when someone with push access dispatches them on a branch. Since 2026-10-29 these no longer run on pull requests. Adam accepted the original pull-request exposure on PR #68; the mitigation is dedicated low-limit keys ([#68](https://github.com/Programcode/DebateLab/pull/68#discussion_r4090563258)). - The production canary account's credentials reach `prod-canary.yml` through Actions secrets, approved 2026-09-24 and recorded in AGENTS.md ([#68](https://github.com/Programcode/DebateLab/pull/68#discussion_r4090664336)). - Manual canary runs can exhaust the account's daily quota; that shows as a quota error, not a false pass ([#68](https://github.com/Programcode/DebateLab/pull/68#discussion_r4093482054)). - CodeRabbit rated the shared preview-origin allowlist from PR #95 as a high retained concern: an approved production preview runs unmerged code that can receive demo-session handoffs ([#95 security review](https://github.com/Programcode/DebateLab/pull/95)). [docs/operations.md](../operations.md#opt-in-previews) describes the current setup. This one has no recorded decision yet. diff --git a/docs/testing.md b/docs/testing.md index d817cfa03..5273fccdb 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -110,7 +110,7 @@ This proves the Vite production bundle compiles, prerenders the public pages, ch npm run test:sw ``` -Run this after `npm run build`. It serves `dist/` through Vite preview (which applies the `vercel.json` rewrites, so app routes get `app-shell.html` as in production) and proves that a v2 worker cache upgrades to this build's caches and claims the page; that after one visit to `/app`, Home, Library, History, Settings and Timer open offline as fresh navigations, the timer runs, the offline indicator shows, and switching to Spanish works; that a visited public page answers offline only with itself and an unvisited one gets the offline page; that an unvisited route shows recovery UI and an intent-prefetched one opens; and that private API paths and `/auth/callback` do not enter Cache Storage. It uses a stub session and blocks external requests, so it is not authenticated backend or deployed-host proof. +Run this after `npm run build`; `npm run verify:full` and the nightly browser workflow run it. It serves `dist/` through Vite preview (which applies the `vercel.json` rewrites, so app routes get `app-shell.html` as in production) and proves that a v2 worker cache upgrades to this build's caches and claims the page; that after one visit to `/app`, Home, Library, History, Settings and Timer open offline as fresh navigations, the timer runs, the offline indicator shows, and switching to Spanish works; that a visited public page answers offline only with itself and an unvisited one gets the offline page; that an unvisited route shows recovery UI and an intent-prefetched one opens; and that private API paths and `/auth/callback` do not enter Cache Storage. It uses a stub session and blocks external requests, so it is not authenticated backend or deployed-host proof. ## Credential-free browser gate @@ -118,7 +118,7 @@ Run this after `npm run build`. It serves `dist/` through Vite preview (which ap P=$(bash .superset/e2e-port.sh) && CI=1 PORT=$P PLAYWRIGHT_BASE_URL=http://127.0.0.1:$P npm run test:e2e ``` -Pull requests run only tests tagged `@smoke` on desktop Chromium (`npm run test:e2e:smoke`, about 30 tests). Tag a test `@smoke` only when it covers a whole user path that would otherwise go unchecked until the nightly run. +Pull requests run only tests tagged `@smoke` on desktop Chromium (`npm run test:e2e:smoke`, 26 tests): the startup shell, Login, every public and signed-in page in English, and the first-run tour. Each test pays a full page load from the dev server, about 6 seconds on a 2-CPU runner, so the set stays small enough for a 3-minute check. Tool journeys (Crossfire, Sparring, the free tools, layout regressions) run in the nightly suite. Tag a test `@smoke` only when a break there would stop most users and nothing else on the pull request would catch it. `.superset/e2e-port.sh` gives each worktree its own port from `18100-18179`, recorded in `~/.superset/debatelab-ports/allocations.json`, so two worktrees can run the gate at once. Without `PORT`, Playwright falls back to `18080`. Playwright starts a new Vite server on that port and runs: @@ -135,7 +135,7 @@ The one-time DebateOwl welcome stays closed whenever `VITE_E2E` or `VITE_LOCAL_P `e2e/first-run.spec.ts` drives the same first run against stubbed Supabase routes in English and Hungarian, so it proves the client flow only. -`e2e/free-tools.spec.ts` (tag `@smoke`) opens every `/tools` page signed out and checks its heading, canonical URL and JSON-LD, and that it makes no Supabase request. On a paused Playwright clock it runs the timer past the BP protected minute, through a reload, and runs the two Public Forum prep banks; it also draws a motion set and reloads its permalink, times a pasted speech, lays out a BP flow sheet under print media, and checks that `/embed/debate-timer` is `noindex` and requests nothing from another origin. The format data, timer transitions, custom-format links, motion filters and speech-time arithmetic have unit tests in `src/test/free-tools/`. The dev server cannot prove the response headers: `src/test/vercel-deployment-config.test.ts` checks which paths may be framed. +`e2e/free-tools.spec.ts` (nightly) opens every `/tools` page signed out and checks its heading, canonical URL and JSON-LD, and that it makes no Supabase request. On a paused Playwright clock it runs the timer past the BP protected minute, through a reload, and runs the two Public Forum prep banks; it also draws a motion set and reloads its permalink, times a pasted speech, lays out a BP flow sheet under print media, and checks that `/embed/debate-timer` is `noindex` and requests nothing from another origin. The format data, timer transitions, custom-format links, motion filters and speech-time arithmetic have unit tests in `src/test/free-tools/`. The dev server cannot prove the response headers: `src/test/vercel-deployment-config.test.ts` checks which paths may be framed. `src/test/i18n-keys.test.ts` fails when a literal `t()` key in `src/` is missing from `en.json`, or when a call that passes `count` lacks its `_one` and `_other` forms. Dynamic keys such as ``t(`nav.${key}`)`` are out of its reach. `src/test/i18n.test.ts` requires the same keys in all three catalogs. Together they prove every literal key has a Spanish and Hungarian entry, not that the translation is natural. @@ -151,20 +151,16 @@ AI client and Edge tests prove that every request path carries a validated langu npm run verify ``` -The gate runs, in order: +This is the pull-request check run locally, in about 100 seconds on the VPS. [`scripts/verify.mjs`](../scripts/verify.mjs) runs two lanes side by side: -1. documentation checks -2. the guide jargon check (`npm run seo:jargon`): no generated guide in `public/` other than `/lincoln-douglas-debate` and `/policy-debate` may use framework, contention, warrant or impact calculus more than once in its visible text or its JSON-LD ([`scripts/check_jargon.py`](../scripts/check_jargon.py)) -3. lint -4. type checking -5. unit tests -6. collaboration service tests -7. native Edge Function tests -8. production build -9. production service-worker offline smoke -10. the desktop Chromium `@smoke` set (`npm run verify:full` adds the full desktop and mobile suite) +1. documentation checks, the guide jargon check, lint, production build, native Edge Function tests +2. type checking, unit tests, collaboration service tests, the desktop Chromium `@smoke` set -It does not start local Supabase, run pgTAP, run WebKit, or use protected staging. +The jargon check (`npm run seo:jargon`) fails when a generated guide in `public/` other than `/lincoln-douglas-debate` and `/policy-debate` uses framework, contention, warrant or impact calculus more than once in its visible text or its JSON-LD ([`scripts/check_jargon.py`](../scripts/check_jargon.py)). + +Each step logs to `.verify/npm-verify/.log`. The terminal shows one line per step, then the end of each failed step's log. Both lanes run to the end, so one run reports every failure. Two lanes keep the peak about 4 GB above idle; NODE_OPTIONS defaults to `--max-old-space-size=3072`. + +`npm run verify:full` adds the production service-worker offline test and the full desktop and mobile suite. Neither starts local Supabase, runs pgTAP, runs WebKit, or uses protected staging. ## Database and RLS gate @@ -180,7 +176,7 @@ In Superset, run: ./.superset/db-gate.sh ``` -The wrapper acquires `~/.superset/debatelab-db.lock`, starts full local Supabase, runs `npm run test:db`, then holds the same lock while a controlled Mailpit relay, collaboration process, fake model provider, and Vite client run the real-Auth desktop/mobile workspace and full-stack feature journeys. The `Database` workflow runs this same gate on pull requests that touch its paths, and retains its logs and browser traces on failure. It stops every owned process and the stack on exit, including after failure. To run only the database portion under the same lock, use `./.superset/db-gate.sh --db-only`. Outside Superset, run it with: +The wrapper acquires `~/.superset/debatelab-db.lock`, starts full local Supabase, runs `npm run test:db`, then holds the same lock while a controlled Mailpit relay, collaboration process, fake model provider, and Vite client run the real-Auth desktop/mobile workspace and full-stack feature journeys. The `Database` workflow runs this full gate daily and on demand, and retains its logs and browser traces on failure. It stops every owned process and the stack on exit, including after failure. To run only the database portion under the same lock, use `./.superset/db-gate.sh --db-only`: it starts Postgres alone (`supabase db start`), which pgTAP and the database scripts need and nothing more, in about 35 seconds locally. Pull requests that change the schema run this. Outside Superset, run it with: ```bash npx supabase db start @@ -233,7 +229,7 @@ The specs cover web-researched case generation, verified Factoids citations, Ask A live failure means the pipeline broke with this model. It does not prove how production models behave, because each model has its own latency and JSON quality. -`.github/workflows/live-e2e.yml` runs this nightly at 03:17 UTC, on demand, and on pull requests that change `supabase/functions/`, `src/lib/ai/`, `e2e/live/`, the fake provider, or the gate, using the `LIVE_OPENROUTER_API_KEY` and `LIVE_EXA_API_KEY` repository secrets. The gate exits before starting anything when either key is missing, so a misconfigured job fails instead of skipping. +`.github/workflows/live-e2e.yml` runs this nightly at 03:17 UTC and on demand (`gh workflow run live-e2e.yml --ref ` checks a branch before merging), using the `LIVE_OPENROUTER_API_KEY` and `LIVE_EXA_API_KEY` repository secrets. The gate exits before starting anything when either key is missing, so a misconfigured job fails instead of skipping. ### Model matrix @@ -243,7 +239,7 @@ LIVE_PROVIDERS=1 LIVE_MODEL=as-requested LIVE_OPENROUTER_API_KEY=... LIVE_EXA_AP `LIVE_MODEL=as-requested` switches the gate to `e2e/model-matrix/`. The proxy forwards whatever model the Edge Function chose, and the spec picks each model in `MODEL_CATALOG` from the Generate page's model menu and generates one researched case. A model that exhausts the `ai` function's shared 130-second request budget or returns invalid JSON fails its test. Each test prints the provider latency (`latency : `) so the run log shows how close each model gets to the budget. -`.github/workflows/model-matrix.yml` runs this on Mondays at 04:29 UTC, on demand, and on pull requests that change `supabase/functions/_shared/`, `supabase/functions/ai/`, or `src/lib/ai/`. It is not a required check, because it runs only on those paths. One run costs about $1 in model and search usage. +`.github/workflows/model-matrix.yml` runs this on Mondays at 04:29 UTC and on demand. A branch that changes model routing, prompts, or the `ai` function can run it with `gh workflow run model-matrix.yml --ref `. One run costs about $1 in model and search usage. ## Agentic browser tests @@ -278,7 +274,7 @@ To write a test, import `test`, `expect`, and the fixtures from [`agentic/suppor A bug bash runs several explorations in parallel, one charter per area and persona, then proves each finding with a repro test. Ask an agent to "bug bash "; the `e2e` skill's `bug-bash` topic has the procedure. A finding counts only when its repro test fails for the reported reason. Tag repro tests `{ tags: ["repro"] }` so the scheduled suite skips them until the fix lands, then remove the tag so they become regression tests. The client runs on a Vite dev server, so a route that is still compiling can look like a dead link to an explorer. Reproduce before you believe it. -[`.github/workflows/agentic-e2e.yml`](../.github/workflows/agentic-e2e.yml) runs the suite nightly at 03:47 UTC and on pull requests that change the app (`src/`, `public/`, Edge Functions, migrations, dependencies) or the suite, in four jobs: the page sweep and everything else, on each viewport. Every Sunday at 05:13 UTC it runs a bug bash instead: one exploration per persona, each with its own charter. On demand with a goal, it runs one exploration with the chosen persona and viewport. It uses the `LIVE_OPENROUTER_API_KEY` secret (the `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` repository variables override the models), fails when the key is missing, and writes the report to the job summary. Explorations never fail the workflow: their findings are leads to reproduce. The suite is not a required check: model judgments can flake, and a red nightly is a finding to triage. +[`.github/workflows/agentic-e2e.yml`](../.github/workflows/agentic-e2e.yml) runs the suite nightly at 03:47 UTC and on demand (`gh workflow run agentic-e2e.yml --ref `), never on pull requests, in four jobs: the page sweep and everything else, on each viewport. Every Sunday at 05:13 UTC it runs a bug bash instead: one exploration per persona, each with its own charter. On demand with a goal, it runs one exploration with the chosen persona and viewport. It uses the `LIVE_OPENROUTER_API_KEY` secret (the `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` repository variables override the models), fails when the key is missing, and writes the report to the job summary. Explorations never fail the workflow: their findings are leads to reproduce. The suite is not a required check: model judgments can flake, and a red nightly is a finding to triage. e2e is pre-1.0 and its minor releases have broken APIs, so it is pinned exactly. Read its changelog before upgrading, refresh the vendored skill as its header says, and expect the replay cache to re-record after a minor bump. @@ -300,7 +296,7 @@ npx playwright test --config=playwright.production.config.ts The last two checks are the only production writes, and they touch only the canary account: its `ai_requests` usage rows and whatever the Generate page saves for it. The dedicated canary account has Pro promo access capped at 10 requests a day. Captures are disabled because they could retain its session. -`.github/workflows/prod-canary.yml` runs after every successful Vercel production deployment of the `debate-navigator` project (GitHub environment `Production – debate-navigator`; the `debatelab-staging` project's production deployments do not start it), daily at 06:41 UTC, and on demand. Main can deploy more than 10 times a day, so deployment runs set `CANARY_AI_REQUESTS=false` and skip the two AI checks. The signed-in session check still runs on every deployment. Only the daily and manual runs make real AI requests: two per run, plus at most two retries. It reads the password and email from secrets and the public anon key from the `PROD_SUPABASE_ANON_KEY` repository variable. +`.github/workflows/prod-canary.yml` runs after every push to `main`, daily at 06:41 UTC, and on demand. A push run first waits, up to 15 minutes, for Vercel's production deployment of that commit to the `debate-navigator` project (GitHub environment `Production – debate-navigator`), and skips when `main` moves on first, because the newer commit's run checks the newer deployment. Main can deploy more than 10 times a day, so push runs set `CANARY_AI_REQUESTS=false` and skip the two AI checks. The signed-in session check still runs after every deployment. Only the daily and manual runs make real AI requests: two per run, plus at most two retries. It reads the password and email from secrets and the public anon key from the `PROD_SUPABASE_ANON_KEY` repository variable. The local stack begins at port `54321` and is not parallel-safe across worktrees. The task that starts it must stop it before finishing or abandoning the run. Use `supabase stop --no-backup` unless another active task explicitly owns the same stack. @@ -371,11 +367,11 @@ Tests are not the proof that a change works; using it is. Two local recipes that ## CI evidence -[`.github/workflows/deploy.yml`](../.github/workflows/deploy.yml), the `Quality` workflow, runs documentation, lint, type, unit, collaboration, Edge Function, and build checks. On pull requests a parallel `browser-smoke` job runs the `@smoke` set on desktop Chromium. [`.github/workflows/browser-nightly.yml`](../.github/workflows/browser-nightly.yml) runs the full desktop and mobile suite nightly and on demand. [`.github/workflows/database.yml`](../.github/workflows/database.yml) runs the database gate when backend-facing paths change. +[`.github/workflows/deploy.yml`](../.github/workflows/deploy.yml), the `Quality` workflow, is the pull-request check: documentation, lint, type, unit, collaboration, Edge Function, and build checks, and the `@smoke` set on desktop Chromium, in parallel jobs. [`.github/workflows/database.yml`](../.github/workflows/database.yml) runs pgTAP on pull requests that change the schema, and the full database gate daily. [`.github/workflows/browser-nightly.yml`](../.github/workflows/browser-nightly.yml) runs the full desktop and mobile suite daily. [operations.md](operations.md#ci-workflows) lists every schedule and how to dispatch one on a branch. [`.github/workflows/staging-e2e.yml`](../.github/workflows/staging-e2e.yml) owns protected staging conditions. -[`.github/workflows/agentic-e2e.yml`](../.github/workflows/agentic-e2e.yml) runs the agentic suite and on-demand explorations. [`.github/workflows/live-e2e.yml`](../.github/workflows/live-e2e.yml) runs the live provider journeys, and [`.github/workflows/prod-canary.yml`](../.github/workflows/prod-canary.yml) runs the production canary. `main` has no branch protection or rulesets (checked 2026-10-01; GitHub Free does not enforce them on private repositories), so no check is required. Path-filtered workflows that skip therefore don't block merges. Read the PR's checks before merging. +[`.github/workflows/agentic-e2e.yml`](../.github/workflows/agentic-e2e.yml) runs the agentic suite and on-demand explorations. [`.github/workflows/live-e2e.yml`](../.github/workflows/live-e2e.yml) runs the live provider journeys, and [`.github/workflows/prod-canary.yml`](../.github/workflows/prod-canary.yml) runs the production canary. `main` has no branch protection or rulesets (checked 2026-10-01; GitHub Free does not enforce them on private repositories), so no check is required. Path-filtered workflows that don't run therefore don't block merges. Read the PR's checks before merging. To see why a run failed, use `npm run ci:failures` (current branch's PR), `npm run ci:failures -- ` or `npm run ci:failures -- `. It works while the run is still going, as soon as a job fails. It lists failed jobs and steps, prints the Playwright or Vitest summary from each job log, downloads the `*-failure-` artifacts into ignored `.ci-failures/`, and prints each failed test's error plus the dialogs and headings from its `error-context.md` page snapshot. A dialog the spec never answered shows up there. Protected staging uploads no artifact, so only its job log is available. diff --git a/e2e/admin-announcements.spec.ts b/e2e/admin-announcements.spec.ts index 8e5969b20..ba7d59198 100644 --- a/e2e/admin-announcements.spec.ts +++ b/e2e/admin-announcements.spec.ts @@ -36,7 +36,7 @@ function countAnnouncementListRequests(page: Page) { return () => count; } -test('user sees and permanently dismisses an active announcement', { tag: '@smoke' }, async ({ page }) => { +test('user sees and permanently dismisses an active announcement', async ({ page }) => { let dismissBody: Record | null = null; await page.route('**/functions/v1/announcements*', async (route) => { const request = route.request(); diff --git a/e2e/crossfire.spec.ts b/e2e/crossfire.spec.ts index 5fdd42899..618ba7a54 100644 --- a/e2e/crossfire.spec.ts +++ b/e2e/crossfire.spec.ts @@ -129,7 +129,7 @@ async function prepare(page: Page, requests: CrossfireRequest[], failFirstAnswer await page.getByTestId('crossfire-argument').fill('Private cars create congestion and unsafe streets.'); } -test('Question the AI completes a grounded Crossfire debrief', { tag: '@smoke' }, async ({ page }, testInfo) => { +test('Question the AI completes a grounded Crossfire debrief', async ({ page }, testInfo) => { const requests: CrossfireRequest[] = []; await prepare(page, requests); await page.getByTestId('mode-question-ai').click(); diff --git a/e2e/first-run.spec.ts b/e2e/first-run.spec.ts index de286fba5..b25117b83 100644 --- a/e2e/first-run.spec.ts +++ b/e2e/first-run.spec.ts @@ -162,7 +162,7 @@ for (const locale of ['en', 'hu'] as const) { await expect(tooltip).toHaveCount(0); }); - test(`[${locale}] a tool offers a first debate from the typed motion instead of the switch dialog`, locale === 'en' ? { tag: '@smoke' } : {}, async ({ page }) => { + test(`[${locale}] a tool offers a first debate from the typed motion instead of the switch dialog`, async ({ page }) => { const backend = await freshStart(page, locale, true); await page.goto('/app/generate', { waitUntil: 'domcontentloaded' }); diff --git a/e2e/free-tools.spec.ts b/e2e/free-tools.spec.ts index c6baf0d87..40770f08c 100644 --- a/e2e/free-tools.spec.ts +++ b/e2e/free-tools.spec.ts @@ -37,7 +37,7 @@ function watchErrors(page: Page): string[] { return errors; } -test.describe('free debate tools', { tag: '@smoke' }, () => { +test.describe('free debate tools', () => { for (const tool of TOOL_PAGES) { test(`${tool.path} renders signed out with its head and structured data`, async ({ page }) => { const supabase = watchSupabase(page); diff --git a/e2e/quality-backlog.spec.ts b/e2e/quality-backlog.spec.ts index 542e8c911..1613c2252 100644 --- a/e2e/quality-backlog.spec.ts +++ b/e2e/quality-backlog.spec.ts @@ -107,7 +107,7 @@ test('Flowchart deletes a selected node at once and Undo puts it back', async ({ // VS-05: a long motion and long item titles once widened the dashboard past a phone screen. for (const locale of locales) { - test(`Current Debate fits a 360px phone with sources attached in ${locale}`, { tag: '@smoke' }, async ({ page }) => { + test(`Current Debate fits a 360px phone with sources attached in ${locale}`, async ({ page }) => { const motion = 'This House believes that national governments should nationalise all critical digital infrastructure including undersea cables'; await page.setViewportSize({ width: 360, height: 780 }); await setLocale(page, locale); @@ -142,7 +142,7 @@ for (const locale of locales) { } // VS-09: picking a language closes the menu; the choice already persists. -test('language menu closes once a language is chosen', { tag: '@smoke' }, async ({ page }) => { +test('language menu closes once a language is chosen', async ({ page }) => { await setLocale(page, 'en'); await page.goto('/'); // Below lg the section links and the language control sit in the menu. diff --git a/e2e/sparring.spec.ts b/e2e/sparring.spec.ts index 6c991ba5f..febe94665 100644 --- a/e2e/sparring.spec.ts +++ b/e2e/sparring.spec.ts @@ -42,7 +42,7 @@ test.beforeEach(async ({ page }) => { await stubSupabaseSession(page); }); -test('completes an LD sparring round through mocked opponent turns and judging', { tag: '@smoke' }, async ({ page }) => { +test('completes an LD sparring round through mocked opponent turns and judging', async ({ page }) => { let responseNumber = 0; const actions: string[] = []; diff --git a/package.json b/package.json index 778cae643..f5b9972b7 100644 --- a/package.json +++ b/package.json @@ -36,8 +36,8 @@ "test:sw": "node scripts/test-service-worker-offline.mjs", "test:e2e:staging": "playwright test --project=staging", "test:agentic": "bash .claude/skills/verify-debatelab/bin/verify.sh e2e run", - "verify": "npm run docs:check && npm run seo:jargon && npm run lint && npm run typecheck && npm test && npm run test:collaboration && npm run test:edge && npm run build && npm run test:sw && P=$(bash .superset/e2e-port.sh) && CI=1 PORT=$P PLAYWRIGHT_BASE_URL=http://127.0.0.1:$P npm run test:e2e:smoke", - "verify:full": "npm run verify && P=$(bash .superset/e2e-port.sh) && CI=1 PORT=$P PLAYWRIGHT_BASE_URL=http://127.0.0.1:$P npm run test:e2e" + "verify": "node scripts/verify.mjs", + "verify:full": "npm run verify && npm run test:sw && P=$(bash .superset/e2e-port.sh) && CI=1 PORT=$P PLAYWRIGHT_BASE_URL=http://127.0.0.1:$P npm run test:e2e" }, "dependencies": { "@fontsource-variable/fredoka": "5.3.0", diff --git a/scripts/prerender.mjs b/scripts/prerender.mjs index 5fe0b7267..3b3369b68 100644 --- a/scripts/prerender.mjs +++ b/scripts/prerender.mjs @@ -141,6 +141,11 @@ const server = await createServer({ mode: 'production', appType: 'custom', logLevel: 'warn', + // Its own dependency cache: in the shared node_modules/.vite this server + // re-optimizes the deps a running dev server is serving, and that server's + // pages then fail to load their lazy routes (npm run verify builds while the + // smoke tests run). + cacheDir: 'node_modules/.vite-prerender', plugins: [builtAssetUrls()], server: { middlewareMode: true, hmr: false, ws: false }, }); diff --git a/scripts/verify.mjs b/scripts/verify.mjs new file mode 100644 index 000000000..fe020327e --- /dev/null +++ b/scripts/verify.mjs @@ -0,0 +1,101 @@ +#!/usr/bin/env node +// `npm run verify`: the pull-request check (deploy.yml), run locally. Two +// lanes run side by side, each a chain of steps, so the run takes as long as +// the slower lane instead of the sum of every step. Each step writes its +// output to .verify/npm-verify/.log; the terminal shows one line per +// step, a timing summary, and the end of each failed step's log. Every lane +// runs to the end, so one run reports every failure. +// +// The VPS is short on memory. Alone, lint, build and smoke each take about +// 2 GB and Vitest with three workers about 1.8 GB, so the lanes are ordered to +// run at most two of them at once (about 4 GB above idle). A third lane was no +// faster and took 6 GB. NODE_OPTIONS defaults to a 3 GB heap per Node process. +import { execFileSync, spawn } from "node:child_process"; +import { createWriteStream, mkdirSync, readFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; + +const LOG_DIR = ".verify/npm-verify"; +const TAIL_LINES = 40; + +const port = execFileSync("bash", [".superset/e2e-port.sh"], { encoding: "utf8" }).trim(); +const smokeEnv = { CI: "1", PORT: port, PLAYWRIGHT_BASE_URL: `http://127.0.0.1:${port}` }; + +const LANES = [ + [ + { name: "docs", command: "npm run docs:check" }, + { name: "jargon", command: "npm run seo:jargon" }, + { name: "lint", command: "npm run lint" }, + { name: "build", command: "npm run build" }, + { name: "edge", command: "npm run test:edge" }, + ], + [ + { name: "typecheck", command: "npm run typecheck" }, + { name: "unit", command: "npm test -- --maxWorkers=3" }, + { name: "collaboration", command: "npm run test:collaboration" }, + { name: "smoke", command: "npm run test:e2e:smoke", env: smokeEnv }, + ], +]; + +const env = { ...process.env, NODE_OPTIONS: process.env.NODE_OPTIONS || "--max-old-space-size=3072" }; +const running = new Set(); +const results = []; +const seconds = (ms) => `${(ms / 1000).toFixed(1)}s`; + +rmSync(LOG_DIR, { recursive: true, force: true }); +mkdirSync(LOG_DIR, { recursive: true }); + +function runStep(step) { + const log = join(LOG_DIR, `${step.name}.log`); + const started = Date.now(); + return new Promise((resolve) => { + const out = createWriteStream(log); + // Its own process group, so an interrupt stops the step's whole tree. + const child = spawn("bash", ["-c", step.command], { + detached: true, + env: { ...env, ...step.env }, + stdio: ["ignore", "pipe", "pipe"], + }); + running.add(child); + child.stdout.pipe(out, { end: false }); + child.stderr.pipe(out, { end: false }); + child.on("close", (code) => { + running.delete(child); + out.end(); + const result = { ...step, log, ok: code === 0, ms: Date.now() - started }; + results.push(result); + console.log(`${result.ok ? "ok " : "FAIL"} ${step.name.padEnd(14)} ${seconds(result.ms).padStart(7)}`); + resolve(); + }); + }); +} + +async function runLane(steps) { + for (const step of steps) await runStep(step); +} + +for (const signal of ["SIGINT", "SIGTERM"]) { + process.on(signal, () => { + for (const child of running) { + try { + process.kill(-child.pid, "SIGTERM"); + } catch { + // Already gone. + } + } + process.exit(130); + }); +} + +const started = Date.now(); +console.log(`verify: ${LANES.flat().length} steps in ${LANES.length} lanes, logs in ${LOG_DIR}/`); +await Promise.all(LANES.map(runLane)); + +const failed = results.filter((result) => !result.ok); +for (const result of failed) { + const lines = readFileSync(result.log, "utf8").trimEnd().split("\n"); + console.log(`\n--- ${result.name} failed: last ${Math.min(TAIL_LINES, lines.length)} lines of ${result.log}`); + console.log(lines.slice(-TAIL_LINES).join("\n")); +} +const slowest = [...results].sort((a, b) => b.ms - a.ms).slice(0, 3).map((r) => `${r.name} ${seconds(r.ms)}`); +console.log(`\nverify: ${failed.length ? `${failed.length} failed (${failed.map((r) => r.name).join(", ")})` : "all passed"} in ${seconds(Date.now() - started)}. Slowest: ${slowest.join(", ")}.`); +process.exit(failed.length ? 1 : 0); diff --git a/src/test/setup.ts b/src/test/setup.ts index f45617299..f54ab5061 100644 --- a/src/test/setup.ts +++ b/src/test/setup.ts @@ -1,7 +1,18 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ +import { webcrypto } from "node:crypto"; +import { ReadableStream, TransformStream, WritableStream } from "node:stream/web"; import "@testing-library/jest-dom"; import "../i18n"; +// The vmThreads pool (vitest.config.ts) runs each file in a bare jsdom +// context, which lacks these Node globals that the browsers we ship to have. +if (typeof globalThis.crypto?.randomUUID !== "function") { + Object.defineProperty(globalThis, "crypto", { configurable: true, value: webcrypto }); +} +for (const [name, value] of Object.entries({ ReadableStream, TransformStream, WritableStream })) { + if (!(name in globalThis)) Object.defineProperty(globalThis, name, { configurable: true, writable: true, value }); +} + const createStorage = () => { let store: Record = {}; return { diff --git a/vitest.config.ts b/vitest.config.ts index a924a324e..adc491dc3 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -6,6 +6,10 @@ export default defineConfig({ plugins: [react()], test: { environment: "jsdom", + // Each file still gets a fresh jsdom context, but the workers are reused + // instead of rebuilding jsdom per file: 43s instead of 95s on 2 CPUs. + // src/test/setup.ts adds the Node globals a bare VM context lacks. + pool: "vmThreads", globals: true, setupFiles: ["./src/test/setup.ts"], // Globals and env stubbed with vi.stubGlobal / vi.stubEnv reset after From 074cc66f6f2e1092bd3d4482f4979761e33e5d78 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 29 Oct 2026 19:08:16 +0000 Subject: [PATCH 2/2] docs: verify runs the offline service-worker test --- docs/testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing.md b/docs/testing.md index 43fa8d287..e41638259 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -110,7 +110,7 @@ This proves the Vite production bundle compiles, prerenders the public pages, ch npm run test:sw ``` -Run this after `npm run build`; `npm run verify:full` and the nightly browser workflow run it. It serves `dist/` through Vite preview (which applies the `vercel.json` rewrites, so app routes get `app-shell.html` as in production) and proves that a v2 worker cache upgrades to this build's caches and claims the page; that after one visit to `/app`, Home, Library, History, Settings and Timer open offline as fresh navigations, the timer runs, the offline indicator shows, and switching to Spanish works; that a visited public page answers offline only with itself and an unvisited one gets the offline page; that an unvisited route shows recovery UI and an intent-prefetched one opens; and that private API paths and `/auth/callback` do not enter Cache Storage. It uses a stub session and blocks external requests, so it is not authenticated backend or deployed-host proof. +Run this after `npm run build`; `npm run verify` runs it right after its build step, and the nightly browser workflow runs it too. It serves `dist/` through Vite preview (which applies the `vercel.json` rewrites, so app routes get `app-shell.html` as in production) and proves that a v2 worker cache upgrades to this build's caches and claims the page; that after one visit to `/app`, Home, Library, History, Settings and Timer open offline as fresh navigations, the timer runs, the offline indicator shows, and switching to Spanish works; that a visited public page answers offline only with itself and an unvisited one gets the offline page; that an unvisited route shows recovery UI and an intent-prefetched one opens; and that private API paths and `/auth/callback` do not enter Cache Storage. It uses a stub session and blocks external requests, so it is not authenticated backend or deployed-host proof. ## Credential-free browser gate